1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: zalloc.h
60 * Author: Avadis Tevanian, Jr.
61 * Date: 1985
62 *
63 */
64
65 #ifdef KERNEL_PRIVATE
66
67 #ifndef _KERN_ZALLOC_H_
68 #define _KERN_ZALLOC_H_
69
70 #include <mach/machine/vm_types.h>
71 #include <mach_debug/zone_info.h>
72 #include <kern/kern_types.h>
73 #include <sys/cdefs.h>
74 #include <os/alloc_util.h>
75 #include <os/atomic.h>
76
77 #ifdef XNU_KERNEL_PRIVATE
78 #include <kern/startup.h>
79 #endif /* XNU_KERNEL_PRIVATE */
80
81 #if XNU_KERNEL_PRIVATE && !defined(ZALLOC_ALLOW_DEPRECATED)
82 #define __zalloc_deprecated(msg) __deprecated_msg(msg)
83 #else
84 #define __zalloc_deprecated(msg)
85 #endif
86
87 /*
88 * Enable this macro to force type safe zalloc/zalloc_ro/...
89 */
90 #ifndef ZALLOC_TYPE_SAFE
91 #if __has_ptrcheck
92 #define ZALLOC_TYPE_SAFE 1
93 #else
94 #define ZALLOC_TYPE_SAFE 0
95 #endif
96 #endif /* !ZALLOC_TYPE_SAFE */
97
98 __BEGIN_DECLS __ASSUME_PTR_ABI_SINGLE_BEGIN
99
100 /*!
101 * @macro __zpercpu
102 *
103 * @abstract
104 * Annotation that helps denoting a per-cpu pointer that requires usage of
105 * @c zpercpu_*() for access.
106 */
107 #define __zpercpu __unsafe_indexable
108
109 /*!
110 * @typedef zone_id_t
111 *
112 * @abstract
113 * The type for a zone ID.
114 */
115 typedef uint16_t zone_id_t;
116
117 /**
118 * @enum zone_create_flags_t
119 *
120 * @abstract
121 * Set of flags to pass to zone_create().
122 *
123 * @discussion
124 * Some kernel-wide policies affect all possible created zones.
125 * Explicit @c ZC_* win over such policies.
126 */
127 __options_decl(zone_create_flags_t, uint64_t, {
128 /** The default value to pass to zone_create() */
129 ZC_NONE = 0x00000000,
130
131 /** Force the created zone to use VA sequestering */
132 ZC_SEQUESTER = 0x00000001,
133 /** Force the created zone @b NOT to use VA sequestering */
134 ZC_NOSEQUESTER = 0x00000002,
135
136 /** Enable per-CPU zone caching for this zone */
137 ZC_CACHING = 0x00000010,
138 /** Disable per-CPU zone caching for this zone */
139 ZC_NOCACHING = 0x00000020,
140
141 /** Allocate zone pages as Read-only **/
142 ZC_READONLY = 0x00800000,
143
144 /** Mark zone as a per-cpu zone */
145 ZC_PERCPU = 0x01000000,
146
147 /** Force the created zone to clear every allocation on free */
148 ZC_ZFREE_CLEARMEM = 0x02000000,
149
150 /** Mark zone as non collectable by zone_gc() */
151 ZC_NOGC = 0x04000000,
152
153 /** Do not encrypt this zone during hibernation */
154 ZC_NOENCRYPT = 0x08000000,
155
156 /** Type requires alignment to be preserved */
157 ZC_ALIGNMENT_REQUIRED = 0x10000000,
158
159 /** Do not track this zone when gzalloc is engaged */
160 ZC_NOGZALLOC = 0x20000000,
161
162 /** Don't asynchronously replenish the zone via callouts */
163 ZC_NOCALLOUT = 0x40000000,
164
165 /** Can be zdestroy()ed, not default unlike zinit() */
166 ZC_DESTRUCTIBLE = 0x80000000,
167
168 #ifdef XNU_KERNEL_PRIVATE
169 /** Use guard pages in PGZ mode */
170 ZC_PGZ_USE_GUARDS = 0x0100000000000000,
171
172 /** Zone doesn't support TBI tagging */
173 ZC_NOTBITAG = 0x0200000000000000,
174
175 /** This zone will back a kalloc type */
176 ZC_KALLOC_TYPE = 0x0400000000000000,
177
178 /** This zone will back a kalloc heap */
179 ZC_KALLOC_HEAP = 0x0800000000000000,
180
181 /* unused 0x1000000000000000, */
182
183 /** This zone belongs to the VM submap */
184 ZC_VM = 0x2000000000000000,
185 #if __LP64__
186 #define ZC_VM_LP64 ZC_VM
187 #else
188 #define ZC_VM_LP64 ZC_NONE
189 #endif
190
191 /** Disable kasan quarantine for this zone */
192 ZC_KASAN_NOQUARANTINE = 0x4000000000000000,
193
194 /** Disable kasan redzones for this zone */
195 ZC_KASAN_NOREDZONE = 0x8000000000000000,
196 #endif /* XNU_KERNEL_PRIVATE */
197 });
198
199 /*!
200 * @union zone_or_view
201 *
202 * @abstract
203 * A type used for calls that admit both a zone or a zone view.
204 *
205 * @discussion
206 * @c zalloc() and @c zfree() and their variants can act on both
207 * zones and zone views.
208 */
209 union zone_or_view {
210 struct zone_view *zov_view;
211 struct zone *zov_zone;
212 struct kalloc_type_view *zov_kt_heap;
213 #ifdef __cplusplus
zone_or_view(struct zone_view * zv)214 inline zone_or_view(struct zone_view *zv) : zov_view(zv) {
215 }
zone_or_view(struct zone * z)216 inline zone_or_view(struct zone *z) : zov_zone(z) {
217 }
zone_or_view(struct kalloc_type_view * kth)218 inline zone_or_view(struct kalloc_type_view *kth) : zov_kt_heap(kth) {
219 }
220 #endif
221 };
222 #ifdef __cplusplus
223 typedef union zone_or_view zone_or_view_t;
224 #else
225 typedef union zone_or_view zone_or_view_t __attribute__((transparent_union));
226 #endif
227
228 /*!
229 * @enum zone_create_ro_id_t
230 *
231 * @abstract
232 * Zone creation IDs for external read only zones
233 *
234 * @discussion
235 * Kexts that desire to use the RO allocator should:
236 * 1. Add a zone creation id below
237 * 2. Add a corresponding ID to @c zone_reserved_id_t
238 * 3. Use @c zone_create_ro with ID from #1 to create a RO zone.
239 * 4. Save the zone ID returned from #3 in a SECURITY_READ_ONLY_LATE variable.
240 * 5. Use the saved ID for zalloc_ro/zfree_ro, etc.
241 */
242 __enum_decl(zone_create_ro_id_t, zone_id_t, {
243 ZC_RO_ID_SANDBOX,
244 ZC_RO_ID_PROFILE,
245 ZC_RO_ID_PROTOBOX,
246 ZC_RO_ID_SB_FILTER,
247 ZC_RO_ID__LAST = ZC_RO_ID_SB_FILTER,
248 });
249
250 /*!
251 * @function zone_create
252 *
253 * @abstract
254 * Creates a zone with the specified parameters.
255 *
256 * @discussion
257 * A Zone is a slab allocator that returns objects of a given size very quickly.
258 *
259 * @param name the name for the new zone.
260 * @param size the size of the elements returned by this zone.
261 * @param flags a set of @c zone_create_flags_t flags.
262 *
263 * @returns the created zone, this call never fails.
264 */
265 extern zone_t zone_create(
266 const char *name __unsafe_indexable,
267 vm_size_t size,
268 zone_create_flags_t flags);
269
270 /*!
271 * @function zone_create_ro
272 *
273 * @abstract
274 * Creates a read only zone with the specified parameters from kexts
275 *
276 * @discussion
277 * See notes under @c zone_create_ro_id_t wrt creation and use of RO zones in
278 * kexts. Do not use this API to create read only zones in xnu.
279 *
280 * @param name the name for the new zone.
281 * @param size the size of the elements returned by this zone.
282 * @param flags a set of @c zone_create_flags_t flags.
283 * @param zc_ro_id an ID declared in @c zone_create_ro_id_t
284 *
285 * @returns the zone ID of the created zone, this call never fails.
286 */
287 extern zone_id_t zone_create_ro(
288 const char *name __unsafe_indexable,
289 vm_size_t size,
290 zone_create_flags_t flags,
291 zone_create_ro_id_t zc_ro_id);
292
293 /*!
294 * @function zdestroy
295 *
296 * @abstract
297 * Destroys a zone previously made with zone_create.
298 *
299 * @discussion
300 * Zones must have been made destructible for @c zdestroy() to be allowed,
301 * passing @c ZC_DESTRUCTIBLE at @c zone_create() time.
302 *
303 * @param zone the zone to destroy.
304 */
305 extern void zdestroy(
306 zone_t zone);
307
308 /*!
309 * @function zone_require
310 *
311 * @abstract
312 * Requires for a given pointer to belong to the specified zone.
313 *
314 * @discussion
315 * The function panics if the check fails as it indicates that the kernel
316 * internals have been compromised.
317 *
318 * @param zone the zone the address needs to belong to.
319 * @param addr the element address to check.
320 */
321 extern void zone_require(
322 zone_t zone,
323 void *addr __unsafe_indexable);
324
325 /*!
326 * @function zone_require_ro
327 *
328 * @abstract
329 * Version of zone require intended for zones created with ZC_READONLY
330 *
331 * @discussion
332 * This check is not sufficient to fully trust the element.
333 *
334 * Another check of its content must be performed to prove
335 * that the element is "the right one", a typical technique
336 * for when the RO data structure is 1:1 with a mutable one,
337 * is a simple circularity check with a very strict lifetime
338 * (both the mutable and read-only data structures are made
339 * and destroyed as close as possible).
340 *
341 * @param zone_id the zone id the address needs to belong to.
342 * @param elem_size the element size for this zone.
343 * @param addr the element address to check.
344 */
345 extern void zone_require_ro(
346 zone_id_t zone_id,
347 vm_size_t elem_size,
348 void *addr __unsafe_indexable);
349
350 /*!
351 * @function zone_require_ro_range_contains
352 *
353 * @abstract
354 * Version of zone require intended for zones created with ZC_READONLY
355 * that only checks that the zone is RO and that the address is in
356 * the zone's submap
357 *
358 * @param zone_id the zone id the address needs to belong to.
359 * @param addr the element address to check.
360 */
361 extern void zone_require_ro_range_contains(
362 zone_id_t zone_id,
363 void *addr __unsafe_indexable);
364
365 /*!
366 * @enum zalloc_flags_t
367 *
368 * @brief
369 * Flags that can be passed to @c zalloc_internal or @c zalloc_flags.
370 *
371 * @discussion
372 * It is encouraged that any callsite passing flags uses exactly one of:
373 * @c Z_WAITOK, @c Z_NOWAIT or @c Z_NOPAGEWAIT, the default being @c Z_WAITOK
374 * if nothing else was specified.
375 *
376 * If any @c Z_NO*WAIT flag is passed alongside @c Z_WAITOK,
377 * then @c Z_WAITOK is ignored.
378 *
379 * @const Z_WAITOK
380 * Means that it's OK for zalloc() to block to wait for memory,
381 * when Z_WAITOK is passed, zalloc will never return NULL.
382 *
383 * @const Z_NOWAIT
384 * Passing this flag means that zalloc is not allowed to ever block.
385 *
386 * @const Z_NOPAGEWAIT
387 * Passing this flag means that zalloc is allowed to wait due to lock
388 * contention, but will not wait for the VM to wait for pages when
389 * under memory pressure.
390 *
391 * @const Z_ZERO
392 * Passing this flags means that the returned memory has been zeroed out.
393 *
394 * @const Z_NOFAIL
395 * Passing this flag means that the caller expects the allocation to always
396 * succeed. This will result in a panic if this assumption isn't correct.
397 *
398 * This flag is incompatible with @c Z_NOWAIT or @c Z_NOPAGEWAIT. It also can't
399 * be used on exhaustible zones.
400 *
401 * @const Z_REALLOCF
402 * For the realloc family of functions,
403 * free the incoming memory on failure cases.
404 *
405 #if XNU_KERNEL_PRIVATE
406 * @const Z_FULLSIZE
407 * Used to indicate that the caller will use all available space in excess
408 * from the requested allocation size.
409 *
410 * @const Z_SKIP_KASAN
411 * Tell zalloc() not to do any kasan adjustments.
412 *
413 * @const Z_PGZ
414 * Used by zalloc internally to denote an allocation that we will try
415 * to guard with PGZ.
416 *
417 * @const Z_VM_TAG_BT_BIT
418 * Used to blame allocation accounting on the first kext
419 * found in the backtrace of the allocation.
420 *
421 * @const Z_NOZZC
422 * Used internally to mark allocations that will skip zero validation.
423 *
424 * @const Z_PCPU
425 * Used internally for the percpu paths.
426 *
427 * @const Z_VM_TAG_MASK
428 * Represents bits in which a vm_tag_t for the allocation can be passed.
429 * (used by kalloc for the zone tagging debugging feature).
430 #endif
431 */
432 __options_decl(zalloc_flags_t, uint32_t, {
433 // values smaller than 0xff are shared with the M_* flags from BSD MALLOC
434 Z_WAITOK = 0x0000,
435 Z_NOWAIT = 0x0001,
436 Z_NOPAGEWAIT = 0x0002,
437 Z_ZERO = 0x0004,
438 Z_REALLOCF = 0x0008,
439
440 #if XNU_KERNEL_PRIVATE
441 Z_FULLSIZE = 0x0200,
442 #if KASAN
443 Z_SKIP_KASAN = 0x0400,
444 #else
445 Z_SKIP_KASAN = 0x0000,
446 #endif
447 Z_PGZ = 0x0800,
448 Z_VM_TAG_BT_BIT = 0x1000,
449 Z_PCPU = 0x2000,
450 Z_NOZZC = 0x4000,
451 #endif /* XNU_KERNEL_PRIVATE */
452 Z_NOFAIL = 0x8000,
453
454 /* convenient c++ spellings */
455 Z_NOWAIT_ZERO = Z_NOWAIT | Z_ZERO,
456 Z_WAITOK_ZERO = Z_WAITOK | Z_ZERO,
457 Z_WAITOK_ZERO_NOFAIL = Z_WAITOK | Z_ZERO | Z_NOFAIL, /* convenient spelling for c++ */
458
459 Z_KPI_MASK = Z_WAITOK | Z_NOWAIT | Z_NOPAGEWAIT | Z_ZERO,
460 #if XNU_KERNEL_PRIVATE
461 Z_ZERO_VM_TAG_BT_BIT = Z_ZERO | Z_VM_TAG_BT_BIT,
462 /** used by kalloc to propagate vm tags for -zt */
463 Z_VM_TAG_MASK = 0xffff0000,
464
465 #define Z_VM_TAG_SHIFT 16
466 #define Z_VM_TAG(fl, tag) ((zalloc_flags_t)((fl) | ((tag) << Z_VM_TAG_SHIFT)))
467 #define Z_VM_TAG_BT(fl, tag) ((zalloc_flags_t)(Z_VM_TAG(fl, tag) | Z_VM_TAG_BT_BIT))
468 #endif
469 });
470
471 /*
472 * This type is used so that kalloc_internal has good calling conventions
473 * for callers who want to cheaply both know the allocated address
474 * and the actual size of the allocation.
475 */
476 struct kalloc_result {
477 void *addr __sized_by(size);
478 vm_size_t size;
479 };
480
481 /*!
482 * @function zalloc
483 *
484 * @abstract
485 * Allocates an element from a specified zone.
486 *
487 * @discussion
488 * If the zone isn't exhaustible and is expandable, this call never fails.
489 *
490 * @param zone_or_view the zone or zone view to allocate from
491 *
492 * @returns NULL or the allocated element
493 */
494 __attribute__((malloc))
495 extern void *__unsafe_indexable zalloc(
496 zone_or_view_t zone_or_view);
497
498 /*!
499 * @function zalloc_noblock
500 *
501 * @abstract
502 * Allocates an element from a specified zone, but never blocks.
503 *
504 * @discussion
505 * This call is suitable for preemptible code, however allocation
506 * isn't allowed from interrupt context.
507 *
508 * @param zone_or_view the zone or zone view to allocate from
509 *
510 * @returns NULL or the allocated element
511 */
512 __attribute__((malloc))
513 extern void *__unsafe_indexable zalloc_noblock(
514 zone_or_view_t zone_or_view);
515
516 /*!
517 * @function zalloc_flags()
518 *
519 * @abstract
520 * Allocates an element from a specified zone, with flags.
521 *
522 * @param zone_or_view the zone or zone view to allocate from
523 * @param flags a collection of @c zalloc_flags_t.
524 *
525 * @returns NULL or the allocated element
526 */
527 __attribute__((malloc))
528 extern void *__unsafe_indexable zalloc_flags(
529 zone_or_view_t zone_or_view,
530 zalloc_flags_t flags);
531
532 /*!
533 * @macro zalloc_id
534 *
535 * @abstract
536 * Allocates an element from a specified zone ID, with flags.
537 *
538 * @param zid The proper @c ZONE_ID_* constant.
539 * @param flags a collection of @c zalloc_flags_t.
540 *
541 * @returns NULL or the allocated element
542 */
543 __attribute__((malloc))
544 extern void *__unsafe_indexable zalloc_id(
545 zone_id_t zid,
546 zalloc_flags_t flags);
547
548 /*!
549 * @function zalloc_ro
550 *
551 * @abstract
552 * Allocates an element from a specified read-only zone.
553 *
554 * @param zone_id the zone id to allocate from
555 * @param flags a collection of @c zalloc_flags_t.
556 *
557 * @returns NULL or the allocated element
558 */
559 __attribute__((malloc))
560 extern void *__unsafe_indexable zalloc_ro(
561 zone_id_t zone_id,
562 zalloc_flags_t flags);
563
564 /*!
565 * @function zalloc_ro_mut
566 *
567 * @abstract
568 * Modifies an element from a specified read-only zone.
569 *
570 * @discussion
571 * Modifying compiler-assisted authenticated pointers using this function will
572 * not result in a signed pointer being written. The caller is expected to
573 * sign the value appropriately beforehand if they wish to do this.
574 *
575 * @param zone_id the zone id to allocate from
576 * @param elem element to be modified
577 * @param offset offset from element
578 * @param new_data pointer to new data
579 * @param new_data_size size of modification
580 *
581 */
582 extern void zalloc_ro_mut(
583 zone_id_t zone_id,
584 void *elem __unsafe_indexable,
585 vm_offset_t offset,
586 const void *new_data __sized_by(new_data_size),
587 vm_size_t new_data_size);
588
589 /*!
590 * @function zalloc_ro_update_elem
591 *
592 * @abstract
593 * Update the value of an entire element allocated in the read only allocator.
594 *
595 * @param zone_id the zone id to allocate from
596 * @param elem element to be modified
597 * @param new_data pointer to new data
598 *
599 */
600 #define zalloc_ro_update_elem(zone_id, elem, new_data) ({ \
601 const typeof(*(elem)) *__new_data = (new_data); \
602 zalloc_ro_mut(zone_id, elem, 0, __new_data, sizeof(*__new_data)); \
603 })
604
605 /*!
606 * @function zalloc_ro_update_field
607 *
608 * @abstract
609 * Update a single field of an element allocated in the read only allocator.
610 *
611 * @param zone_id the zone id to allocate from
612 * @param elem element to be modified
613 * @param field the element field to be modified
614 * @param new_data pointer to new data
615 *
616 */
617 #define zalloc_ro_update_field(zone_id, elem, field, value) ({ \
618 const typeof((elem)->field) *__value = (value); \
619 zalloc_ro_mut(zone_id, elem, offsetof(typeof(*(elem)), field), \
620 __value, sizeof((elem)->field)); \
621 })
622
623 #if __LP64__
624 #define ZRO_ATOMIC_LONG(op) ZRO_ATOMIC_##op##_64
625 #else
626 #define ZRO_ATOMIC_LONG(op) ZRO_ATOMIC_##op##_32
627 #endif
628
629 /*!
630 * @enum zro_atomic_op_t
631 *
632 * @brief
633 * Flags that can be used with @c zalloc_ro_*_atomic to specify the desired
634 * atomic operations.
635 *
636 * @discussion
637 * This enum provides all flavors of atomic operations supported in sizes 8,
638 * 16, 32, 64 bits.
639 *
640 * @const ZRO_ATOMIC_OR_*
641 * To perform an @s os_atomic_or
642 *
643 * @const ZRO_ATOMIC_XOR_*
644 * To perform an @s os_atomic_xor
645 *
646 * @const ZRO_ATOMIC_AND_*
647 * To perform an @s os_atomic_and
648 *
649 * @const ZRO_ATOMIC_ADD_*
650 * To perform an @s os_atomic_add
651 *
652 * @const ZRO_ATOMIC_XCHG_*
653 * To perform an @s os_atomic_xchg
654 *
655 */
656 __enum_decl(zro_atomic_op_t, uint32_t, {
657 ZRO_ATOMIC_OR_8 = 0x00000010 | 1,
658 ZRO_ATOMIC_OR_16 = 0x00000010 | 2,
659 ZRO_ATOMIC_OR_32 = 0x00000010 | 4,
660 ZRO_ATOMIC_OR_64 = 0x00000010 | 8,
661
662 ZRO_ATOMIC_XOR_8 = 0x00000020 | 1,
663 ZRO_ATOMIC_XOR_16 = 0x00000020 | 2,
664 ZRO_ATOMIC_XOR_32 = 0x00000020 | 4,
665 ZRO_ATOMIC_XOR_64 = 0x00000020 | 8,
666
667 ZRO_ATOMIC_AND_8 = 0x00000030 | 1,
668 ZRO_ATOMIC_AND_16 = 0x00000030 | 2,
669 ZRO_ATOMIC_AND_32 = 0x00000030 | 4,
670 ZRO_ATOMIC_AND_64 = 0x00000030 | 8,
671
672 ZRO_ATOMIC_ADD_8 = 0x00000040 | 1,
673 ZRO_ATOMIC_ADD_16 = 0x00000040 | 2,
674 ZRO_ATOMIC_ADD_32 = 0x00000040 | 4,
675 ZRO_ATOMIC_ADD_64 = 0x00000040 | 8,
676
677 ZRO_ATOMIC_XCHG_8 = 0x00000050 | 1,
678 ZRO_ATOMIC_XCHG_16 = 0x00000050 | 2,
679 ZRO_ATOMIC_XCHG_32 = 0x00000050 | 4,
680 ZRO_ATOMIC_XCHG_64 = 0x00000050 | 8,
681
682 /* cconvenient spellings */
683 ZRO_ATOMIC_OR_LONG = ZRO_ATOMIC_LONG(OR),
684 ZRO_ATOMIC_XOR_LONG = ZRO_ATOMIC_LONG(XOR),
685 ZRO_ATOMIC_AND_LONG = ZRO_ATOMIC_LONG(AND),
686 ZRO_ATOMIC_ADD_LONG = ZRO_ATOMIC_LONG(ADD),
687 ZRO_ATOMIC_XCHG_LONG = ZRO_ATOMIC_LONG(XCHG),
688 });
689
690 /*!
691 * @function zalloc_ro_mut_atomic
692 *
693 * @abstract
694 * Atomically update an offset in an element allocated in the read only
695 * allocator. Do not use directly. Use via @c zalloc_ro_update_field_atomic.
696 *
697 * @param zone_id the zone id to allocate from
698 * @param elem element to be modified
699 * @param offset offset in the element to be modified
700 * @param op atomic operation to perform (see @c zro_atomic_op_t)
701 * @param value value for the atomic operation
702 *
703 */
704 extern uint64_t zalloc_ro_mut_atomic(
705 zone_id_t zone_id,
706 void *elem __unsafe_indexable,
707 vm_offset_t offset,
708 zro_atomic_op_t op,
709 uint64_t value);
710
711 /*!
712 * @macro zalloc_ro_update_field_atomic
713 *
714 * @abstract
715 * Atomically update a single field of an element allocated in the read only
716 * allocator.
717 *
718 * @param zone_id the zone id to allocate from
719 * @param elem element to be modified
720 * @param field the element field to be modified
721 * @param op atomic operation to perform (see @c zro_atomic_op_t)
722 * @param value value for the atomic operation
723 *
724 */
725 #define zalloc_ro_update_field_atomic(zone_id, elem, field, op, value) ({ \
726 const typeof((elem)->field) __value = (value); \
727 static_assert(sizeof(__value) == (op & 0xf)); \
728 (os_atomic_basetypeof(&(elem)->field))zalloc_ro_mut_atomic(zone_id, \
729 elem, offsetof(typeof(*(elem)), field), op, (uint64_t)__value); \
730 })
731
732 /*!
733 * @function zalloc_ro_clear
734 *
735 * @abstract
736 * Zeroes an element from a specified read-only zone.
737 *
738 * @param zone_id the zone id to allocate from
739 * @param elem element to be modified
740 * @param offset offset from element
741 * @param size size of modification
742 */
743 extern void zalloc_ro_clear(
744 zone_id_t zone_id,
745 void *elem __unsafe_indexable,
746 vm_offset_t offset,
747 vm_size_t size);
748
749 /*!
750 * @function zalloc_ro_clear_field
751 *
752 * @abstract
753 * Zeroes the specified field of an element from a specified read-only zone.
754 *
755 * @param zone_id the zone id to allocate from
756 * @param elem element to be modified
757 * @param field offset from element
758 */
759 #define zalloc_ro_clear_field(zone_id, elem, field) \
760 zalloc_ro_clear(zone_id, elem, offsetof(typeof(*(elem)), field), \
761 sizeof((elem)->field))
762
763 /*!
764 * @function zfree_id()
765 *
766 * @abstract
767 * Frees an element previously allocated with @c zalloc_id().
768 *
769 * @param zone_id the zone id to free the element to.
770 * @param addr the address to free
771 */
772 extern void zfree_id(
773 zone_id_t zone_id,
774 void *addr __unsafe_indexable);
775
776 /*!
777 * @function zfree_ro()
778 *
779 * @abstract
780 * Frees an element previously allocated with @c zalloc_ro().
781 *
782 * @param zone_id the zone id to free the element to.
783 * @param addr the address to free
784 */
785 extern void zfree_ro(
786 zone_id_t zone_id,
787 void *addr __unsafe_indexable);
788
789 /*!
790 * @function zfree
791 *
792 * @abstract
793 * Frees an element allocated with @c zalloc*.
794 *
795 * @discussion
796 * If the element being freed doesn't belong to the specified zone,
797 * then this call will panic.
798 *
799 * @param zone_or_view the zone or zone view to free the element to.
800 * @param elem the element to free
801 */
802 extern void zfree(
803 zone_or_view_t zone_or_view,
804 void *elem __unsafe_indexable);
805
806 /*
807 * This macro sets "elem" to NULL on free.
808 *
809 * Note: all values passed to zfree*() might be in the element to be freed,
810 * temporaries must be taken, and the resetting to be done prior to free.
811 */
812 #define zfree(zone, elem) ({ \
813 __auto_type __zfree_zone = (zone); \
814 (zfree)(__zfree_zone, (void *)os_ptr_load_and_erase(elem)); \
815 })
816
817 #define zfree_id(zid, elem) ({ \
818 zone_id_t __zfree_zid = (zid); \
819 (zfree_id)(__zfree_zid, (void *)os_ptr_load_and_erase(elem)); \
820 })
821
822 #define zfree_ro(zid, elem) ({ \
823 zone_id_t __zfree_zid = (zid); \
824 (zfree_ro)(__zfree_zid, (void *)os_ptr_load_and_erase(elem)); \
825 })
826
827 /* deprecated KPIS */
828
829 __zalloc_deprecated("use zone_create()")
830 extern zone_t zinit(
831 vm_size_t size, /* the size of an element */
832 vm_size_t maxmem, /* maximum memory to use */
833 vm_size_t alloc, /* allocation size */
834 const char *name __unsafe_indexable);
835
836
837 #pragma mark: zone views
838 /*!
839 * @typedef zone_stats_t
840 *
841 * @abstract
842 * The opaque type for per-cpu zone stats that are accumulated per zone
843 * or per zone-view.
844 */
845 typedef struct zone_stats *__zpercpu zone_stats_t;
846
847 /*!
848 * @typedef zone_view_t
849 *
850 * @abstract
851 * A view on a zone for accounting purposes.
852 *
853 * @discussion
854 * A zone view uses the zone it references for the allocations backing store,
855 * but does the allocation accounting at the view level.
856 *
857 * These accounting are surfaced by @b zprint(1) and similar tools,
858 * which allow for cheap but finer grained understanding of allocations
859 * without any fragmentation cost.
860 *
861 * Zone views are protected by the kernel lockdown and can't be initialized
862 * dynamically. They must be created using @c ZONE_VIEW_DEFINE().
863 */
864 typedef struct zone_view *zone_view_t;
865 struct zone_view {
866 zone_t zv_zone;
867 zone_stats_t zv_stats;
868 const char *zv_name __unsafe_indexable;
869 zone_view_t zv_next;
870 };
871
872 #ifdef XNU_KERNEL_PRIVATE
873 /*!
874 * @enum zone_kheap_id_t
875 *
876 * @brief
877 * Enumerate a particular kalloc heap.
878 *
879 * @discussion
880 * More documentation about heaps is available in @c <kern/kalloc.h>.
881 *
882 * @const KHEAP_ID_NONE
883 * This value denotes regular zones, not used by kalloc.
884 *
885 * @const KHEAP_ID_DEFAULT
886 * Indicates zones part of the KHEAP_DEFAULT heap.
887 *
888 * @const KHEAP_ID_DATA_BUFFERS
889 * Indicates zones part of the KHEAP_DATA_BUFFERS heap.
890 *
891 * @const KHEAP_ID_KT_VAR
892 * Indicates zones part of the KHEAP_KT_VAR heap.
893 */
894 __enum_decl(zone_kheap_id_t, uint32_t, {
895 KHEAP_ID_NONE,
896 KHEAP_ID_DEFAULT,
897 KHEAP_ID_DATA_BUFFERS,
898 KHEAP_ID_KT_VAR,
899
900 #define KHEAP_ID_COUNT (KHEAP_ID_KT_VAR + 1)
901 });
902
903 /*!
904 * @macro ZONE_VIEW_DECLARE
905 *
906 * @abstract
907 * (optionally) declares a zone view (in a header).
908 *
909 * @param var the name for the zone view.
910 */
911 #define ZONE_VIEW_DECLARE(var) \
912 extern struct zone_view var[1]
913
914 /*!
915 * @macro ZONE_VIEW_DEFINE
916 *
917 * @abstract
918 * Defines a given zone view and what it points to.
919 *
920 * @discussion
921 * Zone views can either share a pre-existing zone,
922 * or perform a lookup into a kalloc heap for the zone
923 * backing the bucket of the proper size.
924 *
925 * Zone views are initialized during the @c STARTUP_SUB_ZALLOC phase,
926 * as the last rank. If views on zones are created, these must have been
927 * created before this stage.
928 *
929 * This macro should not be used to create zone views from default
930 * kalloc heap, KALLOC_TYPE_DEFINE should be used instead.
931 *
932 * @param var the name for the zone view.
933 * @param name a string describing the zone view.
934 * @param heap_or_zone a @c KHEAP_ID_* constant or a pointer to a zone.
935 * @param size the element size to be allocated from this view.
936 */
937 #define ZONE_VIEW_DEFINE(var, name, heap_or_zone, size) \
938 SECURITY_READ_ONLY_LATE(struct zone_view) var[1] = { { \
939 .zv_name = name, \
940 } }; \
941 static __startup_data struct zone_view_startup_spec \
942 __startup_zone_view_spec_ ## var = { var, { heap_or_zone }, size }; \
943 STARTUP_ARG(ZALLOC, STARTUP_RANK_LAST, zone_view_startup_init, \
944 &__startup_zone_view_spec_ ## var)
945
946 #endif /* XNU_KERNEL_PRIVATE */
947
948
949 #ifdef XNU_KERNEL_PRIVATE
950 #pragma mark - XNU only interfaces
951
952 #include <kern/cpu_number.h>
953
954 #pragma GCC visibility push(hidden)
955
956 #pragma mark XNU only: zalloc (extended)
957
958 #define ZALIGN_NONE (sizeof(uint8_t) - 1)
959 #define ZALIGN_16 (sizeof(uint16_t) - 1)
960 #define ZALIGN_32 (sizeof(uint32_t) - 1)
961 #define ZALIGN_PTR (sizeof(void *) - 1)
962 #define ZALIGN_64 (sizeof(uint64_t) - 1)
963 #define ZALIGN(t) (_Alignof(t) - 1)
964
965
966 /*!
967 * @function zalloc_permanent_tag()
968 *
969 * @abstract
970 * Allocates a permanent element from the permanent zone
971 *
972 * @discussion
973 * Memory returned by this function is always 0-initialized.
974 * Note that the size of this allocation can not be determined
975 * by zone_element_size so it should not be used for copyio.
976 *
977 * @param size the element size (must be smaller than PAGE_SIZE)
978 * @param align_mask the required alignment for this allocation
979 * @param tag the tag to use for allocations larger than a page.
980 *
981 * @returns the allocated element
982 */
983 __attribute__((malloc))
984 extern void *__unsafe_indexable zalloc_permanent_tag(
985 vm_size_t size,
986 vm_offset_t align_mask,
987 vm_tag_t tag);
988
989 /*!
990 * @function zalloc_permanent()
991 *
992 * @abstract
993 * Allocates a permanent element from the permanent zone
994 *
995 * @discussion
996 * Memory returned by this function is always 0-initialized.
997 * Note that the size of this allocation can not be determined
998 * by zone_element_size so it should not be used for copyio.
999 *
1000 * @param size the element size (must be smaller than PAGE_SIZE)
1001 * @param align_mask the required alignment for this allocation
1002 *
1003 * @returns the allocated element
1004 */
1005 #define zalloc_permanent(size, align) \
1006 zalloc_permanent_tag(size, align, VM_KERN_MEMORY_KALLOC)
1007
1008 /*!
1009 * @function zalloc_permanent_type()
1010 *
1011 * @abstract
1012 * Allocates a permanent element of a given type with its natural alignment.
1013 *
1014 * @discussion
1015 * Memory returned by this function is always 0-initialized.
1016 *
1017 * @param type_t the element type
1018 *
1019 * @returns the allocated element
1020 */
1021 #define zalloc_permanent_type(type_t) \
1022 __unsafe_forge_single(type_t *, \
1023 zalloc_permanent(sizeof(type_t), ZALIGN(type_t)))
1024
1025 /*!
1026 * @function zalloc_first_proc_made()
1027 *
1028 * @abstract
1029 * Declare that the "early" allocation phase is done.
1030 */
1031 extern void
1032 zalloc_first_proc_made(void);
1033
1034 #pragma mark XNU only: per-cpu allocations
1035
1036 /*!
1037 * @macro zpercpu_get_cpu()
1038 *
1039 * @abstract
1040 * Get a pointer to a specific CPU slot of a given per-cpu variable.
1041 *
1042 * @param ptr the per-cpu pointer (returned by @c zalloc_percpu*()).
1043 * @param cpu the specified CPU number as returned by @c cpu_number()
1044 *
1045 * @returns the per-CPU slot for @c ptr for the specified CPU.
1046 */
1047 #define zpercpu_get_cpu(ptr, cpu) \
1048 __zpcpu_cast(ptr, __zpcpu_demangle(ptr) + ptoa((unsigned)cpu))
1049
1050 /*!
1051 * @macro zpercpu_get()
1052 *
1053 * @abstract
1054 * Get a pointer to the current CPU slot of a given per-cpu variable.
1055 *
1056 * @param ptr the per-cpu pointer (returned by @c zalloc_percpu*()).
1057 *
1058 * @returns the per-CPU slot for @c ptr for the current CPU.
1059 */
1060 #define zpercpu_get(ptr) \
1061 zpercpu_get_cpu(ptr, cpu_number())
1062
1063 /*!
1064 * @macro zpercpu_foreach()
1065 *
1066 * @abstract
1067 * Enumerate all per-CPU slots by address.
1068 *
1069 * @param it the name for the iterator
1070 * @param ptr the per-cpu pointer (returned by @c zalloc_percpu*()).
1071 */
1072 #define zpercpu_foreach(it, ptr) \
1073 for (typeof(ptr) it = zpercpu_get_cpu(ptr, 0), \
1074 __end_##it = zpercpu_get_cpu(ptr, zpercpu_count()); \
1075 it < __end_##it; it = __zpcpu_next(it))
1076
1077 /*!
1078 * @macro zpercpu_foreach_cpu()
1079 *
1080 * @abstract
1081 * Enumerate all per-CPU slots by CPU slot number.
1082 *
1083 * @param cpu the name for cpu number iterator.
1084 */
1085 #define zpercpu_foreach_cpu(cpu) \
1086 for (unsigned cpu = 0; cpu < zpercpu_count(); cpu++)
1087
1088 /*!
1089 * @function zalloc_percpu()
1090 *
1091 * @abstract
1092 * Allocates an element from a per-cpu zone.
1093 *
1094 * @discussion
1095 * The returned pointer cannot be used directly and must be manipulated
1096 * through the @c zpercpu_get*() interfaces.
1097 *
1098 * @param zone_or_view the zone or zone view to allocate from
1099 * @param flags a collection of @c zalloc_flags_t.
1100 *
1101 * @returns NULL or the allocated element
1102 */
1103 extern void *__zpercpu zalloc_percpu(
1104 zone_or_view_t zone_or_view,
1105 zalloc_flags_t flags);
1106
1107 /*!
1108 * @function zfree_percpu()
1109 *
1110 * @abstract
1111 * Frees an element previously allocated with @c zalloc_percpu().
1112 *
1113 * @param zone_or_view the zone or zone view to free the element to.
1114 * @param addr the address to free
1115 */
1116 extern void zfree_percpu(
1117 zone_or_view_t zone_or_view,
1118 void *__zpercpu addr);
1119
1120 /*!
1121 * @function zalloc_percpu_permanent()
1122 *
1123 * @abstract
1124 * Allocates a permanent percpu-element from the permanent percpu zone.
1125 *
1126 * @discussion
1127 * Memory returned by this function is always 0-initialized.
1128 *
1129 * @param size the element size (must be smaller than PAGE_SIZE)
1130 * @param align_mask the required alignment for this allocation
1131 *
1132 * @returns the allocated element
1133 */
1134 extern void *__zpercpu zalloc_percpu_permanent(
1135 vm_size_t size,
1136 vm_offset_t align_mask);
1137
1138 /*!
1139 * @function zalloc_percpu_permanent_type()
1140 *
1141 * @abstract
1142 * Allocates a permanent percpu-element from the permanent percpu zone of a given
1143 * type with its natural alignment.
1144 *
1145 * @discussion
1146 * Memory returned by this function is always 0-initialized.
1147 *
1148 * @param type_t the element type
1149 *
1150 * @returns the allocated element
1151 */
1152 #define zalloc_percpu_permanent_type(type_t) \
1153 ((type_t *__zpercpu)zalloc_percpu_permanent(sizeof(type_t), ZALIGN(type_t)))
1154
1155
1156 #pragma mark XNU only: zone creation (extended)
1157
1158 /*!
1159 * @enum zone_reserved_id_t
1160 *
1161 * @abstract
1162 * Well known pre-registered zones, allowing use of zone_id_require()
1163 *
1164 * @discussion
1165 * @c ZONE_ID__* aren't real zone IDs.
1166 *
1167 * @c ZONE_ID__ZERO reserves zone index 0 so that it can't be used, as 0 is too
1168 * easy a value to produce (by malice or accident).
1169 *
1170 * @c ZONE_ID__FIRST_RO_EXT is the first external read only zone ID that corresponds
1171 * to the first @c zone_create_ro_id_t. There is a 1:1 mapping between zone IDs
1172 * belonging to [ZONE_ID__FIRST_RO_EXT - ZONE_ID__LAST_RO_EXT] and zone creations IDs
1173 * listed in @c zone_create_ro_id_t.
1174 *
1175 * @c ZONE_ID__FIRST_DYNAMIC is the first dynamic zone ID that can be used by
1176 * @c zone_create().
1177 */
1178 __enum_decl(zone_reserved_id_t, zone_id_t, {
1179 ZONE_ID__ZERO,
1180
1181 ZONE_ID_PERMANENT,
1182 ZONE_ID_PERCPU_PERMANENT,
1183
1184 ZONE_ID_THREAD_RO,
1185 ZONE_ID_MAC_LABEL,
1186 ZONE_ID_PROC_RO,
1187 ZONE_ID_PROC_SIGACTS_RO,
1188 ZONE_ID_KAUTH_CRED,
1189 ZONE_ID_CS_BLOB,
1190
1191 ZONE_ID_SANDBOX_RO,
1192 ZONE_ID_PROFILE_RO,
1193 ZONE_ID_PROTOBOX,
1194 ZONE_ID_SB_FILTER,
1195
1196 ZONE_ID__FIRST_RO = ZONE_ID_THREAD_RO,
1197 ZONE_ID__FIRST_RO_EXT = ZONE_ID_SANDBOX_RO,
1198 ZONE_ID__LAST_RO_EXT = ZONE_ID_SB_FILTER,
1199 ZONE_ID__LAST_RO = ZONE_ID__LAST_RO_EXT,
1200
1201 ZONE_ID_PMAP,
1202 ZONE_ID_VM_MAP,
1203 ZONE_ID_VM_MAP_ENTRY,
1204 ZONE_ID_VM_MAP_HOLES,
1205 ZONE_ID_VM_MAP_COPY,
1206 ZONE_ID_VM_PAGES,
1207 ZONE_ID_IPC_PORT,
1208 ZONE_ID_IPC_PORT_SET,
1209 ZONE_ID_IPC_VOUCHERS,
1210 ZONE_ID_TASK,
1211 ZONE_ID_PROC,
1212 ZONE_ID_THREAD,
1213 ZONE_ID_TURNSTILE,
1214 ZONE_ID_SEMAPHORE,
1215 ZONE_ID_SELECT_SET,
1216 ZONE_ID_FILEPROC,
1217
1218 ZONE_ID__FIRST_DYNAMIC,
1219 });
1220
1221 /*!
1222 * @const ZONE_ID_ANY
1223 * The value to pass to @c zone_create_ext() to allocate a non pre-registered
1224 * Zone ID.
1225 */
1226 #define ZONE_ID_ANY ((zone_id_t)-1)
1227
1228 /*!
1229 * @const ZONE_ID_INVALID
1230 * An invalid zone_id_t that corresponds to nothing.
1231 */
1232 #define ZONE_ID_INVALID ((zone_id_t)-2)
1233
1234 /**!
1235 * @function zone_name
1236 *
1237 * @param zone the specified zone
1238 * @returns the name of the specified zone.
1239 */
1240 const char *__unsafe_indexable zone_name(
1241 zone_t zone);
1242
1243 /**!
1244 * @function zone_heap_name
1245 *
1246 * @param zone the specified zone
1247 * @returns the name of the heap this zone is part of, or "".
1248 */
1249 const char *__unsafe_indexable zone_heap_name(
1250 zone_t zone);
1251
1252 /*!
1253 * @function zone_create_ext
1254 *
1255 * @abstract
1256 * Creates a zone with the specified parameters.
1257 *
1258 * @discussion
1259 * This is an extended version of @c zone_create().
1260 *
1261 * @param name the name for the new zone.
1262 * @param size the size of the elements returned by this zone.
1263 * @param flags a set of @c zone_create_flags_t flags.
1264 * @param desired_zid a @c zone_reserved_id_t value or @c ZONE_ID_ANY.
1265 *
1266 * @param extra_setup a block that can perform non trivial initialization
1267 * on the zone before it is marked valid.
1268 * This block can call advanced setups like:
1269 * - zone_set_exhaustible()
1270 * - zone_set_noexpand()
1271 *
1272 * @returns the created zone, this call never fails.
1273 */
1274 extern zone_t zone_create_ext(
1275 const char *name __unsafe_indexable,
1276 vm_size_t size,
1277 zone_create_flags_t flags,
1278 zone_id_t desired_zid,
1279 void (^extra_setup)(zone_t));
1280
1281 /*!
1282 * @macro ZONE_DECLARE
1283 *
1284 * @abstract
1285 * Declares a zone variable and its associated type.
1286 *
1287 * @param var the name of the variable to declare.
1288 * @param type_t the type of elements in the zone.
1289 */
1290 #define ZONE_DECLARE(var, type_t) \
1291 extern zone_t var; \
1292 __ZONE_DECLARE_TYPE(var, type_t)
1293
1294 /*!
1295 * @macro ZONE_DECLARE_ID
1296 *
1297 * @abstract
1298 * Declares the type associated with a zone ID.
1299 *
1300 * @param id the name of zone ID to associate a type with.
1301 * @param type_t the type of elements in the zone.
1302 */
1303 #define ZONE_DECLARE_ID(id, type_t) \
1304 __ZONE_DECLARE_TYPE(id, type_t)
1305
1306 /*!
1307 * @macro ZONE_DEFINE
1308 *
1309 * @abstract
1310 * Declares a zone variable to automatically initialize with the specified
1311 * parameters.
1312 *
1313 * @discussion
1314 * Using ZONE_DEFINE_TYPE is preferred, but not always possible.
1315 *
1316 * @param var the name of the variable to declare.
1317 * @param name the name for the zone
1318 * @param size the size of the elements returned by this zone.
1319 * @param flags a set of @c zone_create_flags_t flags.
1320 */
1321 #define ZONE_DEFINE(var, name, size, flags) \
1322 SECURITY_READ_ONLY_LATE(zone_t) var; \
1323 static_assert(((flags) & ZC_DESTRUCTIBLE) == 0); \
1324 static __startup_data struct zone_create_startup_spec \
1325 __startup_zone_spec_ ## var = { &var, name, size, flags, \
1326 ZONE_ID_ANY, NULL }; \
1327 STARTUP_ARG(ZALLOC, STARTUP_RANK_MIDDLE, zone_create_startup, \
1328 &__startup_zone_spec_ ## var)
1329
1330 /*!
1331 * @macro ZONE_DEFINE_TYPE
1332 *
1333 * @abstract
1334 * Defines a zone variable to automatically initialize with the specified
1335 * parameters, associated with a particular type.
1336 *
1337 * @param var the name of the variable to declare.
1338 * @param name the name for the zone
1339 * @param type_t the type of elements in the zone.
1340 * @param flags a set of @c zone_create_flags_t flags.
1341 */
1342 #define ZONE_DEFINE_TYPE(var, name, type_t, flags) \
1343 ZONE_DEFINE(var, name, sizeof(type_t), flags); \
1344 __ZONE_DECLARE_TYPE(var, type_t)
1345
1346 /*!
1347 * @macro ZONE_DEFINE_ID
1348 *
1349 * @abstract
1350 * Initializes a given zone automatically during startup with the specified
1351 * parameters.
1352 *
1353 * @param zid a @c zone_reserved_id_t value.
1354 * @param name the name for the zone
1355 * @param type_t the type of elements in the zone.
1356 * @param flags a set of @c zone_create_flags_t flags.
1357 */
1358 #define ZONE_DEFINE_ID(zid, name, type_t, flags) \
1359 ZONE_DECLARE_ID(zid, type_t); \
1360 ZONE_INIT(NULL, name, sizeof(type_t), flags, zid, NULL)
1361
1362 /*!
1363 * @macro ZONE_INIT
1364 *
1365 * @abstract
1366 * Initializes a given zone automatically during startup with the specified
1367 * parameters.
1368 *
1369 * @param var the name of the variable to initialize.
1370 * @param name the name for the zone
1371 * @param size the size of the elements returned by this zone.
1372 * @param flags a set of @c zone_create_flags_t flags.
1373 * @param desired_zid a @c zone_reserved_id_t value or @c ZONE_ID_ANY.
1374 * @param extra_setup a block that can perform non trivial initialization
1375 * (@see @c zone_create_ext()).
1376 */
1377 #define ZONE_INIT(var, name, size, flags, desired_zid, extra_setup) \
1378 __ZONE_INIT(__LINE__, var, name, size, flags, desired_zid, extra_setup)
1379
1380 /*!
1381 * @function zone_id_require
1382 *
1383 * @abstract
1384 * Requires for a given pointer to belong to the specified zone, by ID and size.
1385 *
1386 * @discussion
1387 * The function panics if the check fails as it indicates that the kernel
1388 * internals have been compromised.
1389 *
1390 * This is a variant of @c zone_require() which:
1391 * - isn't sensitive to @c zone_t::elem_size being compromised,
1392 * - is slightly faster as it saves one load and a multiplication.
1393 *
1394 * @param zone_id the zone ID the address needs to belong to.
1395 * @param elem_size the size of elements for this zone.
1396 * @param addr the element address to check.
1397 */
1398 extern void zone_id_require(
1399 zone_id_t zone_id,
1400 vm_size_t elem_size,
1401 void *addr __unsafe_indexable);
1402
1403 /* Make zone as non expandable, to be called from the zone_create_ext() setup hook */
1404 extern void zone_set_noexpand(
1405 zone_t zone,
1406 vm_size_t max_elements);
1407
1408 /* Make zone exhaustible, to be called from the zone_create_ext() setup hook */
1409 extern void zone_set_exhaustible(
1410 zone_t zone,
1411 vm_size_t max_elements);
1412
1413 /*!
1414 * @function zone_fill_initially
1415 *
1416 * @brief
1417 * Initially fill a non collectable zone to have the specified amount of
1418 * elements.
1419 *
1420 * @discussion
1421 * This function must be called on a non collectable permanent zone before it
1422 * has been used yet.
1423 *
1424 * @param zone The zone to fill.
1425 * @param nelems The number of elements to be able to hold.
1426 */
1427 extern void zone_fill_initially(
1428 zone_t zone,
1429 vm_size_t nelems);
1430
1431 #pragma mark XNU only: PGZ support
1432
1433 /*!
1434 * @function pgz_owned()
1435 *
1436 * @brief
1437 * Returns whether an address is PGZ owned.
1438 *
1439 * @param addr The address to translate.
1440 * @returns Whether it is PGZ owned
1441 */
1442 #if CONFIG_PROB_GZALLOC
1443 extern bool pgz_owned(mach_vm_address_t addr) __pure2;
1444 #else
1445 #define pgz_owned(addr) false
1446 #endif
1447
1448 /*!
1449 * @function pgz_decode()
1450 *
1451 * @brief
1452 * Translates a PGZ protected virtual address to its unprotected
1453 * backing store.
1454 *
1455 * @discussion
1456 * This is exposed so that the VM can lookup the vm_page_t for PGZ protected
1457 * elements since the PGZ protected virtual addresses are maintained by PGZ
1458 * at the pmap level without the VM involvment.
1459 *
1460 * "allow_invalid" schemes relying on sequestering also need this
1461 * to perform the locking attempts on the unprotected address.
1462 *
1463 * @param addr The address to translate.
1464 * @param size The object size.
1465 * @returns The unprotected address or @c addr.
1466 */
1467 #if CONFIG_PROB_GZALLOC
1468 #define pgz_decode(addr, size) \
1469 ((typeof(addr))__pgz_decode((mach_vm_address_t)(addr), size))
1470 #else
1471 #define pgz_decode(addr, size) (addr)
1472 #endif
1473
1474 /*!
1475 * @function pgz_decode_allow_invalid()
1476 *
1477 * @brief
1478 * Translates a PGZ protected virtual address to its unprotected
1479 * backing store, but doesn't assert it is still allocated/valid.
1480 *
1481 * @discussion
1482 * "allow_invalid" schemes relying on sequestering also need this
1483 * to perform the locking attempts on the unprotected address.
1484 *
1485 * @param addr The address to translate.
1486 * @param want_zid The expected zone ID for the element.
1487 * @returns The unprotected address or @c addr.
1488 */
1489 #if CONFIG_PROB_GZALLOC
1490 #define pgz_decode_allow_invalid(addr, want_zid) \
1491 ((typeof(addr))__pgz_decode_allow_invalid((vm_offset_t)(addr), want_zid))
1492 #else
1493 #define pgz_decode_allow_invalid(addr, zid) (addr)
1494 #endif
1495
1496 #pragma mark XNU only: misc & implementation details
1497
1498 struct zone_create_startup_spec {
1499 zone_t *z_var;
1500 const char *z_name __unsafe_indexable;
1501 vm_size_t z_size;
1502 zone_create_flags_t z_flags;
1503 zone_id_t z_zid;
1504 void (^z_setup)(zone_t);
1505 };
1506
1507 extern void zone_create_startup(
1508 struct zone_create_startup_spec *spec);
1509
1510 #define __ZONE_DECLARE_TYPE(var, type_t) \
1511 __attribute__((visibility("hidden"))) \
1512 extern type_t *__zalloc__##var##__type_name
1513
1514 #define __ZONE_INIT1(ns, var, name, size, flags, zid, setup) \
1515 static __startup_data struct zone_create_startup_spec \
1516 __startup_zone_spec_ ## ns = { var, name, size, flags, zid, setup }; \
1517 STARTUP_ARG(ZALLOC, STARTUP_RANK_MIDDLE, zone_create_startup, \
1518 &__startup_zone_spec_ ## ns)
1519
1520 #define __ZONE_INIT(ns, var, name, size, flags, zid, setup) \
1521 __ZONE_INIT1(ns, var, name, size, flags, zid, setup) \
1522
1523 #define __zalloc_cast(namespace, expr) \
1524 __unsafe_forge_single(typeof(__zalloc__##namespace##__type_name), expr)
1525
1526 #define zalloc_id(zid, flags) __zalloc_cast(zid, (zalloc_id)(zid, flags))
1527 #define zalloc_ro(zid, flags) __zalloc_cast(zid, (zalloc_ro)(zid, flags))
1528 #if ZALLOC_TYPE_SAFE
1529 #define zalloc(zov) __zalloc_cast(zov, (zalloc)(zov))
1530 #define zalloc_noblock(zov) __zalloc_cast(zov, (zalloc_noblock)(zov))
1531 #define zalloc_flags(zov, fl) __zalloc_cast(zov, (zalloc_flags)(zov, fl))
1532 #endif
1533
1534 struct zone_view_startup_spec {
1535 zone_view_t zv_view;
1536 union {
1537 zone_kheap_id_t zv_heapid;
1538 zone_t *zv_zone;
1539 };
1540 vm_size_t zv_size;
1541 };
1542
1543 extern void zone_view_startup_init(
1544 struct zone_view_startup_spec *spec);
1545
1546 extern void zone_userspace_reboot_checks(void);
1547
1548 #if VM_TAG_SIZECLASSES
1549 extern void __zone_site_register(
1550 vm_allocation_site_t *site);
1551
1552 #define VM_ALLOC_SITE_TAG() ({ \
1553 __PLACE_IN_SECTION("__DATA, __data") \
1554 static vm_allocation_site_t site = { .refcount = 2, }; \
1555 STARTUP_ARG(ZALLOC, STARTUP_RANK_LAST, __zone_site_register, &site); \
1556 site.tag; \
1557 })
1558 #else /* VM_TAG_SIZECLASSES */
1559 #define VM_ALLOC_SITE_TAG() VM_KERN_MEMORY_NONE
1560 #endif /* !VM_TAG_SIZECLASSES */
1561
1562 static inline zalloc_flags_t
__zone_flags_mix_tag(zalloc_flags_t flags,vm_tag_t tag)1563 __zone_flags_mix_tag(zalloc_flags_t flags, vm_tag_t tag)
1564 {
1565 return (flags & Z_VM_TAG_MASK) ? flags : Z_VM_TAG(flags, (uint32_t)tag);
1566 }
1567
1568 #if DEBUG || DEVELOPMENT
1569 # if __LP64__
1570 # define ZPCPU_MANGLE_BIT (1ul << 63)
1571 # else /* !__LP64__ */
1572 # define ZPCPU_MANGLE_BIT (1ul << 31)
1573 # endif /* !__LP64__ */
1574 #else /* !(DEBUG || DEVELOPMENT) */
1575 # define ZPCPU_MANGLE_BIT 0ul
1576 #endif /* !(DEBUG || DEVELOPMENT) */
1577
1578 #define __zpcpu_mangle(ptr) (__zpcpu_addr(ptr) & ~ZPCPU_MANGLE_BIT)
1579 #define __zpcpu_demangle(ptr) (__zpcpu_addr(ptr) | ZPCPU_MANGLE_BIT)
1580 #define __zpcpu_addr(e) ((vm_address_t)(e))
1581 #define __zpcpu_cast(ptr, e) __unsafe_forge_single(typeof(ptr), e)
1582 #define __zpcpu_next(ptr) __zpcpu_cast(ptr, __zpcpu_addr(ptr) + PAGE_SIZE)
1583
1584 /**
1585 * @macro __zpcpu_mangle_for_boot()
1586 *
1587 * @discussion
1588 * Per-cpu variables allocated in zones (as opposed to percpu globals) that need
1589 * to function early during boot (before @c STARTUP_SUB_ZALLOC) might use static
1590 * storage marked @c __startup_data and replace it with the proper allocation
1591 * at the end of the @c STARTUP_SUB_ZALLOC phase (@c STARTUP_RANK_LAST).
1592 *
1593 * However, some devices boot from a cpu where @c cpu_number() != 0. This macro
1594 * provides the proper mangling of the storage into a "fake" percpu pointer so
1595 * that accesses through @c zpercpu_get() functions properly.
1596 *
1597 * This is invalid to use after the @c STARTUP_SUB_ZALLOC phase has completed.
1598 */
1599 #define __zpcpu_mangle_for_boot(ptr) ({ \
1600 assert(startup_phase < STARTUP_SUB_ZALLOC); \
1601 __zpcpu_cast(ptr, __zpcpu_mangle(__zpcpu_addr(ptr) - ptoa(cpu_number()))); \
1602 })
1603
1604 extern unsigned zpercpu_count(void) __pure2;
1605
1606 #if CONFIG_PROB_GZALLOC
1607
1608 extern vm_offset_t __pgz_decode(
1609 mach_vm_address_t addr,
1610 mach_vm_size_t size);
1611
1612 extern vm_offset_t __pgz_decode_allow_invalid(
1613 vm_offset_t offs,
1614 zone_id_t zid);
1615
1616 #endif
1617 #if DEBUG || DEVELOPMENT
1618 extern size_t zone_pages_wired;
1619 extern size_t zone_guard_pages;
1620 #endif /* DEBUG || DEVELOPMENT */
1621 #if CONFIG_ZLEAKS
1622 extern uint32_t zleak_active;
1623 extern vm_size_t zleak_max_zonemap_size;
1624 extern vm_size_t zleak_global_tracking_threshold;
1625 extern vm_size_t zleak_per_zone_tracking_threshold;
1626
1627 extern kern_return_t zleak_update_threshold(
1628 vm_size_t *arg,
1629 uint64_t value);
1630 #endif /* CONFIG_ZLEAKS */
1631
1632 extern uint32_t zone_map_jetsam_limit;
1633
1634 extern kern_return_t zone_map_jetsam_set_limit(uint32_t value);
1635
1636 extern zone_t percpu_u64_zone;
1637
1638 #pragma GCC visibility pop
1639 #endif /* XNU_KERNEL_PRIVATE */
1640
1641 /*
1642 * This macro is currently used by AppleImage4 (rdar://83924635)
1643 */
1644 #define __zalloc_ptr_load_and_erase(elem) \
1645 os_ptr_load_and_erase(elem)
1646
1647 __ASSUME_PTR_ABI_SINGLE_END __END_DECLS
1648
1649 #endif /* _KERN_ZALLOC_H_ */
1650
1651 #endif /* KERNEL_PRIVATE */
1652