1 /*
2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: mach/vm_param.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Machine independent virtual memory parameters.
64 *
65 */
66
67 #ifndef _MACH_VM_PARAM_H_
68 #define _MACH_VM_PARAM_H_
69
70 #include <mach/machine/vm_param.h>
71
72 #ifdef KERNEL
73
74 #include <vm/vm_memtag.h>
75
76 #ifndef ASSEMBLER
77 #include <mach/vm_types.h>
78 #endif /* ASSEMBLER */
79
80 #include <os/base.h>
81 #include <os/overflow.h>
82
83 /*
84 * The machine independent pages are refered to as PAGES. A page
85 * is some number of hardware pages, depending on the target machine.
86 */
87
88 #ifndef ASSEMBLER
89
90 #define PAGE_SIZE_64 (unsigned long long)PAGE_SIZE /* pagesize in addr units */
91 #define PAGE_MASK_64 (unsigned long long)PAGE_MASK /* mask for off in page */
92
93 /*
94 * Convert addresses to pages and vice versa. No rounding is used.
95 * The atop_32 and ptoa_32 macros should not be use on 64 bit types.
96 * The round_page_64 and trunc_page_64 macros should be used instead.
97 */
98
99 #define atop_32(x) ((uint32_t)(x) >> PAGE_SHIFT)
100 #define ptoa_32(x) ((uint32_t)(x) << PAGE_SHIFT)
101 #define atop_64(x) ((uint64_t)(x) >> PAGE_SHIFT)
102 #define ptoa_64(x) ((uint64_t)(x) << PAGE_SHIFT)
103
104 #define atop_kernel(x) ((vm_address_t)(x) >> PAGE_SHIFT)
105 #define ptoa_kernel(x) ((vm_address_t)(x) << PAGE_SHIFT)
106
107 /*
108 * While the following block is enabled, the legacy atop and ptoa
109 * macros will behave correctly. If not, they will generate
110 * invalid lvalue errors.
111 */
112
113 #if 1
114 #define atop(x) ((vm_address_t)(x) >> PAGE_SHIFT)
115 #define ptoa(x) ((vm_address_t)(x) << PAGE_SHIFT)
116 #else
117 #define atop(x) (0UL = 0)
118 #define ptoa(x) (0UL = 0)
119 #endif
120
121 /*
122 * Page-size rounding macros for the Public fixed-width VM types.
123 */
124 #define mach_vm_round_page(x) (((mach_vm_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
125 #define mach_vm_trunc_page(x) ((mach_vm_offset_t)(x) & ~((signed)PAGE_MASK))
126
127 #define round_page_overflow(in, out) __os_warn_unused(({ \
128 bool __ovr = os_add_overflow(in, (__typeof__(*out))PAGE_MASK, out); \
129 *out &= ~((__typeof__(*out))PAGE_MASK); \
130 __ovr; \
131 }))
132
133 static inline int OS_WARN_RESULT
mach_vm_round_page_overflow(mach_vm_offset_t in,mach_vm_offset_t * out)134 mach_vm_round_page_overflow(mach_vm_offset_t in, mach_vm_offset_t *out)
135 {
136 return round_page_overflow(in, out);
137 }
138
139 #define memory_object_round_page(x) (((memory_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
140 #define memory_object_trunc_page(x) ((memory_object_offset_t)(x) & ~((signed)PAGE_MASK))
141
142 /*
143 * Rounding macros for the legacy (scalable with the current task's
144 * address space size) VM types.
145 */
146
147 #define round_page(x) (((vm_offset_t)(x) + PAGE_MASK) & ~((vm_offset_t)PAGE_MASK))
148 #define trunc_page(x) ((vm_offset_t)(x) & ~((vm_offset_t)PAGE_MASK))
149
150 /*
151 * Round off or truncate to the nearest page. These will work
152 * for either addresses or counts. (i.e. 1 byte rounds to 1 page
153 * bytes. The round_page_32 and trunc_page_32 macros should not be
154 * use on 64 bit types. The round_page_64 and trunc_page_64 macros
155 * should be used instead.
156 *
157 * These should only be used in the rare case the size of the address
158 * or length is hard-coded as 32 or 64 bit. Otherwise, the macros
159 * associated with the specific VM type should be used.
160 */
161
162 #define round_page_32(x) (((uint32_t)(x) + PAGE_MASK) & ~((uint32_t)PAGE_MASK))
163 #define trunc_page_32(x) ((uint32_t)(x) & ~((uint32_t)PAGE_MASK))
164 #define round_page_64(x) (((uint64_t)(x) + PAGE_MASK_64) & ~((uint64_t)PAGE_MASK_64))
165 #define trunc_page_64(x) ((uint64_t)(x) & ~((uint64_t)PAGE_MASK_64))
166
167 #define round_page_mask_32(x, mask) (((uint32_t)(x) + (mask)) & ~((uint32_t)(mask)))
168 #define trunc_page_mask_32(x, mask) ((uint32_t)(x) & ~((uint32_t)(mask)))
169 #define round_page_mask_64(x, mask) (((uint64_t)(x) + (mask)) & ~((uint64_t)(mask)))
170 #define trunc_page_mask_64(x, mask) ((uint64_t)(x) & ~((uint64_t)(mask)))
171
172 /*
173 * Enable the following block to find uses of xxx_32 macros that should
174 * be xxx_64. These macros only work in C code, not C++. The resulting
175 * binaries are not functional. Look for invalid lvalue errors in
176 * the compiler output.
177 *
178 * Enabling the following block will also find use of the xxx_64 macros
179 * that have been passed pointers. The parameters should be case to an
180 * unsigned long type first. Look for invalid operands to binary + error
181 * in the compiler output.
182 */
183
184 #if 0
185 #undef atop_32
186 #undef ptoa_32
187 #undef round_page_32
188 #undef trunc_page_32
189 #undef atop_64
190 #undef ptoa_64
191 #undef round_page_64
192 #undef trunc_page_64
193
194 #ifndef __cplusplus
195
196 #define atop_32(x) \
197 (__builtin_choose_expr (sizeof(x) != sizeof(uint64_t), \
198 (*(long *)0), \
199 (0UL)) = 0)
200
201 #define ptoa_32(x) \
202 (__builtin_choose_expr (sizeof(x) != sizeof(uint64_t), \
203 (*(long *)0), \
204 (0UL)) = 0)
205
206 #define round_page_32(x) \
207 (__builtin_choose_expr (sizeof(x) != sizeof(uint64_t), \
208 (*(long *)0), \
209 (0UL)) = 0)
210
211 #define trunc_page_32(x) \
212 (__builtin_choose_expr (sizeof(x) != sizeof(uint64_t), \
213 (*(long *)0), \
214 (0UL)) = 0)
215 #else
216
217 #define atop_32(x) (0)
218 #define ptoa_32(x) (0)
219 #define round_page_32(x) (0)
220 #define trunc_page_32(x) (0)
221
222 #endif /* ! __cplusplus */
223
224 #define atop_64(x) ((uint64_t)((x) + (uint8_t *)0))
225 #define ptoa_64(x) ((uint64_t)((x) + (uint8_t *)0))
226 #define round_page_64(x) ((uint64_t)((x) + (uint8_t *)0))
227 #define trunc_page_64(x) ((uint64_t)((x) + (uint8_t *)0))
228
229 #endif
230
231 /*
232 * Determine whether an address is page-aligned, or a count is
233 * an exact page multiple.
234 */
235
236 #define page_aligned(x) (((x) & PAGE_MASK) == 0)
237
238 extern vm_size_t mem_size; /* 32-bit size of memory - limited by maxmem - deprecated */
239 extern uint64_t max_mem; /* 64-bit size of memory - limited by maxmem */
240
241 /*
242 * The VM compressor pager uses 32-bit page numbers, so this limits the size
243 * of anonymous memory objects to 0xffffffff pages.
244 * When we need to allocate a chunk of anonymous memory over that size,
245 * we have to allocate more than one chunk.
246 */
247 #define ANON_MAX_PAGES 0xFFFFFFFFULL
248 #define ANON_MAX_SIZE (ANON_MAX_PAGES << PAGE_SHIFT)
249 /*
250 * Work-around for <rdar://problem/6626493>
251 * Break large anonymous memory areas into 128MB chunks to alleviate
252 * the cost of copying when copy-on-write is not possible because a small
253 * portion of it being wired.
254 */
255 #define ANON_CHUNK_SIZE (128ULL * 1024 * 1024) /* 128MB */
256
257 /*
258 * The 'medium' malloc allocator would like its regions
259 * to be chunked up into MALLOC_MEDIUM_CHUNK_SIZE chunks
260 * and backed by different objects. This avoids contention
261 * on a single large object and showed solid improvements on high
262 * core machines with workloads involving video and graphics processing.
263 */
264 #define MALLOC_MEDIUM_CHUNK_SIZE (8ULL * 1024 * 1024) /* 8 MB */
265
266 #ifdef KERNEL_PRIVATE
267 extern uint64_t sane_size; /* Memory size to use for defaults calculations */
268 #endif /* KERNEL_PRIVATE */
269
270 #ifdef XNU_KERNEL_PRIVATE
271
272 #include <kern/debug.h>
273 #include <vm/vm_memtag.h>
274
275 extern uint64_t mem_actual; /* 64-bit size of memory - not limited by maxmem */
276 extern uint64_t max_mem_actual; /* Size of physical memory adjusted by maxmem */
277 extern addr64_t vm_last_addr; /* Highest kernel virtual address known to the VM system */
278 extern addr64_t first_avail_phys; /* First available physical address */
279
280 extern const vm_offset_t vm_min_kernel_address;
281 extern const vm_offset_t vm_max_kernel_address;
282
283 extern vm_offset_t vm_kernel_stext;
284 extern vm_offset_t vm_kernel_etext;
285 extern vm_offset_t vm_kernel_slid_base;
286 extern vm_offset_t vm_kernel_slid_top;
287 extern vm_offset_t vm_kernel_slide;
288
289 #if CONFIG_SPTM
290 typedef struct {
291 vm_offset_t unslid_base;
292 vm_offset_t unslid_top;
293 vm_offset_t slid_base;
294 vm_offset_t slid_top;
295 vm_offset_t slide;
296 } vm_image_offsets;
297
298 extern vm_image_offsets vm_sptm_offsets;
299 extern vm_image_offsets vm_txm_offsets;
300 #endif /* CONFIG_SPTM */
301
302 extern vm_offset_t vm_kernel_addrperm;
303 extern vm_offset_t vm_kext_base;
304 extern vm_offset_t vm_kext_top;
305 extern vm_offset_t vm_kernel_base;
306 extern vm_offset_t vm_kernel_top;
307 extern vm_offset_t vm_hib_base;
308
309 extern vm_offset_t vm_kernel_builtinkmod_text;
310 extern vm_offset_t vm_kernel_builtinkmod_text_end;
311
312 /**
313 * While these function's implementations are machine specific, due to the need
314 * to prevent header file circular dependencies, they need to be externed here
315 * for usage in the sliding/unsliding macros.
316 */
317 __BEGIN_DECLS
318 vm_offset_t ml_static_slide(vm_offset_t vaddr);
319 vm_offset_t ml_static_unslide(vm_offset_t vaddr);
320 __END_DECLS
321
322 /**
323 * Determine whether a given address is an address within a static region (i.e.,
324 * coming from TEXT or DATA) that was slid during boot. Addresses of this type
325 * should have the slide removed before exposing them to userspace so as to not
326 * leak the slide itself to userspace.
327 *
328 * @param addr The virtual address to check.
329 *
330 * @return True if the address is a static/slid kernel address, false otherwise.
331 */
332 static inline bool
vm_is_addr_slid(vm_offset_t addr)333 vm_is_addr_slid(vm_offset_t addr)
334 {
335 const vm_offset_t stripped_addr = (vm_offset_t)VM_KERNEL_STRIP_PTR(addr);
336 const bool is_slid_kern_addr =
337 (stripped_addr >= vm_kernel_slid_base) && (stripped_addr < vm_kernel_slid_top);
338
339 #if CONFIG_SPTM
340 const bool is_slid_sptm_addr =
341 (stripped_addr >= vm_sptm_offsets.slid_base) && (stripped_addr < vm_sptm_offsets.slid_top);
342
343 const bool is_slid_txm_addr =
344 (stripped_addr >= vm_txm_offsets.slid_base) && (stripped_addr < vm_txm_offsets.slid_top);
345
346 return is_slid_kern_addr || is_slid_sptm_addr || is_slid_txm_addr;
347 #else
348 return is_slid_kern_addr;
349 #endif /* CONFIG_SPTM */
350 }
351
352 #define VM_KERNEL_IS_SLID(_o) (vm_is_addr_slid((vm_offset_t)(_o)))
353
354 #define VM_KERNEL_SLIDE(_u) (ml_static_slide((vm_offset_t)(_u)))
355
356 /*
357 * The following macros are to be used when exposing kernel addresses to
358 * userspace via any of the various debug or info facilities that might exist
359 * (e.g. stackshot, proc_info syscall, etc.). It is important to understand
360 * the goal of each macro and choose the right one depending on what you are
361 * trying to do. Misuse of these macros can result in critical data leaks
362 * which in turn lead to all sorts of system vulnerabilities. It is invalid to
363 * call these macros on a non-kernel address (NULL is allowed).
364 *
365 * VM_KERNEL_UNSLIDE:
366 * Use this macro when you are exposing an address to userspace which is
367 * *guaranteed* to be a "static" kernel or kext address (i.e. coming from text
368 * or data sections). These are the addresses which get "slid" via ASLR on
369 * kernel or kext load, and it's precisely the slide value we are trying to
370 * protect from userspace.
371 *
372 * VM_KERNEL_ADDRHIDE:
373 * Use when exposing an address for internal purposes: debugging, tracing,
374 * etc. The address will be unslid if necessary. Other addresses will be
375 * hidden on customer builds, and unmodified on internal builds.
376 *
377 * VM_KERNEL_ADDRHASH:
378 * Use this macro when exposing a kernel address to userspace on customer
379 * builds. The address can be from the static kernel or kext regions, or the
380 * kernel heap. The address will be unslid or hashed as appropriate.
381 *
382 *
383 * ** SECURITY WARNING: The following macros can leak kernel secrets.
384 * Use *only* in performance *critical* code.
385 *
386 * VM_KERNEL_ADDRPERM:
387 * VM_KERNEL_UNSLIDE_OR_PERM:
388 * Use these macros when exposing a kernel address to userspace on customer
389 * builds. The address can be from the static kernel or kext regions, or the
390 * kernel heap. The address will be unslid or permuted as appropriate.
391 *
392 * Nesting of these macros should be considered invalid.
393 */
394
395 #define __DO_UNSLIDE(_v) (ml_static_unslide((vm_offset_t)VM_KERNEL_STRIP_PTR(_v)))
396
397 #if DEBUG || DEVELOPMENT
398 #define VM_KERNEL_ADDRHIDE(_v) (VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_address_t)VM_KERNEL_STRIP_PTR(_v))
399 #else
400 #define VM_KERNEL_ADDRHIDE(_v) (VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_address_t)0)
401 #endif /* DEBUG || DEVELOPMENT */
402
403 #define VM_KERNEL_ADDRHASH(_v) vm_kernel_addrhash((vm_offset_t)(_v))
404
405 #define VM_KERNEL_UNSLIDE_OR_PERM(_v) ({ \
406 VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : \
407 VM_KERNEL_ADDRESS(_v) ? ((vm_offset_t)VM_KERNEL_STRIP_PTR(_v) + vm_kernel_addrperm) : \
408 (vm_offset_t)VM_KERNEL_STRIP_PTR(_v); \
409 })
410
411 #define VM_KERNEL_UNSLIDE(_v) ({ \
412 VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_offset_t)0; \
413 })
414
415 #define VM_KERNEL_ADDRPERM(_v) VM_KERNEL_UNSLIDE_OR_PERM(_v)
416
417 #undef mach_vm_round_page
418 #undef round_page
419 #undef round_page_32
420 #undef round_page_64
421
422 static inline int
mach_vm_size_unit(mach_vm_size_t size)423 mach_vm_size_unit(mach_vm_size_t size)
424 {
425 uint32_t bits = 64u - (uint32_t)__builtin_clzll((size / 10) | 1);
426
427 return "BKMGTPE"[bits / 10];
428 }
429
430 static inline uint32_t
mach_vm_size_pretty(mach_vm_size_t size)431 mach_vm_size_pretty(mach_vm_size_t size)
432 {
433 uint32_t bits = 64u - (uint32_t)__builtin_clzll((size / 10) | 1);
434
435 return (uint32_t)(size >> (bits - bits % 10));
436 }
437
438 static inline mach_vm_offset_t
mach_vm_round_page(mach_vm_offset_t x)439 mach_vm_round_page(mach_vm_offset_t x)
440 {
441 if (round_page_overflow(x, &x)) {
442 panic("overflow detected");
443 }
444 return x;
445 }
446
447 static inline vm_offset_t
round_page(vm_offset_t x)448 round_page(vm_offset_t x)
449 {
450 if (round_page_overflow(x, &x)) {
451 panic("overflow detected");
452 }
453 return x;
454 }
455
456 static inline mach_vm_offset_t
round_page_64(mach_vm_offset_t x)457 round_page_64(mach_vm_offset_t x)
458 {
459 if (round_page_overflow(x, &x)) {
460 panic("overflow detected");
461 }
462 return x;
463 }
464
465 static inline uint32_t
round_page_32(uint32_t x)466 round_page_32(uint32_t x)
467 {
468 if (round_page_overflow(x, &x)) {
469 panic("overflow detected");
470 }
471 return x;
472 }
473
474
475 /*!
476 * @typedef vm_packing_params_t
477 *
478 * @brief
479 * Data structure representing the packing parameters for a given packed pointer
480 * encoding.
481 *
482 * @discussion
483 * Several data structures wish to pack their pointers on less than 64bits
484 * on LP64 in order to save memory.
485 *
486 * Adopters are supposed to define 3 macros:
487 * - @c *_BITS: number of storage bits used for the packing,
488 * - @c *_SHIFT: number of non significant low bits (expected to be 0),
489 * - @c *_BASE: the base against which to encode.
490 *
491 * The encoding is a no-op when @c *_BITS is equal to @c __WORDSIZE and
492 * @c *_SHIFT is 0.
493 *
494 *
495 * The convenience macro @c VM_PACKING_PARAMS can be used to create
496 * a @c vm_packing_params_t structure out of those definitions.
497 *
498 * It is customary to declare a constant global per scheme for the sake
499 * of debuggers to be able to dynamically decide how to unpack various schemes.
500 *
501 *
502 * This uses 2 possible schemes (who both preserve @c NULL):
503 *
504 * 1. When the storage bits and shift are sufficiently large (strictly more than
505 * VM_KERNEL_POINTER_SIGNIFICANT_BITS), a sign-extension scheme can be used.
506 *
507 * This allows to represent any kernel pointer.
508 *
509 * 2. Else, a base-relative scheme can be used, typical bases are:
510 *
511 * - @c KERNEL_PMAP_HEAP_RANGE_START when only pointers to heap (zone)
512 * allocated objects need to be packed,
513 *
514 * - @c VM_MIN_KERNEL_AND_KEXT_ADDRESS when pointers to kernel globals also
515 * need this.
516 *
517 * When such an ecoding is used, @c zone_restricted_va_max() must be taught
518 * about it.
519 */
520 typedef struct vm_packing_params {
521 vm_offset_t vmpp_base;
522 uint8_t vmpp_bits;
523 uint8_t vmpp_shift;
524 bool vmpp_base_relative;
525 } vm_packing_params_t;
526
527
528 /*!
529 * @macro VM_PACKING_IS_BASE_RELATIVE
530 *
531 * @brief
532 * Whether the packing scheme with those parameters will be base-relative.
533 */
534 #define VM_PACKING_IS_BASE_RELATIVE(ns) \
535 (ns##_BITS + ns##_SHIFT <= VM_KERNEL_POINTER_SIGNIFICANT_BITS)
536
537
538 /*!
539 * @macro VM_PACKING_PARAMS
540 *
541 * @brief
542 * Constructs a @c vm_packing_params_t structure based on the convention that
543 * macros with the @c _BASE, @c _BITS and @c _SHIFT suffixes have been defined
544 * to the proper values.
545 */
546 #define VM_PACKING_PARAMS(ns) \
547 (vm_packing_params_t){ \
548 .vmpp_base = ns##_BASE, \
549 .vmpp_bits = ns##_BITS, \
550 .vmpp_shift = ns##_SHIFT, \
551 .vmpp_base_relative = VM_PACKING_IS_BASE_RELATIVE(ns), \
552 }
553
554 /**
555 * @function vm_pack_pointer
556 *
557 * @brief
558 * Packs a pointer according to the specified parameters.
559 *
560 * @discussion
561 * The convenience @c VM_PACK_POINTER macro allows to synthesize
562 * the @c params argument.
563 *
564 * @param ptr The pointer to pack.
565 * @param params The encoding parameters.
566 * @returns The packed pointer.
567 */
568 static inline vm_offset_t
vm_pack_pointer(vm_offset_t ptr,vm_packing_params_t params)569 vm_pack_pointer(vm_offset_t ptr, vm_packing_params_t params)
570 {
571 if (ptr != 0) {
572 ptr = vm_memtag_canonicalize_address(ptr);
573 }
574
575 if (!params.vmpp_base_relative) {
576 return ptr >> params.vmpp_shift;
577 }
578 if (ptr) {
579 return (ptr - params.vmpp_base) >> params.vmpp_shift;
580 }
581 return (vm_offset_t)0;
582 }
583 #define VM_PACK_POINTER(ptr, ns) \
584 vm_pack_pointer(ptr, VM_PACKING_PARAMS(ns))
585
586 /**
587 * @function vm_unpack_pointer
588 *
589 * @brief
590 * Unpacks a pointer packed with @c vm_pack_pointer().
591 *
592 * @discussion
593 * The convenience @c VM_UNPACK_POINTER macro allows to synthesize
594 * the @c params argument.
595 *
596 * @param packed The packed value to decode.
597 * @param params The encoding parameters.
598 * @returns The unpacked pointer.
599 */
600 static inline vm_offset_t
vm_unpack_pointer(vm_offset_t packed,vm_packing_params_t params)601 vm_unpack_pointer(vm_offset_t packed, vm_packing_params_t params)
602 {
603 if (!params.vmpp_base_relative) {
604 intptr_t addr = (intptr_t)packed;
605 addr <<= __WORDSIZE - params.vmpp_bits;
606 addr >>= __WORDSIZE - params.vmpp_bits - params.vmpp_shift;
607 return vm_memtag_fixup_ptr((vm_offset_t)addr);
608 }
609 if (packed) {
610 return vm_memtag_fixup_ptr((packed << params.vmpp_shift) + params.vmpp_base);
611 }
612 return (vm_offset_t)0;
613 }
614 #define VM_UNPACK_POINTER(packed, ns) \
615 vm_unpack_pointer(packed, VM_PACKING_PARAMS(ns))
616
617 /**
618 * @function vm_packing_max_packable
619 *
620 * @brief
621 * Returns the largest packable address for the given parameters.
622 *
623 * @discussion
624 * The convenience @c VM_PACKING_MAX_PACKABLE macro allows to synthesize
625 * the @c params argument.
626 *
627 * @param params The encoding parameters.
628 * @returns The largest packable pointer.
629 */
630 static inline vm_offset_t
vm_packing_max_packable(vm_packing_params_t params)631 vm_packing_max_packable(vm_packing_params_t params)
632 {
633 if (!params.vmpp_base_relative) {
634 return VM_MAX_KERNEL_ADDRESS;
635 }
636
637 vm_offset_t ptr = params.vmpp_base +
638 (((1ul << params.vmpp_bits) - 1) << params.vmpp_shift);
639
640 return ptr >= params.vmpp_base ? ptr : VM_MAX_KERNEL_ADDRESS;
641 }
642 #define VM_PACKING_MAX_PACKABLE(ns) \
643 vm_packing_max_packable(VM_PACKING_PARAMS(ns))
644
645
646 __abortlike
647 extern void
648 vm_packing_pointer_invalid(vm_offset_t ptr, vm_packing_params_t params);
649
650 /**
651 * @function vm_verify_pointer_packable
652 *
653 * @brief
654 * Panics if the specified pointer cannot be packed with the specified
655 * parameters.
656 *
657 * @discussion
658 * The convenience @c VM_VERIFY_POINTER_PACKABLE macro allows to synthesize
659 * the @c params argument.
660 *
661 * The convenience @c VM_ASSERT_POINTER_PACKABLE macro allows to synthesize
662 * the @c params argument, and is erased when assertions are disabled.
663 *
664 * @param ptr The packed value to decode.
665 * @param params The encoding parameters.
666 */
667 static inline void
vm_verify_pointer_packable(vm_offset_t ptr,vm_packing_params_t params)668 vm_verify_pointer_packable(vm_offset_t ptr, vm_packing_params_t params)
669 {
670 if (ptr != 0) {
671 ptr = vm_memtag_canonicalize_address(ptr);
672 }
673
674 if (ptr & ((1ul << params.vmpp_shift) - 1)) {
675 vm_packing_pointer_invalid(ptr, params);
676 }
677 if (!params.vmpp_base_relative || ptr == 0) {
678 return;
679 }
680 if (ptr <= params.vmpp_base || ptr > vm_packing_max_packable(params)) {
681 vm_packing_pointer_invalid(ptr, params);
682 }
683 }
684 #define VM_VERIFY_POINTER_PACKABLE(ptr, ns) \
685 vm_verify_pointer_packable(ptr, VM_PACKING_PARAMS(ns))
686
687 #if DEBUG || DEVELOPMENT
688 #define VM_ASSERT_POINTER_PACKABLE(ptr, ns) \
689 VM_VERIFY_POINTER_PACKABLE(ptr, ns)
690 #else
691 #define VM_ASSERT_POINTER_PACKABLE(ptr, ns) ((void)(ptr))
692 #endif
693
694 /**
695 * @function vm_verify_pointer_range
696 *
697 * @brief
698 * Panics if some pointers in the specified range can't be packed with the
699 * specified parameters.
700 *
701 * @param subsystem The subsystem requiring the packing.
702 * @param min_address The smallest address of the range.
703 * @param max_address The largest address of the range.
704 * @param params The encoding parameters.
705 */
706 extern void
707 vm_packing_verify_range(
708 const char *subsystem,
709 vm_offset_t min_address,
710 vm_offset_t max_address,
711 vm_packing_params_t params);
712
713 #endif /* XNU_KERNEL_PRIVATE */
714
715 extern vm_size_t page_size;
716 extern vm_size_t page_mask;
717 extern int page_shift;
718
719 /* We need a way to get rid of compiler warnings when we cast from */
720 /* a 64 bit value to an address (which may be 32 bits or 64-bits). */
721 /* An intptr_t is used convert the value to the right precision, and */
722 /* then to an address. This macro is also used to convert addresses */
723 /* to 32-bit integers, which is a hard failure for a 64-bit kernel */
724 #include <stdint.h>
725 #ifndef __CAST_DOWN_CHECK
726 #define __CAST_DOWN_CHECK
727
728 #define CAST_DOWN( type, addr ) \
729 ( ((type)((uintptr_t) (addr)/(sizeof(type) < sizeof(uintptr_t) ? 0 : 1))) )
730
731 #define CAST_DOWN_EXPLICIT( type, addr ) ( ((type)((uintptr_t) (addr))) )
732
733 #endif /* __CAST_DOWN_CHECK */
734
735 #endif /* ASSEMBLER */
736
737 #endif /* KERNEL */
738
739 #endif /* _MACH_VM_PARAM_H_ */
740