1 /*
2 * Copyright (c) 2025 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/mach_vm.h>
30
31 #include <vm/pmap.h>
32 #include <vm/vm_fault.h>
33 #include <vm/vm_map_internal.h>
34 #include <vm/vm_map_xnu.h>
35 #include <vm/vm_pageout_xnu.h>
36 #include <vm/vm_sanitize_internal.h>
37
38 #include <sys/errno.h> /* for the sysctl tests */
39
40 #if HAS_MTE
41 #include <vm/vm_mteinfo_internal.h>
42
43
44 /*
45 * This test verifies that the KMA_COMPRESSOR flag translates to tag storage
46 * pages getting allocated when `vm_mte_tag_storage_for_compressor` is on.
47 */
48 static int
vm_mte_test_tag_storage_for_compressor(int64_t in,int64_t * out)49 vm_mte_test_tag_storage_for_compressor(int64_t in, int64_t *out)
50 {
51 kern_return_t kr = KERN_FAILURE;
52 kma_flags_t flags = KMA_NOFAIL | KMA_COMPRESSOR;
53 vm_size_t page_count = 0;
54 vm_page_t page = NULL;
55 vm_page_t page_list = NULL;
56 uint64_t cpu_free_claimed_count = 0;
57 uint32_t correct_alloc_count = 0;
58 bool expect_tag_storage = (bool)in;
59 int ret = 0;
60
61 *out = 1;
62 disable_preemption();
63 cpu_free_claimed_count = counter_load(&vm_cpu_free_claimed_count);
64 enable_preemption();
65 if (cpu_free_claimed_count) {
66 page_count = MIN(cpu_free_claimed_count, VMP_FREE_BATCH_SIZE);
67 printf("%s: page count: %lu\n", __func__, page_count);
68 } else {
69 printf("%s: no pages on CPU free claimed queue. Skipping...\n",
70 __func__);
71 *out = 1;
72 return 0;
73 }
74
75 kr = vm_page_alloc_list(page_count, flags, &page_list);
76 if (kr != KERN_SUCCESS) {
77 printf("%s: vm_page_alloc_list failed, kr=%d\n", __func__, kr);
78 *out = 0;
79 return ENOMEM;
80 }
81
82 /*
83 * We can't guarantee that _all_ pages will be tag storage (if we want that),
84 * so if at least one page is tag storage when we want it to be, we will be
85 * happy.
86 *
87 * If we are not expecting tag storage, _no_ page can be tag storage.
88 */
89 _vm_page_list_foreach(page, page_list) {
90 if (expect_tag_storage == vm_page_is_tag_storage(page)) {
91 correct_alloc_count++;
92 }
93 }
94 if ((expect_tag_storage && !correct_alloc_count) ||
95 (!expect_tag_storage && (correct_alloc_count != page_count))) {
96 printf("%s: %lu tag storage pages were allocated.\n",
97 __func__,
98 expect_tag_storage ? correct_alloc_count : (page_count - correct_alloc_count));
99 *out = 0;
100 ret = ENOMEM;
101 goto done;
102 }
103
104 done:
105 vm_page_free_list(page_list, FALSE);
106 return ret;
107 }
108 SYSCTL_TEST_REGISTER(vm_mte_tag_storage_for_compressor, vm_mte_test_tag_storage_for_compressor);
109
110 /*
111 * This test verifies that if VM_MEMORY_STACK is specified in the mte_ts_vmtag
112 * boot-arg, we get a tag storage page back.
113 */
114 static int
vm_mte_test_tag_storage_for_vm_tag(int64_t in,int64_t * out)115 vm_mte_test_tag_storage_for_vm_tag(int64_t in, int64_t *out)
116 {
117 kern_return_t kr = KERN_FAILURE;
118 mach_vm_offset_ut addr_u;
119 mach_vm_address_t addr;
120 uint64_t cpu_free_claimed_count = 0;
121 ppnum_t pnum = -1;
122 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_ANYWHERE();
123 int error = -1;
124 int ret = 0;
125 struct {
126 uint32_t vm_tag;
127 bool mte;
128 bool expect_ts;
129 } args;
130
131 *out = 1;
132 error = copyin(in, &args, sizeof(args));
133 if (error) {
134 printf("%s: copyin from userspace failed, error: %d.\n",
135 __func__, error);
136 *out = 0;
137 return 1;
138 }
139
140 vmk_flags.vmf_mte = args.mte;
141 vmk_flags.vm_tag = args.vm_tag;
142 disable_preemption();
143 cpu_free_claimed_count = counter_load(&vm_cpu_free_claimed_count);
144 enable_preemption();
145 if (!cpu_free_claimed_count) {
146 printf("%s: no pages on CPU free claimed queue.\n", __func__);
147 *out = 1;
148 return 0;
149 }
150 printf("%s: free claimed count: %llu\n",
151 __func__, cpu_free_claimed_count);
152
153 kr = mach_vm_allocate_kernel(current_map(), &addr_u, vm_sanitize_wrap_size(PAGE_SIZE),
154 vmk_flags);
155 if (kr != KERN_SUCCESS) {
156 printf("%s: mach_vm_allocate_kernel failed with kr=%d.\n",
157 __func__, kr);
158 *out = 0;
159 return 1;
160 }
161
162 /* Fault in the page */
163 addr = vm_sanitize_addr(current_map(), addr_u);
164 kr = vm_fault(
165 current_map(),
166 addr,
167 VM_PROT_READ | VM_PROT_WRITE,
168 FALSE, /* change_wiring */
169 VM_KERN_MEMORY_NONE,
170 THREAD_UNINT,
171 NULL,
172 0);
173 if (kr != KERN_SUCCESS) {
174 printf("%s: vm_fault failed with kr=%d.\n", __func__, kr);
175 *out = 0;
176 ret = 1;
177 goto done;
178 }
179
180 /* Check the physical page */
181 pnum = vm_map_get_phys_page(current_map(), (vm_offset_t)addr);
182 if (vm_page_is_tag_storage(vm_page_find_canonical(pnum)) ^ args.expect_ts) {
183 printf("%s: tag: %d, expected tag storage: %d, mte: %d, pnum: %d\n",
184 __func__, args.vm_tag, args.expect_ts, args.mte, pnum);
185 if (args.expect_ts) {
186 /* We can't guarantee that we get a tag storage page, so pass anyway. */
187 goto done;
188 }
189 *out = 0;
190 ret = 1;
191 goto done;
192 }
193
194 done:
195 mach_vm_deallocate(current_map(), addr, PAGE_SIZE);
196 return ret;
197 }
198 SYSCTL_TEST_REGISTER(vm_mte_tag_storage_for_vm_tag, vm_mte_test_tag_storage_for_vm_tag);
199
200 #endif /* HAS_MTE */
201