1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_vm.h>
33 #include <mach/memory_object.h>
34 #include <mach/vm_map.h>
35 #include <mach/vm_statistics.h>
36 #include <mach/vm32_map_server.h>
37 #include <mach/mach_host.h>
38 #include <mach/host_priv.h>
39 #include <mach/upl.h>
40
41 #include <kern/ledger.h>
42 #include <kern/host.h>
43
44 #include <device/device_port.h>
45 #include <vm/memory_object_internal.h>
46 #include <vm/vm_fault.h>
47 #include <vm/vm_fault_internal.h>
48 #include <vm/vm_map_internal.h>
49 #include <vm/vm_object_internal.h>
50 #include <vm/vm_pageout_internal.h>
51 #include <vm/vm_protos.h>
52 #include <vm/vm_memtag.h>
53 #include <vm/vm_memory_entry_xnu.h>
54 #include <vm/vm_kern_xnu.h>
55 #include <vm/vm_iokit.h>
56 #include <vm/vm_page_internal.h>
57 #include <vm/vm_shared_region_xnu.h>
58 #include <vm/vm_far.h>
59 #include <vm/vm_upl.h>
60
61 #include <kern/zalloc.h>
62 #include <kern/zalloc_internal.h>
63
64 #include <sys/code_signing.h>
65 #include <sys/errno.h> /* for the sysctl tests */
66
67 #include <tests/xnupost.h> /* for testing-related functions and macros */
68
69
70 extern ledger_template_t task_ledger_template;
71
72 extern kern_return_t
73 vm_map_copy_adjust_to_target(
74 vm_map_copy_t copy_map,
75 vm_map_offset_t offset,
76 vm_map_size_t size,
77 vm_map_t target_map,
78 boolean_t copy,
79 vm_map_copy_t *target_copy_map_p,
80 vm_map_offset_t *overmap_start_p,
81 vm_map_offset_t *overmap_end_p,
82 vm_map_offset_t *trimmed_start_p);
83
84 #define VM_TEST_COLLAPSE_COMPRESSOR 0
85 #define VM_TEST_WIRE_AND_EXTRACT 0
86 #define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0
87 #if __arm64__
88 #define VM_TEST_KERNEL_OBJECT_FAULT 0
89 #endif /* __arm64__ */
90 #define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG)
91
92 #if VM_TEST_COLLAPSE_COMPRESSOR
93 extern boolean_t vm_object_collapse_compressor_allowed;
94 #include <IOKit/IOLib.h>
95 static void
vm_test_collapse_compressor(void)96 vm_test_collapse_compressor(void)
97 {
98 vm_object_size_t backing_size, top_size;
99 vm_object_t backing_object, top_object;
100 vm_map_offset_t backing_offset, top_offset;
101 unsigned char *backing_address, *top_address;
102 kern_return_t kr;
103
104 printf("VM_TEST_COLLAPSE_COMPRESSOR:\n");
105
106 /* create backing object */
107 backing_size = 15 * PAGE_SIZE;
108 backing_object = vm_object_allocate(backing_size, kernel_map->serial_id);
109 assert(backing_object != VM_OBJECT_NULL);
110 printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
111 backing_object);
112 /* map backing object */
113 backing_offset = 0;
114 kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
115 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(),
116 backing_object, 0, FALSE,
117 VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
118 assert(kr == KERN_SUCCESS);
119 backing_address = (unsigned char *) backing_offset;
120 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
121 "mapped backing object %p at 0x%llx\n",
122 backing_object, (uint64_t) backing_offset);
123 /* populate with pages to be compressed in backing object */
124 backing_address[0x1 * PAGE_SIZE] = 0xB1;
125 backing_address[0x4 * PAGE_SIZE] = 0xB4;
126 backing_address[0x7 * PAGE_SIZE] = 0xB7;
127 backing_address[0xa * PAGE_SIZE] = 0xBA;
128 backing_address[0xd * PAGE_SIZE] = 0xBD;
129 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
130 "populated pages to be compressed in "
131 "backing_object %p\n", backing_object);
132 /* compress backing object */
133 vm_object_pageout(backing_object);
134 printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
135 backing_object);
136 /* wait for all the pages to be gone */
137 while (*(volatile int *)&backing_object->resident_page_count != 0) {
138 IODelay(10);
139 }
140 printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
141 backing_object);
142 /* populate with pages to be resident in backing object */
143 backing_address[0x0 * PAGE_SIZE] = 0xB0;
144 backing_address[0x3 * PAGE_SIZE] = 0xB3;
145 backing_address[0x6 * PAGE_SIZE] = 0xB6;
146 backing_address[0x9 * PAGE_SIZE] = 0xB9;
147 backing_address[0xc * PAGE_SIZE] = 0xBC;
148 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
149 "populated pages to be resident in "
150 "backing_object %p\n", backing_object);
151 /* leave the other pages absent */
152 /* mess with the paging_offset of the backing_object */
153 assert(backing_object->paging_offset == 0);
154 backing_object->paging_offset = 3 * PAGE_SIZE;
155
156 /* create top object */
157 top_size = 9 * PAGE_SIZE;
158 top_object = vm_object_allocate(top_size, backing_object->vmo_provenance);
159 assert(top_object != VM_OBJECT_NULL);
160 printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
161 top_object);
162 /* map top object */
163 top_offset = 0;
164 kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
165 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(),
166 top_object, 0, FALSE,
167 VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
168 assert(kr == KERN_SUCCESS);
169 top_address = (unsigned char *) top_offset;
170 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
171 "mapped top object %p at 0x%llx\n",
172 top_object, (uint64_t) top_offset);
173 /* populate with pages to be compressed in top object */
174 top_address[0x3 * PAGE_SIZE] = 0xA3;
175 top_address[0x4 * PAGE_SIZE] = 0xA4;
176 top_address[0x5 * PAGE_SIZE] = 0xA5;
177 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
178 "populated pages to be compressed in "
179 "top_object %p\n", top_object);
180 /* compress top object */
181 vm_object_pageout(top_object);
182 printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
183 top_object);
184 /* wait for all the pages to be gone */
185 while (top_object->resident_page_count != 0) {
186 IODelay(10);
187 }
188 printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
189 top_object);
190 /* populate with pages to be resident in top object */
191 top_address[0x0 * PAGE_SIZE] = 0xA0;
192 top_address[0x1 * PAGE_SIZE] = 0xA1;
193 top_address[0x2 * PAGE_SIZE] = 0xA2;
194 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
195 "populated pages to be resident in "
196 "top_object %p\n", top_object);
197 /* leave the other pages absent */
198
199 /* link the 2 objects */
200 vm_object_reference(backing_object);
201 top_object->shadow = backing_object;
202 top_object->vo_shadow_offset = 3 * PAGE_SIZE;
203 printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
204 top_object, backing_object);
205
206 /* unmap backing object */
207 vm_map_remove(kernel_map,
208 backing_offset,
209 backing_offset + backing_size,
210 VM_MAP_REMOVE_NO_FLAGS);
211 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
212 "unmapped backing_object %p [0x%llx:0x%llx]\n",
213 backing_object,
214 (uint64_t) backing_offset,
215 (uint64_t) (backing_offset + backing_size));
216
217 /* collapse */
218 printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
219 vm_object_lock(top_object);
220 vm_object_collapse(top_object, 0, FALSE);
221 vm_object_unlock(top_object);
222 printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object);
223
224 /* did it work? */
225 if (top_object->shadow != VM_OBJECT_NULL) {
226 printf("VM_TEST_COLLAPSE_COMPRESSOR: not collapsed\n");
227 printf("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
228 if (vm_object_collapse_compressor_allowed) {
229 panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL");
230 }
231 } else {
232 /* check the contents of the mapping */
233 unsigned char expect[9] =
234 { 0xA0, 0xA1, 0xA2, /* resident in top */
235 0xA3, 0xA4, 0xA5, /* compressed in top */
236 0xB9, /* resident in backing + shadow_offset */
237 0xBD, /* compressed in backing + shadow_offset + paging_offset */
238 0x00 }; /* absent in both */
239 unsigned char actual[9];
240 unsigned int i, errors;
241
242 errors = 0;
243 for (i = 0; i < sizeof(actual); i++) {
244 actual[i] = (unsigned char) top_address[i * PAGE_SIZE];
245 if (actual[i] != expect[i]) {
246 errors++;
247 }
248 }
249 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
250 "actual [%x %x %x %x %x %x %x %x %x] "
251 "expect [%x %x %x %x %x %x %x %x %x] "
252 "%d errors\n",
253 actual[0], actual[1], actual[2], actual[3],
254 actual[4], actual[5], actual[6], actual[7],
255 actual[8],
256 expect[0], expect[1], expect[2], expect[3],
257 expect[4], expect[5], expect[6], expect[7],
258 expect[8],
259 errors);
260 if (errors) {
261 panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL");
262 } else {
263 printf("VM_TEST_COLLAPSE_COMPRESSOR: PASS\n");
264 }
265 }
266 }
267 #else /* VM_TEST_COLLAPSE_COMPRESSOR */
268 #define vm_test_collapse_compressor()
269 #endif /* VM_TEST_COLLAPSE_COMPRESSOR */
270
271 #if VM_TEST_WIRE_AND_EXTRACT
272 extern ppnum_t vm_map_get_phys_page(vm_map_t map,
273 vm_offset_t offset);
274 static void
vm_test_wire_and_extract(void)275 vm_test_wire_and_extract(void)
276 {
277 ledger_t ledger;
278 vm_map_t user_map, wire_map;
279 mach_vm_address_t user_addr, wire_addr;
280 mach_vm_size_t user_size, wire_size;
281 mach_vm_offset_t cur_offset;
282 vm_prot_t cur_prot, max_prot;
283 ppnum_t user_ppnum, wire_ppnum;
284 kern_return_t kr;
285
286 ledger = ledger_instantiate(task_ledger_template,
287 LEDGER_CREATE_ACTIVE_ENTRIES);
288 pmap_t user_pmap = pmap_create_options(ledger, 0, PMAP_CREATE_64BIT);
289 assert(user_pmap);
290 user_map = vm_map_create_options(user_pmap,
291 0x100000000ULL,
292 0x200000000ULL,
293 VM_MAP_CREATE_PAGEABLE);
294 wire_map = vm_map_create_options(NULL,
295 0x100000000ULL,
296 0x200000000ULL,
297 VM_MAP_CREATE_PAGEABLE);
298 user_addr = 0;
299 user_size = 0x10000;
300 kr = mach_vm_allocate(user_map,
301 &user_addr,
302 user_size,
303 VM_FLAGS_ANYWHERE);
304 assert(kr == KERN_SUCCESS);
305 wire_addr = 0;
306 wire_size = user_size;
307 kr = mach_vm_remap(wire_map,
308 &wire_addr,
309 wire_size,
310 0,
311 VM_FLAGS_ANYWHERE,
312 user_map,
313 user_addr,
314 FALSE,
315 &cur_prot,
316 &max_prot,
317 VM_INHERIT_NONE);
318 assert(kr == KERN_SUCCESS);
319 for (cur_offset = 0;
320 cur_offset < wire_size;
321 cur_offset += PAGE_SIZE) {
322 kr = vm_map_wire_and_extract(wire_map,
323 wire_addr + cur_offset,
324 VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
325 TRUE,
326 &wire_ppnum);
327 assert(kr == KERN_SUCCESS);
328 user_ppnum = vm_map_get_phys_page(user_map,
329 user_addr + cur_offset);
330 printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x "
331 "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
332 kr,
333 user_map, user_addr + cur_offset, user_ppnum,
334 wire_map, wire_addr + cur_offset, wire_ppnum);
335 if (kr != KERN_SUCCESS ||
336 wire_ppnum == 0 ||
337 wire_ppnum != user_ppnum) {
338 panic("VM_TEST_WIRE_AND_EXTRACT: FAIL");
339 }
340 }
341 cur_offset -= PAGE_SIZE;
342 kr = vm_map_wire_and_extract(wire_map,
343 wire_addr + cur_offset,
344 VM_PROT_DEFAULT,
345 TRUE,
346 &wire_ppnum);
347 assert(kr == KERN_SUCCESS);
348 printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
349 "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
350 kr,
351 user_map, user_addr + cur_offset, user_ppnum,
352 wire_map, wire_addr + cur_offset, wire_ppnum);
353 if (kr != KERN_SUCCESS ||
354 wire_ppnum == 0 ||
355 wire_ppnum != user_ppnum) {
356 panic("VM_TEST_WIRE_AND_EXTRACT: FAIL");
357 }
358
359 printf("VM_TEST_WIRE_AND_EXTRACT: PASS\n");
360 }
361 #else /* VM_TEST_WIRE_AND_EXTRACT */
362 #define vm_test_wire_and_extract()
363 #endif /* VM_TEST_WIRE_AND_EXTRACT */
364
365 #if VM_TEST_PAGE_WIRE_OVERFLOW_PANIC
366 static void
vm_test_page_wire_overflow_panic(void)367 vm_test_page_wire_overflow_panic(void)
368 {
369 vm_object_t object;
370 vm_page_t page;
371
372 printf("VM_TEST_PAGE_WIRE_OVERFLOW_PANIC: starting...\n");
373
374 object = vm_object_allocate(PAGE_SIZE, VM_MAP_SERIAL_NONE);
375 while ((page = vm_page_grab()) == VM_PAGE_NULL) {
376 VM_PAGE_WAIT();
377 }
378 vm_object_lock(object);
379 vm_page_insert(page, object, 0);
380 vm_page_lock_queues();
381 do {
382 vm_page_wire(page, 1, FALSE);
383 } while (page->wire_count != 0);
384 vm_page_unlock_queues();
385 vm_object_unlock(object);
386 panic("FBDP(%p,%p): wire_count overflow not detected",
387 object, page);
388 }
389 #else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
390 #define vm_test_page_wire_overflow_panic()
391 #endif /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
392
393 #if __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT
394 extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit);
395 static void
vm_test_kernel_object_fault(void)396 vm_test_kernel_object_fault(void)
397 {
398 vm_offset_t stack;
399 uintptr_t frameb[2];
400 int ret;
401
402 kmem_alloc(kernel_map, &stack,
403 kernel_stack_size + ptoa(2),
404 KMA_NOFAIL | KMA_KSTACK | KMA_KOBJECT |
405 KMA_GUARD_FIRST | KMA_GUARD_LAST,
406 VM_KERN_MEMORY_STACK);
407
408 ret = copyinframe((uintptr_t)stack, (char *)frameb, TRUE);
409 if (ret != 0) {
410 printf("VM_TEST_KERNEL_OBJECT_FAULT: PASS\n");
411 } else {
412 printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n");
413 }
414
415 kmem_free_guard(kernel_map, stack, kernel_stack_size + ptoa(2),
416 KMF_GUARD_FIRST | KMF_GUARD_LAST, KMEM_GUARD_NONE);
417 stack = 0;
418 }
419 #else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
420 #define vm_test_kernel_object_fault()
421 #endif /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
422
423 #if VM_TEST_DEVICE_PAGER_TRANSPOSE
424 static void
vm_test_device_pager_transpose(void)425 vm_test_device_pager_transpose(void)
426 {
427 memory_object_t device_pager;
428 vm_object_t anon_object, device_object;
429 vm_size_t size;
430 vm_map_offset_t device_mapping;
431 kern_return_t kr;
432
433 size = 3 * PAGE_SIZE;
434 anon_object = vm_object_allocate(size, kernel_map->serial_id);
435 assert(anon_object != VM_OBJECT_NULL);
436 device_pager = device_pager_setup(NULL, 0, size, 0);
437 assert(device_pager != NULL);
438 device_object = memory_object_to_vm_object(device_pager);
439 assert(device_object != VM_OBJECT_NULL);
440 #if 0
441 /*
442 * Can't actually map this, since another thread might do a
443 * vm_map_enter() that gets coalesced into this object, which
444 * would cause the test to fail.
445 */
446 vm_map_offset_t anon_mapping = 0;
447 kr = vm_map_enter(kernel_map, &anon_mapping, size, 0,
448 VM_MAP_KERNEL_FLAGS_ANYWHERE(),
449 anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
450 VM_INHERIT_DEFAULT);
451 assert(kr == KERN_SUCCESS);
452 #endif
453 device_mapping = 0;
454 kr = mach_vm_map_kernel(kernel_map,
455 vm_sanitize_wrap_addr_ref(&device_mapping),
456 size,
457 0,
458 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(),
459 (void *)device_pager,
460 0,
461 FALSE,
462 VM_PROT_DEFAULT,
463 VM_PROT_ALL,
464 VM_INHERIT_DEFAULT);
465 assert(kr == KERN_SUCCESS);
466 memory_object_deallocate(device_pager);
467
468 vm_object_lock(anon_object);
469 vm_object_activity_begin(anon_object);
470 anon_object->blocked_access = TRUE;
471 vm_object_unlock(anon_object);
472 vm_object_lock(device_object);
473 vm_object_activity_begin(device_object);
474 device_object->blocked_access = TRUE;
475 vm_object_unlock(device_object);
476
477 assert(os_ref_get_count_raw(&anon_object->ref_count) == 1);
478 assert(!anon_object->named);
479 assert(os_ref_get_count_raw(&device_object->ref_count) == 2);
480 assert(device_object->named);
481
482 kr = vm_object_transpose(device_object, anon_object, size);
483 assert(kr == KERN_SUCCESS);
484
485 vm_object_lock(anon_object);
486 vm_object_activity_end(anon_object);
487 anon_object->blocked_access = FALSE;
488 vm_object_unlock(anon_object);
489 vm_object_lock(device_object);
490 vm_object_activity_end(device_object);
491 device_object->blocked_access = FALSE;
492 vm_object_unlock(device_object);
493
494 assert(os_ref_get_count_raw(&anon_object->ref_count) == 2);
495 assert(anon_object->named);
496 #if 0
497 kr = vm_deallocate(kernel_map, anon_mapping, size);
498 assert(kr == KERN_SUCCESS);
499 #endif
500 assert(os_ref_get_count_raw(&device_object->ref_count) == 1);
501 assert(!device_object->named);
502 kr = vm_deallocate(kernel_map, device_mapping, size);
503 assert(kr == KERN_SUCCESS);
504
505 printf("VM_TEST_DEVICE_PAGER_TRANSPOSE: PASS\n");
506 }
507 #else /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
508 #define vm_test_device_pager_transpose()
509 #endif /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
510
511 extern kern_return_t vm_allocate_external(vm_map_t map,
512 vm_offset_t *addr,
513 vm_size_t size,
514 int flags);
515 extern kern_return_t vm_remap_external(vm_map_t target_map,
516 vm_offset_t *address,
517 vm_size_t size,
518 vm_offset_t mask,
519 int flags,
520 vm_map_t src_map,
521 vm_offset_t memory_address,
522 boolean_t copy,
523 vm_prot_t *cur_protection,
524 vm_prot_t *max_protection,
525 vm_inherit_t inheritance);
526 #if PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT
527 extern int debug4k_panic_on_misaligned_sharing;
528 void vm_test_4k(void);
529 void
vm_test_4k(void)530 vm_test_4k(void)
531 {
532 pmap_t test_pmap;
533 vm_map_t test_map;
534 kern_return_t kr;
535 vm_address_t expected_addr;
536 vm_address_t alloc1_addr, alloc2_addr, alloc3_addr, alloc4_addr;
537 vm_address_t alloc5_addr, dealloc_addr, remap_src_addr, remap_dst_addr;
538 vm_size_t alloc1_size, alloc2_size, alloc3_size, alloc4_size;
539 vm_size_t alloc5_size, remap_src_size;
540 vm_address_t fault_addr;
541 vm_prot_t cur_prot, max_prot;
542 int saved_debug4k_panic_on_misaligned_sharing;
543
544 printf("\n\n\nVM_TEST_4K:%d creating 4K map...\n", __LINE__);
545 test_pmap = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT | PMAP_CREATE_FORCE_4K_PAGES);
546 assert(test_pmap != NULL);
547 test_map = vm_map_create_options(test_pmap,
548 MACH_VM_MIN_ADDRESS,
549 MACH_VM_MAX_ADDRESS,
550 VM_MAP_CREATE_PAGEABLE);
551 assert(test_map != VM_MAP_NULL);
552 vm_map_set_page_shift(test_map, FOURK_PAGE_SHIFT);
553 printf("VM_TEST_4K:%d map %p pmap %p page_size 0x%x\n", __LINE__, test_map, test_pmap, VM_MAP_PAGE_SIZE(test_map));
554
555 alloc1_addr = 0;
556 alloc1_size = 1 * FOURK_PAGE_SIZE;
557 expected_addr = 0x1000;
558 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
559 kr = vm_allocate_external(test_map,
560 &alloc1_addr,
561 alloc1_size,
562 VM_FLAGS_ANYWHERE);
563 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
564 assertf(alloc1_addr == expected_addr, "alloc1_addr = 0x%lx expected 0x%lx", alloc1_addr, expected_addr);
565 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
566 expected_addr += alloc1_size;
567
568 printf("VM_TEST_4K:%d vm_deallocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
569 kr = vm_deallocate(test_map, alloc1_addr, alloc1_size);
570 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
571 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
572
573 alloc1_addr = 0;
574 alloc1_size = 1 * FOURK_PAGE_SIZE;
575 expected_addr = 0x1000;
576 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
577 kr = vm_allocate_external(test_map,
578 &alloc1_addr,
579 alloc1_size,
580 VM_FLAGS_ANYWHERE);
581 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
582 assertf(alloc1_addr == expected_addr, "alloc1_addr = 0x%lx expected 0x%lx", alloc1_addr, expected_addr);
583 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
584 expected_addr += alloc1_size;
585
586 alloc2_addr = 0;
587 alloc2_size = 3 * FOURK_PAGE_SIZE;
588 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc2_addr, alloc2_size);
589 kr = vm_allocate_external(test_map,
590 &alloc2_addr,
591 alloc2_size,
592 VM_FLAGS_ANYWHERE);
593 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
594 assertf(alloc2_addr == expected_addr, "alloc2_addr = 0x%lx expected 0x%lx", alloc2_addr, expected_addr);
595 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc2_addr);
596 expected_addr += alloc2_size;
597
598 alloc3_addr = 0;
599 alloc3_size = 18 * FOURK_PAGE_SIZE;
600 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc3_addr, alloc3_size);
601 kr = vm_allocate_external(test_map,
602 &alloc3_addr,
603 alloc3_size,
604 VM_FLAGS_ANYWHERE);
605 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
606 assertf(alloc3_addr == expected_addr, "alloc3_addr = 0x%lx expected 0x%lx\n", alloc3_addr, expected_addr);
607 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc3_addr);
608 expected_addr += alloc3_size;
609
610 alloc4_addr = 0;
611 alloc4_size = 1 * FOURK_PAGE_SIZE;
612 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc4_addr, alloc4_size);
613 kr = vm_allocate_external(test_map,
614 &alloc4_addr,
615 alloc4_size,
616 VM_FLAGS_ANYWHERE);
617 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
618 assertf(alloc4_addr == expected_addr, "alloc4_addr = 0x%lx expected 0x%lx", alloc4_addr, expected_addr);
619 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc3_addr);
620 expected_addr += alloc4_size;
621
622 printf("VM_TEST_4K:%d vm_protect(%p, 0x%lx, 0x%lx, READ)...\n", __LINE__, test_map, alloc2_addr, (1UL * FOURK_PAGE_SIZE));
623 kr = vm_protect(test_map,
624 alloc2_addr,
625 (1UL * FOURK_PAGE_SIZE),
626 FALSE,
627 VM_PROT_READ);
628 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
629
630 for (fault_addr = alloc1_addr;
631 fault_addr < alloc4_addr + alloc4_size + (2 * FOURK_PAGE_SIZE);
632 fault_addr += FOURK_PAGE_SIZE) {
633 printf("VM_TEST_4K:%d write fault at 0x%lx...\n", __LINE__, fault_addr);
634 kr = vm_fault(test_map,
635 fault_addr,
636 VM_PROT_WRITE,
637 FALSE,
638 VM_KERN_MEMORY_NONE,
639 THREAD_UNINT,
640 NULL,
641 0);
642 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
643 if (fault_addr == alloc2_addr) {
644 assertf(kr == KERN_PROTECTION_FAILURE, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_PROTECTION_FAILURE);
645 printf("VM_TEST_4K:%d read fault at 0x%lx...\n", __LINE__, fault_addr);
646 kr = vm_fault(test_map,
647 fault_addr,
648 VM_PROT_READ,
649 FALSE,
650 VM_KERN_MEMORY_NONE,
651 THREAD_UNINT,
652 NULL,
653 0);
654 assertf(kr == KERN_SUCCESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_SUCCESS);
655 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
656 } else if (fault_addr >= alloc4_addr + alloc4_size) {
657 assertf(kr == KERN_INVALID_ADDRESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_INVALID_ADDRESS);
658 } else {
659 assertf(kr == KERN_SUCCESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_SUCCESS);
660 }
661 }
662
663 alloc5_addr = 0;
664 alloc5_size = 7 * FOURK_PAGE_SIZE;
665 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc5_addr, alloc5_size);
666 kr = vm_allocate_external(test_map,
667 &alloc5_addr,
668 alloc5_size,
669 VM_FLAGS_ANYWHERE);
670 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
671 assertf(alloc5_addr == expected_addr, "alloc5_addr = 0x%lx expected 0x%lx", alloc5_addr, expected_addr);
672 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc5_addr);
673 expected_addr += alloc5_size;
674
675 dealloc_addr = vm_map_round_page(alloc5_addr, PAGE_SHIFT);
676 dealloc_addr += FOURK_PAGE_SIZE;
677 printf("VM_TEST_4K:%d vm_deallocate(%p, 0x%lx, 0x%x)...\n", __LINE__, test_map, dealloc_addr, FOURK_PAGE_SIZE);
678 kr = vm_deallocate(test_map, dealloc_addr, FOURK_PAGE_SIZE);
679 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
680 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
681
682 remap_src_addr = vm_map_round_page(alloc3_addr, PAGE_SHIFT);
683 remap_src_addr += FOURK_PAGE_SIZE;
684 remap_src_size = 2 * FOURK_PAGE_SIZE;
685 remap_dst_addr = 0;
686 printf("VM_TEST_4K:%d vm_remap(%p, 0x%lx, 0x%lx, 0x%lx, copy=0)...\n", __LINE__, test_map, remap_dst_addr, remap_src_size, remap_src_addr);
687 kr = vm_remap_external(test_map,
688 &remap_dst_addr,
689 remap_src_size,
690 0, /* mask */
691 VM_FLAGS_ANYWHERE,
692 test_map,
693 remap_src_addr,
694 FALSE, /* copy */
695 &cur_prot,
696 &max_prot,
697 VM_INHERIT_DEFAULT);
698 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
699 assertf(remap_dst_addr == expected_addr, "remap_dst_addr = 0x%lx expected 0x%lx", remap_dst_addr, expected_addr);
700 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, remap_dst_addr);
701 expected_addr += remap_src_size;
702
703 for (fault_addr = remap_dst_addr;
704 fault_addr < remap_dst_addr + remap_src_size;
705 fault_addr += 4096) {
706 printf("VM_TEST_4K:%d write fault at 0x%lx...\n", __LINE__, fault_addr);
707 kr = vm_fault(test_map,
708 fault_addr,
709 VM_PROT_WRITE,
710 FALSE,
711 VM_KERN_MEMORY_NONE,
712 THREAD_UNINT,
713 NULL,
714 0);
715 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
716 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
717 }
718
719 printf("VM_TEST_4K:\n");
720 remap_dst_addr = 0;
721 remap_src_addr = alloc3_addr + 0xc000;
722 remap_src_size = 0x5000;
723 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=0) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
724 kr = vm_remap_external(kernel_map,
725 &remap_dst_addr,
726 remap_src_size,
727 0, /* mask */
728 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
729 test_map,
730 remap_src_addr,
731 FALSE, /* copy */
732 &cur_prot,
733 &max_prot,
734 VM_INHERIT_DEFAULT);
735 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
736 printf("VM_TEST_4K: -> remapped (shared) in map %p at addr 0x%lx\n", kernel_map, remap_dst_addr);
737
738 printf("VM_TEST_4K:\n");
739 remap_dst_addr = 0;
740 remap_src_addr = alloc3_addr + 0xc000;
741 remap_src_size = 0x5000;
742 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=1) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
743 kr = vm_remap_external(kernel_map,
744 &remap_dst_addr,
745 remap_src_size,
746 0, /* mask */
747 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
748 test_map,
749 remap_src_addr,
750 TRUE, /* copy */
751 &cur_prot,
752 &max_prot,
753 VM_INHERIT_DEFAULT);
754 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
755 printf("VM_TEST_4K: -> remapped (COW) in map %p at addr 0x%lx\n", kernel_map, remap_dst_addr);
756
757 printf("VM_TEST_4K:\n");
758 saved_debug4k_panic_on_misaligned_sharing = debug4k_panic_on_misaligned_sharing;
759 debug4k_panic_on_misaligned_sharing = 0;
760 remap_dst_addr = 0;
761 remap_src_addr = alloc1_addr;
762 remap_src_size = alloc1_size + alloc2_size;
763 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=0) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
764 kr = vm_remap_external(kernel_map,
765 &remap_dst_addr,
766 remap_src_size,
767 0, /* mask */
768 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
769 test_map,
770 remap_src_addr,
771 FALSE, /* copy */
772 &cur_prot,
773 &max_prot,
774 VM_INHERIT_DEFAULT);
775 assertf(kr != KERN_SUCCESS, "kr = 0x%x", kr);
776 printf("VM_TEST_4K: -> remap (SHARED) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
777 debug4k_panic_on_misaligned_sharing = saved_debug4k_panic_on_misaligned_sharing;
778
779 printf("VM_TEST_4K:\n");
780 remap_dst_addr = 0;
781 remap_src_addr = alloc1_addr;
782 remap_src_size = alloc1_size + alloc2_size;
783 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=1) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
784 kr = vm_remap_external(kernel_map,
785 &remap_dst_addr,
786 remap_src_size,
787 0, /* mask */
788 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
789 test_map,
790 remap_src_addr,
791 TRUE, /* copy */
792 &cur_prot,
793 &max_prot,
794 VM_INHERIT_DEFAULT);
795 #if 000
796 assertf(kr != KERN_SUCCESS, "kr = 0x%x", kr);
797 printf("VM_TEST_4K: -> remap (COPY) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
798 #else /* 000 */
799 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
800 printf("VM_TEST_4K: -> remap (COPY) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
801 #endif /* 000 */
802
803
804 #if 00
805 printf("VM_TEST_4K:%d vm_map_remove(%p, 0x%llx, 0x%llx)...\n", __LINE__, test_map, test_map->min_offset, test_map->max_offset);
806 vm_map_remove(test_map, test_map->min_offset, test_map->max_offset);
807 #endif
808
809 printf("VM_TEST_4K: PASS\n\n\n\n");
810 }
811 #endif /* PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT */
812
813 #if MACH_ASSERT
814 static void
vm_test_map_copy_adjust_to_target_one(vm_map_copy_t copy_map,vm_map_t target_map)815 vm_test_map_copy_adjust_to_target_one(
816 vm_map_copy_t copy_map,
817 vm_map_t target_map)
818 {
819 kern_return_t kr;
820 vm_map_copy_t target_copy;
821 vm_map_offset_t overmap_start, overmap_end, trimmed_start;
822
823 target_copy = VM_MAP_COPY_NULL;
824 /* size is 2 (4k) pages but range covers 3 pages */
825 kr = vm_map_copy_adjust_to_target(copy_map,
826 0x0 + 0xfff,
827 0x1002,
828 target_map,
829 FALSE,
830 &target_copy,
831 &overmap_start,
832 &overmap_end,
833 &trimmed_start);
834 assert(kr == KERN_SUCCESS);
835 assert(overmap_start == 0);
836 assert(overmap_end == 0);
837 assert(trimmed_start == 0);
838 assertf(target_copy->size == 0x3000,
839 "target_copy %p size 0x%llx\n",
840 target_copy, (uint64_t)target_copy->size);
841 vm_map_copy_discard(target_copy);
842
843 /* 1. adjust_to_target() for bad offset -> error */
844 /* 2. adjust_to_target() for bad size -> error */
845 /* 3. adjust_to_target() for the whole thing -> unchanged */
846 /* 4. adjust_to_target() to trim start by less than 1 page */
847 /* 5. adjust_to_target() to trim end by less than 1 page */
848 /* 6. adjust_to_target() to trim start and end by less than 1 page */
849 /* 7. adjust_to_target() to trim start by more than 1 page */
850 /* 8. adjust_to_target() to trim end by more than 1 page */
851 /* 9. adjust_to_target() to trim start and end by more than 1 page */
852 /* 10. adjust_to_target() to trim start by more than 1 entry */
853 /* 11. adjust_to_target() to trim start by more than 1 entry */
854 /* 12. adjust_to_target() to trim start and end by more than 1 entry */
855 /* 13. adjust_to_target() to trim start and end down to 1 entry */
856 }
857
858 static void
vm_test_map_copy_adjust_to_target(void)859 vm_test_map_copy_adjust_to_target(void)
860 {
861 kern_return_t kr;
862 vm_map_t map4k, map16k;
863 vm_object_t obj1, obj2, obj3, obj4;
864 vm_map_offset_t addr4k, addr16k;
865 vm_map_size_t size4k, size16k;
866 vm_map_copy_t copy4k, copy16k;
867 vm_prot_t curprot, maxprot;
868 vm_map_kernel_flags_t vmk_flags;
869
870 /* create a 4k map */
871 map4k = vm_map_create_options(PMAP_NULL, 0, (uint32_t)-1,
872 VM_MAP_CREATE_PAGEABLE);
873 vm_map_set_page_shift(map4k, 12);
874
875 /* create a 16k map */
876 map16k = vm_map_create_options(PMAP_NULL, 0, (uint32_t)-1,
877 VM_MAP_CREATE_PAGEABLE);
878 vm_map_set_page_shift(map16k, 14);
879
880 /* create 4 VM objects */
881 obj1 = vm_object_allocate(0x100000, map4k->serial_id);
882 obj2 = vm_object_allocate(0x100000, map4k->serial_id);
883 obj3 = vm_object_allocate(0x100000, map4k->serial_id);
884 obj4 = vm_object_allocate(0x100000, map4k->serial_id);
885
886 /* map objects in 4k map */
887 vm_object_reference(obj1);
888 addr4k = 0x1000;
889 size4k = 0x3000;
890 kr = vm_map_enter(map4k, &addr4k, size4k, 0,
891 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(), obj1, 0,
892 FALSE, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
893 VM_INHERIT_DEFAULT);
894 assert(kr == KERN_SUCCESS);
895 assert(addr4k == 0x1000);
896
897 /* map objects in 16k map */
898 vm_object_reference(obj1);
899 addr16k = 0x4000;
900 size16k = 0x8000;
901 kr = vm_map_enter(map16k, &addr16k, size16k, 0,
902 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(), obj1, 0,
903 FALSE, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
904 VM_INHERIT_DEFAULT);
905 assert(kr == KERN_SUCCESS);
906 assert(addr16k == 0x4000);
907
908 /* test for <rdar://60959809> */
909 ipc_port_t mem_entry;
910 memory_object_size_t mem_entry_size;
911 mach_vm_size_t map_size;
912 mem_entry_size = 0x1002;
913 mem_entry = IPC_PORT_NULL;
914 kr = mach_make_memory_entry_64(map16k, &mem_entry_size, addr16k + 0x2fff,
915 MAP_MEM_VM_SHARE | MAP_MEM_USE_DATA_ADDR | VM_PROT_READ,
916 &mem_entry, IPC_PORT_NULL);
917 assertf(kr == KERN_SUCCESS, "kr 0x%x\n", kr);
918 assertf(mem_entry_size == 0x5001, "mem_entry_size 0x%llx\n", (uint64_t) mem_entry_size);
919 map_size = 0;
920 kr = mach_memory_entry_map_size(mem_entry, map4k, 0, 0x1002, &map_size);
921 assertf(kr == KERN_SUCCESS, "kr 0x%x\n", kr);
922 assertf(map_size == 0x3000, "mem_entry %p map_size 0x%llx\n", mem_entry, (uint64_t)map_size);
923 mach_memory_entry_port_release(mem_entry);
924
925 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
926 vmk_flags.vmkf_remap_legacy_mode = true;
927
928 /* create 4k copy map */
929 curprot = VM_PROT_NONE;
930 maxprot = VM_PROT_NONE;
931 kr = vm_map_copy_extract(map4k, addr4k, 0x3000,
932 FALSE, ©4k, &curprot, &maxprot,
933 VM_INHERIT_DEFAULT, vmk_flags);
934 assert(kr == KERN_SUCCESS);
935 assert(copy4k->size == 0x3000);
936
937 /* create 16k copy map */
938 curprot = VM_PROT_NONE;
939 maxprot = VM_PROT_NONE;
940 kr = vm_map_copy_extract(map16k, addr16k, 0x4000,
941 FALSE, ©16k, &curprot, &maxprot,
942 VM_INHERIT_DEFAULT, vmk_flags);
943 assert(kr == KERN_SUCCESS);
944 assert(copy16k->size == 0x4000);
945
946 /* test each combination */
947 // vm_test_map_copy_adjust_to_target_one(copy4k, map4k);
948 // vm_test_map_copy_adjust_to_target_one(copy16k, map16k);
949 // vm_test_map_copy_adjust_to_target_one(copy4k, map16k);
950 vm_test_map_copy_adjust_to_target_one(copy16k, map4k);
951
952 /* assert 1 ref on 4k map */
953 assert(os_ref_get_count_raw(&map4k->map_refcnt) == 1);
954 /* release 4k map */
955 vm_map_deallocate(map4k);
956 /* assert 1 ref on 16k map */
957 assert(os_ref_get_count_raw(&map16k->map_refcnt) == 1);
958 /* release 16k map */
959 vm_map_deallocate(map16k);
960 /* deallocate copy maps */
961 vm_map_copy_discard(copy4k);
962 vm_map_copy_discard(copy16k);
963 /* assert 1 ref on all VM objects */
964 assert(os_ref_get_count_raw(&obj1->ref_count) == 1);
965 assert(os_ref_get_count_raw(&obj2->ref_count) == 1);
966 assert(os_ref_get_count_raw(&obj3->ref_count) == 1);
967 assert(os_ref_get_count_raw(&obj4->ref_count) == 1);
968 /* release all VM objects */
969 vm_object_deallocate(obj1);
970 vm_object_deallocate(obj2);
971 vm_object_deallocate(obj3);
972 vm_object_deallocate(obj4);
973 }
974 #endif /* MACH_ASSERT */
975
976 #if __arm64__ && !KASAN
977 __attribute__((noinline))
978 static void
vm_test_per_mapping_internal_accounting(void)979 vm_test_per_mapping_internal_accounting(void)
980 {
981 ledger_t ledger;
982 pmap_t user_pmap;
983 vm_map_t user_map;
984 kern_return_t kr;
985 ledger_amount_t balance;
986 mach_vm_address_t user_addr, user_remap;
987 vm_map_offset_t device_addr;
988 mach_vm_size_t user_size;
989 vm_prot_t cur_prot, max_prot;
990 upl_size_t upl_size;
991 upl_t upl;
992 unsigned int upl_count;
993 upl_control_flags_t upl_flags;
994 upl_page_info_t *pl;
995 ppnum_t ppnum;
996 vm_object_t device_object;
997 vm_map_offset_t map_start, map_end;
998 int pmap_flags;
999
1000 pmap_flags = 0;
1001 if (sizeof(vm_map_offset_t) == 4) {
1002 map_start = 0x100000000ULL;
1003 map_end = 0x200000000ULL;
1004 pmap_flags |= PMAP_CREATE_64BIT;
1005 } else {
1006 map_start = 0x10000000;
1007 map_end = 0x20000000;
1008 }
1009 /* create a user address space */
1010 ledger = ledger_instantiate(task_ledger_template,
1011 LEDGER_CREATE_ACTIVE_ENTRIES);
1012 assert(ledger);
1013 user_pmap = pmap_create_options(ledger, 0, pmap_flags);
1014 assert(user_pmap);
1015 user_map = vm_map_create(user_pmap,
1016 map_start,
1017 map_end,
1018 TRUE);
1019 assert(user_map);
1020 /* check ledger */
1021 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1022 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1023 assertf(balance == 0, "balance=0x%llx", balance);
1024 /* allocate 1 page in that address space */
1025 user_addr = 0;
1026 user_size = PAGE_SIZE;
1027 kr = mach_vm_allocate(user_map,
1028 &user_addr,
1029 user_size,
1030 VM_FLAGS_ANYWHERE);
1031 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1032 /* check ledger */
1033 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1034 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1035 assertf(balance == 0, "balance=0x%llx", balance);
1036 /* remap the original mapping */
1037 user_remap = 0;
1038 kr = mach_vm_remap(user_map,
1039 &user_remap,
1040 PAGE_SIZE,
1041 0,
1042 VM_FLAGS_ANYWHERE,
1043 user_map,
1044 user_addr,
1045 FALSE, /* copy */
1046 &cur_prot,
1047 &max_prot,
1048 VM_INHERIT_DEFAULT);
1049 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1050 /* check ledger */
1051 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1052 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1053 assertf(balance == 0, "balance=0x%llx", balance);
1054 /* create a UPL from the original mapping */
1055 upl_size = PAGE_SIZE;
1056 upl = NULL;
1057 upl_count = 0;
1058 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1059 kr = vm_map_create_upl(user_map,
1060 (vm_map_offset_t)user_addr,
1061 &upl_size,
1062 &upl,
1063 NULL,
1064 &upl_count,
1065 &upl_flags,
1066 VM_KERN_MEMORY_DIAG);
1067 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1068 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
1069 assert(upl_page_present(pl, 0));
1070 ppnum = upl_phys_page(pl, 0);
1071 /* check ledger */
1072 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1073 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1074 assertf(balance == 0, "balance=0x%llx", balance);
1075 device_object = vm_object_allocate(PAGE_SIZE, kernel_map->serial_id);
1076 assert(device_object);
1077 vm_object_lock(device_object);
1078 VM_OBJECT_SET_PRIVATE(device_object, TRUE);
1079 VM_OBJECT_SET_PHYS_CONTIGUOUS(device_object, TRUE);
1080 device_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
1081 vm_object_unlock(device_object);
1082 kr = vm_object_populate_with_private(device_object, 0,
1083 ppnum, PAGE_SIZE);
1084 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1085
1086 /* check ledger */
1087 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1088 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1089 assertf(balance == 0, "balance=0x%llx", balance);
1090 /* deallocate the original mapping */
1091 kr = mach_vm_deallocate(user_map, user_addr, PAGE_SIZE);
1092 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1093 /* map the device_object in the kernel */
1094 device_addr = 0;
1095 vm_object_reference(device_object);
1096 kr = vm_map_enter(kernel_map,
1097 &device_addr,
1098 PAGE_SIZE,
1099 0,
1100 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(),
1101 device_object,
1102 0,
1103 FALSE, /* copy */
1104 VM_PROT_DEFAULT,
1105 VM_PROT_DEFAULT,
1106 VM_INHERIT_NONE);
1107 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1108 /* access the device pager mapping */
1109 *(char *)device_addr = 'x';
1110 printf("%s:%d 0x%llx: 0x%x\n", __FUNCTION__, __LINE__, (uint64_t)device_addr, *(uint32_t *)device_addr);
1111 /* check ledger */
1112 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1113 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1114 assertf(balance == 0, "balance=0x%llx", balance);
1115 /* fault in the remap addr */
1116 kr = vm_fault(user_map, (vm_map_offset_t)user_remap, VM_PROT_READ,
1117 FALSE, 0, TRUE, NULL, 0);
1118 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1119 /* check ledger */
1120 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1121 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1122 assertf(balance == PAGE_SIZE, "balance=0x%llx", balance);
1123 /* deallocate remapping */
1124 kr = mach_vm_deallocate(user_map, user_remap, PAGE_SIZE);
1125 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1126 /* check ledger */
1127 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1128 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1129 assertf(balance == 0, "balance=0x%llx", balance);
1130 /* TODO: cleanup... */
1131 printf("%s:%d PASS\n", __FUNCTION__, __LINE__);
1132 }
1133 #endif /* __arm64__ && !KASAN */
1134
1135 static void
vm_test_kernel_tag_accounting_kma(kma_flags_t base,kma_flags_t bit)1136 vm_test_kernel_tag_accounting_kma(kma_flags_t base, kma_flags_t bit)
1137 {
1138 vm_tag_t tag = VM_KERN_MEMORY_REASON; /* unused during POST */
1139 uint64_t init_size = vm_tag_get_size(tag);
1140 __assert_only uint64_t final_size = init_size + PAGE_SIZE;
1141 vm_address_t address;
1142 kern_return_t kr;
1143
1144 /*
1145 * Test the matrix of:
1146 * - born with or without bit
1147 * - bit flipped or not
1148 * - dies with or without bit
1149 */
1150 for (uint32_t i = 0; i < 4; i++) {
1151 kma_flags_t flags1 = base | ((i & 1) ? bit : KMA_NONE);
1152 kma_flags_t flags2 = base | ((i & 2) ? bit : KMA_NONE);
1153
1154 kr = kmem_alloc(kernel_map, &address, PAGE_SIZE, flags1, tag);
1155 assert3u(kr, ==, KERN_SUCCESS);
1156
1157 if (flags1 & (KMA_VAONLY | KMA_PAGEABLE)) {
1158 assert3u(init_size, ==, vm_tag_get_size(tag));
1159 } else {
1160 assert3u(final_size, ==, vm_tag_get_size(tag));
1161 }
1162
1163 if ((flags1 ^ flags2) == KMA_VAONLY) {
1164 if (flags1 & KMA_VAONLY) {
1165 kernel_memory_populate(address, PAGE_SIZE,
1166 KMA_KOBJECT | KMA_NOFAIL, tag);
1167 } else {
1168 kernel_memory_depopulate(address, PAGE_SIZE,
1169 KMA_KOBJECT, tag);
1170 }
1171 }
1172
1173 if ((flags1 ^ flags2) == KMA_PAGEABLE) {
1174 if (flags1 & KMA_PAGEABLE) {
1175 kr = vm_map_wire_kernel(kernel_map,
1176 address, address + PAGE_SIZE,
1177 VM_PROT_DEFAULT, tag, false);
1178 assert3u(kr, ==, KERN_SUCCESS);
1179 } else {
1180 kr = vm_map_unwire(kernel_map,
1181 address, address + PAGE_SIZE, false);
1182 assert3u(kr, ==, KERN_SUCCESS);
1183 }
1184 }
1185
1186 if (flags2 & (KMA_VAONLY | KMA_PAGEABLE)) {
1187 assert3u(init_size, ==, vm_tag_get_size(tag));
1188 } else {
1189 assert3u(final_size, ==, vm_tag_get_size(tag));
1190 }
1191
1192 kmem_free(kernel_map, address, PAGE_SIZE);
1193 assert3u(init_size, ==, vm_tag_get_size(tag));
1194 }
1195 }
1196
1197 __attribute__((noinline))
1198 static void
vm_test_kernel_tag_accounting(void)1199 vm_test_kernel_tag_accounting(void)
1200 {
1201 printf("%s: test running\n", __func__);
1202
1203 printf("%s: account (KMA_KOBJECT + populate)...\n", __func__);
1204 vm_test_kernel_tag_accounting_kma(KMA_KOBJECT, KMA_VAONLY);
1205 printf("%s: PASS\n", __func__);
1206
1207 printf("%s: account (regular object + wiring)...\n", __func__);
1208 vm_test_kernel_tag_accounting_kma(KMA_NONE, KMA_PAGEABLE);
1209 printf("%s: PASS\n", __func__);
1210
1211 printf("%s: test passed\n", __func__);
1212
1213 #undef if_bit
1214 }
1215
1216 __attribute__((noinline))
1217 static void
vm_test_collapse_overflow(void)1218 vm_test_collapse_overflow(void)
1219 {
1220 vm_object_t object, backing_object;
1221 vm_object_size_t size;
1222 vm_page_t m;
1223
1224 /* create an object for which (int)(size>>PAGE_SHIFT) = 0 */
1225 size = 0x400000000000ULL;
1226 assert((int)(size >> PAGE_SHIFT) == 0);
1227 backing_object = vm_object_allocate(size + PAGE_SIZE, VM_MAP_SERIAL_NONE);
1228 assert(backing_object);
1229 vm_object_reference(backing_object);
1230 /* insert a page */
1231 m = VM_PAGE_NULL;
1232 while (m == VM_PAGE_NULL) {
1233 m = vm_page_grab();
1234 if (m == VM_PAGE_NULL) {
1235 VM_PAGE_WAIT();
1236 }
1237 }
1238 assert(m);
1239 vm_object_lock(backing_object);
1240 vm_page_insert(m, backing_object, 0);
1241 vm_object_unlock(backing_object);
1242 /* make it back another object */
1243 object = vm_object_allocate(size, VM_MAP_SERIAL_NONE);
1244 assert(object);
1245 vm_object_reference(object);
1246 object->shadow = backing_object;
1247 vm_object_reference(backing_object);
1248 /* trigger a bypass */
1249 vm_object_lock(object);
1250 vm_object_collapse(object, 0, TRUE);
1251 /* check that it did not bypass the backing object */
1252 if (object->shadow != backing_object) {
1253 panic("%s:%d FAIL\n", __FUNCTION__, __LINE__);
1254 }
1255 vm_object_unlock(object);
1256
1257 /* remove the page from the backing object */
1258 vm_object_lock(backing_object);
1259 vm_page_remove(m, TRUE);
1260 vm_object_unlock(backing_object);
1261 /* trigger a bypass */
1262 vm_object_lock(object);
1263 vm_object_collapse(object, 0, TRUE);
1264 /* check that it did bypass the backing object */
1265 if (object->shadow == backing_object) {
1266 panic("%s:%d FAIL\n", __FUNCTION__, __LINE__);
1267 }
1268 vm_page_insert(m, object, 0);
1269 vm_object_unlock(object);
1270
1271 /* cleanup */
1272 vm_object_deallocate(object);
1273 /* "backing_object" already lost its reference during the bypass */
1274 // vm_object_deallocate(backing_object);
1275
1276 printf("%s:%d PASS\n", __FUNCTION__, __LINE__);
1277 }
1278
1279 __attribute__((noinline))
1280 static void
vm_test_physical_size_overflow(void)1281 vm_test_physical_size_overflow(void)
1282 {
1283 vm_map_address_t start;
1284 mach_vm_size_t size;
1285 kern_return_t kr;
1286 mach_vm_size_t phys_size;
1287 bool fail;
1288 int failures = 0;
1289
1290 /* size == 0 */
1291 start = 0x100000;
1292 size = 0x0;
1293 kr = vm_map_range_physical_size(kernel_map,
1294 start,
1295 size,
1296 &phys_size);
1297 fail = (kr != KERN_SUCCESS || phys_size != 0);
1298 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1299 __FUNCTION__, __LINE__,
1300 (fail ? "FAIL" : "PASS"),
1301 (uint64_t)start, size, kr, phys_size);
1302 failures += fail;
1303
1304 /* plain wraparound */
1305 start = 0x100000;
1306 size = 0xffffffffffffffff - 0x10000;
1307 kr = vm_map_range_physical_size(kernel_map,
1308 start,
1309 size,
1310 &phys_size);
1311 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1312 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1313 __FUNCTION__, __LINE__,
1314 (fail ? "FAIL" : "PASS"),
1315 (uint64_t)start, size, kr, phys_size);
1316 failures += fail;
1317
1318 /* wraparound after rounding */
1319 start = 0xffffffffffffff00;
1320 size = 0xf0;
1321 kr = vm_map_range_physical_size(kernel_map,
1322 start,
1323 size,
1324 &phys_size);
1325 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1326 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1327 __FUNCTION__, __LINE__,
1328 (fail ? "FAIL" : "PASS"),
1329 (uint64_t)start, size, kr, phys_size);
1330 failures += fail;
1331
1332 /* wraparound to start after rounding */
1333 start = 0x100000;
1334 size = 0xffffffffffffffff;
1335 kr = vm_map_range_physical_size(kernel_map,
1336 start,
1337 size,
1338 &phys_size);
1339 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1340 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1341 __FUNCTION__, __LINE__,
1342 (fail ? "FAIL" : "PASS"),
1343 (uint64_t)start, size, kr, phys_size);
1344 failures += fail;
1345
1346 if (failures) {
1347 panic("%s: FAIL (failures=%d)", __FUNCTION__, failures);
1348 }
1349 printf("%s: PASS\n", __FUNCTION__);
1350 }
1351
1352 #define PTR_UPPER_SHIFT 60
1353 #define PTR_TAG_SHIFT 56
1354 #define PTR_BITS_MASK (((1ULL << PTR_TAG_SHIFT) - 1) | (0xfULL << PTR_UPPER_SHIFT))
1355
1356
1357 __attribute__((noinline))
1358 static void
vm_test_address_canonicalization(void)1359 vm_test_address_canonicalization(void)
1360 {
1361 T_SKIP("System not designed to support this test, skipping...");
1362 }
1363
1364
1365 kern_return_t
vm_tests(void)1366 vm_tests(void)
1367 {
1368 kern_return_t kr = KERN_SUCCESS;
1369
1370 /* Avoid VM panics because some of our test vm_maps don't have a pmap. */
1371 thread_test_context_t ctx CLEANUP_THREAD_TEST_CONTEXT = {
1372 .test_option_vm_map_allow_null_pmap = true,
1373 };
1374 thread_set_test_context(&ctx);
1375
1376 vm_test_collapse_compressor();
1377 vm_test_wire_and_extract();
1378 vm_test_page_wire_overflow_panic();
1379 vm_test_kernel_object_fault();
1380 vm_test_device_pager_transpose();
1381 #if MACH_ASSERT
1382 vm_test_map_copy_adjust_to_target();
1383 #endif /* MACH_ASSERT */
1384 #if PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT
1385 vm_test_4k();
1386 #endif /* PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT */
1387 #if __arm64__ && !KASAN
1388 vm_test_per_mapping_internal_accounting();
1389 #endif /* __arm64__ && !KASAN */
1390 vm_test_kernel_tag_accounting();
1391 vm_test_collapse_overflow();
1392 vm_test_physical_size_overflow();
1393 vm_test_address_canonicalization();
1394
1395 return kr;
1396 }
1397
1398 static inline vm_map_t
create_map(mach_vm_address_t map_start,mach_vm_address_t map_end)1399 create_map(mach_vm_address_t map_start, mach_vm_address_t map_end)
1400 {
1401 ledger_t ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
1402 pmap_t pmap = pmap_create_options(ledger, 0, PMAP_CREATE_64BIT);
1403 assert(pmap);
1404 ledger_dereference(ledger); // now retained by pmap
1405 vm_map_t map = vm_map_create_options(pmap, map_start, map_end, VM_MAP_CREATE_PAGEABLE);//vm_compute_max_offset
1406 assert(map);
1407
1408 #if CONFIG_SPTM
1409 /* Ensure the map serial looks fine */
1410 if (map->serial_id != pmap->associated_vm_map_serial_id) {
1411 panic("Expected a map and its pmap to have exactly the same serial");
1412 }
1413 #endif /* CONFIG_SPTM */
1414
1415 return map;
1416 }
1417
1418 static inline void
cleanup_map(vm_map_t * map)1419 cleanup_map(vm_map_t *map)
1420 {
1421 assert(*map);
1422 kern_return_t kr = vm_map_terminate(*map);
1423 assert(kr == 0);
1424 vm_map_deallocate(*map); // also destroys pmap
1425 }
1426
1427 kern_return_t
1428 mach_vm_remap_new_external(
1429 vm_map_t target_map,
1430 mach_vm_offset_ut *address,
1431 mach_vm_size_ut size,
1432 mach_vm_offset_ut mask,
1433 int flags,
1434 mach_port_t src_tport,
1435 mach_vm_offset_ut memory_address,
1436 boolean_t copy,
1437 vm_prot_ut *cur_protection_u,
1438 vm_prot_ut *max_protection_u,
1439 vm_inherit_ut inheritance);
1440 kern_return_t
1441 vm_remap_new_external(
1442 vm_map_t target_map,
1443 vm_offset_ut *address,
1444 vm_size_ut size,
1445 vm_offset_ut mask,
1446 int flags,
1447 mach_port_t src_tport,
1448 vm_offset_ut memory_address,
1449 boolean_t copy,
1450 vm_prot_ut *cur_protection,
1451 vm_prot_ut *max_protection,
1452 vm_inherit_ut inheritance);
1453 kern_return_t
1454 mach_vm_remap_external(
1455 vm_map_t target_map,
1456 mach_vm_offset_ut *address,
1457 mach_vm_size_ut size,
1458 mach_vm_offset_ut mask,
1459 int flags,
1460 vm_map_t src_map,
1461 mach_vm_offset_ut memory_address,
1462 boolean_t copy,
1463 vm_prot_ut *cur_protection,
1464 vm_prot_ut *max_protection,
1465 vm_inherit_ut inheritance);
1466 kern_return_t
1467 mach_vm_map_external(
1468 vm_map_t target_map,
1469 mach_vm_offset_ut *address,
1470 mach_vm_size_ut initial_size,
1471 mach_vm_offset_ut mask,
1472 int flags,
1473 ipc_port_t port,
1474 memory_object_offset_ut offset,
1475 boolean_t copy,
1476 vm_prot_ut cur_protection,
1477 vm_prot_ut max_protection,
1478 vm_inherit_ut inheritance);
1479 kern_return_t
1480 mach_vm_wire_external(
1481 host_priv_t host_priv,
1482 vm_map_t map,
1483 mach_vm_address_ut start,
1484 mach_vm_size_ut size,
1485 vm_prot_ut access);
1486 kern_return_t
1487 mach_vm_purgable_control_external(
1488 mach_port_t target_tport,
1489 mach_vm_offset_ut address_u,
1490 vm_purgable_t control,
1491 int *state);
1492 kern_return_t
1493 vm_purgable_control_external(
1494 mach_port_t target_tport,
1495 vm_offset_ut address,
1496 vm_purgable_t control,
1497 int *state);
1498
1499 static int
vm_map_null_tests(__unused int64_t in,int64_t * out)1500 vm_map_null_tests(__unused int64_t in, int64_t *out)
1501 {
1502 kern_return_t kr;
1503
1504 mach_vm_address_t alloced_addr, throwaway_addr;
1505 mach_vm_address_ut throwaway_addr_ut;
1506 vm_address_t vm_throwaway_addr;
1507 vm_address_ut vm_throwaway_addr_ut;
1508 vm32_address_ut alloced_addr32, throwaway_addr32_u;
1509 mach_vm_size_t throwaway_size, size_16kb, read_overwrite_data_size;
1510 vm_size_t vm_size, vm_read_overwrite_data_size, vm_throwaway_size;
1511 vm_size_ut throwaway_size_ut;
1512 vm32_size_t data_size32, size32_16kb;
1513 vm32_size_ut data_size32_u, throwaway_size32_u;
1514 mach_msg_type_number_t read_data_size;
1515 mach_port_t mem_entry_result;
1516 pointer_t read_data;
1517 pointer_ut read_data_u;
1518 vm_prot_t prot_default;
1519 vm_prot_ut prot_allexec_u, prot_default_ut;
1520 vm_map_t map64, map32;
1521 vm_machine_attribute_val_t vm_throwaway_attr_val;
1522 vm_region_extended_info_data_t vm_throwaway_region_extended_info;
1523 vm_region_recurse_info_t vm_throwaway_region_recurse_info;
1524 vm_region_recurse_info_64_t vm_throwaway_region_recurse_info_64;
1525 int throwaway_state;
1526 uint32_t throwaway_depth;
1527 vm_page_info_t page_info;
1528
1529 page_info = 0;
1530 throwaway_state = VM_PURGABLE_STATE_MAX;
1531 vm_throwaway_region_recurse_info_64 = 0;
1532 vm_throwaway_region_recurse_info = 0;
1533 vm_throwaway_attr_val = MATTR_VAL_OFF;
1534
1535 map64 = create_map(0, vm_compute_max_offset(true));
1536 map32 = create_map(0, vm_compute_max_offset(false));
1537
1538 prot_allexec_u = vm_sanitize_wrap_prot(VM_PROT_ALLEXEC);
1539 prot_default_ut = vm_sanitize_wrap_prot(VM_PROT_DEFAULT);
1540 prot_default = VM_PROT_DEFAULT;
1541
1542 size_16kb = 16 * 1024;
1543 size32_16kb = (vm32_size_t) size_16kb;
1544
1545 /*
1546 * Allocate some address in the map, just so we can pass a valid looking address to functions so they don't
1547 * return before checking VM_MAP_NULL
1548 */
1549 kr = mach_vm_allocate(map64, &alloced_addr, size_16kb, VM_FLAGS_ANYWHERE);
1550 assert(kr == KERN_SUCCESS);
1551 kr = vm32_vm_allocate(map32, &alloced_addr32, size32_16kb, VM_FLAGS_ANYWHERE);
1552 assert(kr == KERN_SUCCESS);
1553
1554 /*
1555 * Call a bunch of MIG entrypoints with VM_MAP_NULL. The goal is to verify they check map != VM_MAP_NULL.
1556 * There are no requirements put on the return, so don't assert kr. Just verify no crash occurs.
1557 */
1558 throwaway_size = size_16kb;
1559 kr = _mach_make_memory_entry(VM_MAP_NULL, &throwaway_size, alloced_addr, VM_PROT_DEFAULT, &mem_entry_result, IPC_PORT_NULL);
1560 assert(kr != KERN_SUCCESS);
1561 throwaway_size32_u = vm32_sanitize_wrap_size(size32_16kb);
1562 kr = vm32_mach_make_memory_entry(VM_MAP_NULL, &throwaway_size32_u, alloced_addr32, VM_PROT_DEFAULT, &mem_entry_result, IPC_PORT_NULL);
1563 assert(kr != KERN_SUCCESS);
1564 throwaway_size_ut = vm_sanitize_wrap_size(size_16kb);
1565 kr = vm32_mach_make_memory_entry_64(VM_MAP_NULL, &throwaway_size_ut, alloced_addr, VM_PROT_DEFAULT, &mem_entry_result, IPC_PORT_NULL);
1566 assert(kr != KERN_SUCCESS);
1567 throwaway_size = size_16kb;
1568 kr = mach_make_memory_entry_64(VM_MAP_NULL, &throwaway_size, alloced_addr, VM_PROT_DEFAULT, &mem_entry_result, IPC_PORT_NULL);
1569 assert(kr != KERN_SUCCESS);
1570 vm_size = size_16kb;
1571 kr = mach_make_memory_entry(VM_MAP_NULL, &vm_size, alloced_addr, VM_PROT_DEFAULT, &mem_entry_result, IPC_PORT_NULL);
1572 assert(kr != KERN_SUCCESS);
1573
1574 kr = mach_memory_object_memory_entry(HOST_NULL, true, size_16kb, VM_PROT_DEFAULT, MEMORY_OBJECT_NULL, &mem_entry_result);
1575 assert(kr != KERN_SUCCESS);
1576 kr = mach_memory_object_memory_entry_64(HOST_NULL, true, size_16kb, VM_PROT_DEFAULT, MEMORY_OBJECT_NULL, &mem_entry_result);
1577 assert(kr != KERN_SUCCESS);
1578
1579 throwaway_addr = alloced_addr;
1580 kr = mach_vm_allocate(VM_MAP_NULL, &throwaway_addr, size_16kb, VM_FLAGS_ANYWHERE);
1581 assert(kr != KERN_SUCCESS);
1582 throwaway_addr32_u = alloced_addr32;
1583 kr = vm32_vm_allocate(VM_MAP_NULL, &throwaway_addr32_u, size32_16kb, VM_FLAGS_ANYWHERE);
1584 assert(kr != KERN_SUCCESS);
1585 kr = vm_allocate_external(VM_MAP_NULL, &vm_throwaway_addr, size_16kb, VM_FLAGS_ANYWHERE);
1586 assert(kr != KERN_SUCCESS);
1587
1588 kr = mach_vm_deallocate(VM_MAP_NULL, alloced_addr, size_16kb);
1589 assert(kr != KERN_SUCCESS);
1590 kr = vm_deallocate(VM_MAP_NULL, alloced_addr, size_16kb);
1591 assert(kr != KERN_SUCCESS);
1592 kr = vm32_vm_deallocate(VM_MAP_NULL, throwaway_addr32_u, size32_16kb);
1593 assert(kr != KERN_SUCCESS);
1594
1595 kr = mach_vm_map(VM_MAP_NULL, &throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
1596 assert(kr != KERN_SUCCESS);
1597 kr = mach_vm_map_external(VM_MAP_NULL, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
1598 assert(kr != KERN_SUCCESS);
1599
1600 vm_throwaway_addr = alloced_addr;
1601 kr = vm_map(VM_MAP_NULL, &vm_throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
1602 assert(kr != KERN_SUCCESS);
1603 kr = vm32_vm_map(VM_MAP_NULL, &throwaway_addr32_u, size32_16kb, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
1604 assert(kr != KERN_SUCCESS);
1605 kr = vm32_vm_map_64(VM_MAP_NULL, &throwaway_addr32_u, size32_16kb, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
1606 assert(kr != KERN_SUCCESS);
1607
1608 kr = mach_vm_remap(map64, &throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, VM_MAP_NULL, 0, false, &prot_default, &prot_default, VM_INHERIT_DEFAULT);
1609 assert(kr != KERN_SUCCESS);
1610 kr = mach_vm_remap(VM_MAP_NULL, &throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, map64, 0, false, &prot_default, &prot_default, VM_INHERIT_DEFAULT);
1611 assert(kr != KERN_SUCCESS);
1612 kr = mach_vm_remap_external(map64, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, VM_MAP_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1613 assert(kr != KERN_SUCCESS);
1614 kr = mach_vm_remap_external(VM_MAP_NULL, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, map64, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1615 assert(kr != KERN_SUCCESS);
1616 kr = vm_remap_external(map64, &vm_throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, VM_MAP_NULL, 0, false, &prot_default, &prot_default, VM_INHERIT_DEFAULT);
1617 assert(kr != KERN_SUCCESS);
1618 kr = vm_remap_external(VM_MAP_NULL, &vm_throwaway_addr, size_16kb, 0, VM_FLAGS_ANYWHERE, map64, 0, false, &prot_default, &prot_default, VM_INHERIT_DEFAULT);
1619 assert(kr != KERN_SUCCESS);
1620 kr = vm32_vm_remap(map32, &throwaway_addr32_u, size32_16kb, 0, VM_FLAGS_ANYWHERE, VM_MAP_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1621 assert(kr != KERN_SUCCESS);
1622 kr = vm32_vm_remap(VM_MAP_NULL, &throwaway_addr32_u, size32_16kb, 0, VM_FLAGS_ANYWHERE, map32, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1623 assert(kr != KERN_SUCCESS);
1624
1625 kr = mach_vm_remap_new_external(VM_MAP_NULL, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1626 assert(kr != KERN_SUCCESS);
1627 kr = mach_vm_remap_new_external(map64, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1628 assert(kr != KERN_SUCCESS);
1629
1630 kr = mach_vm_remap_new_external(VM_MAP_NULL, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_allexec_u, &prot_allexec_u, VM_INHERIT_DEFAULT);
1631 assert(kr != KERN_SUCCESS);
1632 kr = mach_vm_remap_new_external(map64, &throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_allexec_u, &prot_allexec_u, VM_INHERIT_DEFAULT);
1633 assert(kr != KERN_SUCCESS);
1634
1635 kr = vm_remap_new_external(VM_MAP_NULL, &vm_throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1636 assert(kr != KERN_SUCCESS);
1637 kr = vm_remap_new_external(map64, &vm_throwaway_addr_ut, size_16kb, 0, VM_FLAGS_ANYWHERE, MACH_PORT_NULL, 0, false, &prot_default_ut, &prot_default_ut, VM_INHERIT_DEFAULT);
1638 assert(kr != KERN_SUCCESS);
1639
1640 kr = mach_vm_wire_external(host_priv_self(), VM_MAP_NULL, throwaway_addr_ut, size_16kb, VM_PROT_DEFAULT);
1641 assert(kr != KERN_SUCCESS);
1642 kr = mach_vm_wire_external(HOST_PRIV_NULL, map64, throwaway_addr_ut, size_16kb, VM_PROT_DEFAULT);
1643 assert(kr != KERN_SUCCESS);
1644
1645 kr = vm_wire(host_priv_self(), VM_MAP_NULL, throwaway_addr, size_16kb, VM_PROT_DEFAULT);
1646 assert(kr != KERN_SUCCESS);
1647 kr = vm_wire(HOST_PRIV_NULL, map64, throwaway_addr, size_16kb, VM_PROT_DEFAULT);
1648 assert(kr != KERN_SUCCESS);
1649
1650 kr = task_wire(VM_MAP_NULL, false);
1651 assert(kr != KERN_SUCCESS);
1652 kr = vm32_task_wire(VM_MAP_NULL, false);
1653 assert(kr != KERN_SUCCESS);
1654
1655 kr = mach_vm_read(VM_MAP_NULL, alloced_addr, size_16kb, &read_data, &read_data_size);
1656 assert(kr != KERN_SUCCESS);
1657 kr = vm_read(VM_MAP_NULL, alloced_addr, size_16kb, &read_data, &read_data_size);
1658 assert(kr != KERN_SUCCESS);
1659 kr = vm32_vm_read(VM_MAP_NULL, alloced_addr32, size32_16kb, &read_data_u, &data_size32);
1660 assert(kr != KERN_SUCCESS);
1661
1662 mach_vm_read_entry_t * mach_re = kalloc_type(mach_vm_read_entry_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1663 (*mach_re)[0].address = alloced_addr;
1664 (*mach_re)[0].size = size_16kb;
1665
1666 vm_read_entry_t * re = kalloc_type(vm_read_entry_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1667 (*re)[0].address = alloced_addr;
1668 (*re)[0].size = (vm_size_t) size_16kb;
1669
1670 vm32_read_entry_t * re_32 = kalloc_type(vm32_read_entry_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1671 (*re_32)[0].address = (vm32_address_t) alloced_addr;
1672 (*re_32)[0].size = (vm32_size_t) size_16kb;
1673
1674 kr = mach_vm_read_list(VM_MAP_NULL, *mach_re, 1);
1675 assert(kr != KERN_SUCCESS);
1676 kr = vm_read_list(VM_MAP_NULL, *re, 1);
1677 assert(kr != KERN_SUCCESS);
1678 kr = vm32_vm_read_list(VM_MAP_NULL, *re_32, 1);
1679 assert(kr != KERN_SUCCESS);
1680
1681 kfree_type(mach_vm_read_entry_t, mach_re);
1682 kfree_type(vm_read_entry_t, re);
1683 kfree_type(vm32_read_entry_t, re_32);
1684
1685 kr = mach_vm_read_overwrite(VM_MAP_NULL, alloced_addr, size_16kb, alloced_addr, &read_overwrite_data_size);
1686 assert(kr != KERN_SUCCESS);
1687 kr = vm_read_overwrite(VM_MAP_NULL, alloced_addr, size_16kb, alloced_addr, &vm_read_overwrite_data_size);
1688 assert(kr != KERN_SUCCESS);
1689 kr = vm32_vm_read_overwrite(VM_MAP_NULL, alloced_addr32, size32_16kb, alloced_addr32, &data_size32_u);
1690 assert(kr != KERN_SUCCESS);
1691
1692 kr = mach_vm_copy(VM_MAP_NULL, alloced_addr, size_16kb, alloced_addr);
1693 assert(kr != KERN_SUCCESS);
1694 kr = vm_copy(VM_MAP_NULL, alloced_addr, size_16kb, alloced_addr);
1695 assert(kr != KERN_SUCCESS);
1696 kr = vm32_vm_copy(VM_MAP_NULL, alloced_addr32, size32_16kb, alloced_addr32);
1697 assert(kr != KERN_SUCCESS);
1698
1699 kr = mach_vm_write(VM_MAP_NULL, alloced_addr, alloced_addr, (mach_msg_type_number_t) size_16kb);
1700 assert(kr != KERN_SUCCESS);
1701 kr = vm_write(VM_MAP_NULL, alloced_addr, alloced_addr, (mach_msg_type_number_t) size_16kb);
1702 assert(kr != KERN_SUCCESS);
1703 kr = vm32_vm_write(VM_MAP_NULL, alloced_addr32, alloced_addr, (mach_msg_type_number_t) size_16kb);
1704 assert(kr != KERN_SUCCESS);
1705
1706 kr = mach_vm_inherit(VM_MAP_NULL, alloced_addr, size_16kb, VM_INHERIT_DEFAULT);
1707 assert(kr != KERN_SUCCESS);
1708 kr = vm_inherit(VM_MAP_NULL, alloced_addr, size_16kb, VM_INHERIT_DEFAULT);
1709 assert(kr != KERN_SUCCESS);
1710 kr = vm32_vm_inherit(VM_MAP_NULL, alloced_addr32, size32_16kb, VM_INHERIT_DEFAULT);
1711
1712 kr = mach_vm_protect(VM_MAP_NULL, alloced_addr, size_16kb, FALSE, VM_PROT_DEFAULT);
1713 assert(kr != KERN_SUCCESS);
1714 kr = vm_protect(VM_MAP_NULL, alloced_addr, size_16kb, FALSE, VM_PROT_DEFAULT);
1715 assert(kr != KERN_SUCCESS);
1716 kr = vm32_vm_protect(VM_MAP_NULL, alloced_addr32, size32_16kb, FALSE, VM_PROT_DEFAULT);
1717 assert(kr != KERN_SUCCESS);
1718
1719 kr = mach_vm_behavior_set(VM_MAP_NULL, alloced_addr, size_16kb, VM_BEHAVIOR_DEFAULT);
1720 assert(kr != KERN_SUCCESS);
1721 kr = vm_behavior_set(VM_MAP_NULL, alloced_addr, size_16kb, VM_BEHAVIOR_DEFAULT);
1722 assert(kr != KERN_SUCCESS);
1723 kr = vm32_vm_behavior_set(VM_MAP_NULL, alloced_addr32, size32_16kb, VM_BEHAVIOR_DEFAULT);
1724 assert(kr != KERN_SUCCESS);
1725
1726 kr = mach_vm_msync(VM_MAP_NULL, alloced_addr, size_16kb, VM_SYNC_ASYNCHRONOUS);
1727 assert(kr != KERN_SUCCESS);
1728 kr = vm_msync(VM_MAP_NULL, alloced_addr, size_16kb, VM_SYNC_ASYNCHRONOUS);
1729 assert(kr != KERN_SUCCESS);
1730 kr = vm32_vm_msync(VM_MAP_NULL, alloced_addr32, size32_16kb, VM_SYNC_ASYNCHRONOUS);
1731 assert(kr != KERN_SUCCESS);
1732
1733 kr = mach_vm_machine_attribute(VM_MAP_NULL, alloced_addr, size_16kb, MATTR_CACHE, &vm_throwaway_attr_val);
1734 assert(kr != KERN_SUCCESS);
1735 kr = vm_machine_attribute(VM_MAP_NULL, alloced_addr, size_16kb, MATTR_CACHE, &vm_throwaway_attr_val);
1736 assert(kr != KERN_SUCCESS);
1737 kr = vm32_vm_machine_attribute(VM_MAP_NULL, alloced_addr32, size32_16kb, MATTR_CACHE, &vm_throwaway_attr_val);
1738 assert(kr != KERN_SUCCESS);
1739
1740 kr = mach_vm_purgable_control_external(MACH_PORT_NULL, throwaway_addr_ut, VM_PURGABLE_PURGE_ALL, &throwaway_state);
1741 assert(kr != KERN_SUCCESS);
1742 kr = vm_purgable_control_external(MACH_PORT_NULL, throwaway_addr_ut, VM_PURGABLE_PURGE_ALL, &throwaway_state);
1743 assert(kr != KERN_SUCCESS);
1744 kr = vm32_vm_purgable_control(VM_MAP_NULL, alloced_addr32, VM_PURGABLE_PURGE_ALL, &throwaway_state);
1745 assert(kr != KERN_SUCCESS);
1746
1747 kr = mach_vm_region(VM_MAP_NULL, &throwaway_addr, &throwaway_size, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&vm_throwaway_region_extended_info, &read_data_size, &mem_entry_result);
1748 assert(kr != KERN_SUCCESS);
1749 kr = vm_region(VM_MAP_NULL, &vm_throwaway_addr, &vm_throwaway_size, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&vm_throwaway_region_extended_info, &read_data_size, &mem_entry_result);
1750 assert(kr != KERN_SUCCESS);
1751 kr = vm_region_64(VM_MAP_NULL, &vm_throwaway_addr, &vm_throwaway_size, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&vm_throwaway_region_extended_info, &read_data_size, &mem_entry_result);
1752 assert(kr != KERN_SUCCESS);
1753 kr = vm32_vm_region(VM_MAP_NULL, &throwaway_addr32_u, &throwaway_size32_u, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&vm_throwaway_region_extended_info, &read_data_size, &mem_entry_result);
1754 assert(kr != KERN_SUCCESS);
1755 kr = vm32_vm_region_64(VM_MAP_NULL, &throwaway_addr32_u, &throwaway_size32_u, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&vm_throwaway_region_extended_info, &read_data_size, &mem_entry_result);
1756 assert(kr != KERN_SUCCESS);
1757
1758 kr = mach_vm_region_recurse(VM_MAP_NULL, &throwaway_addr, &throwaway_size, &throwaway_depth, vm_throwaway_region_recurse_info, &read_data_size);
1759 assert(kr != KERN_SUCCESS);
1760 kr = vm_region_recurse(VM_MAP_NULL, &vm_throwaway_addr, &vm_throwaway_size, &throwaway_depth, vm_throwaway_region_recurse_info, &read_data_size);
1761 assert(kr != KERN_SUCCESS);
1762 kr = vm_region_recurse_64(VM_MAP_NULL, &vm_throwaway_addr, &vm_throwaway_size, &throwaway_depth, vm_throwaway_region_recurse_info_64, &read_data_size);
1763 assert(kr != KERN_SUCCESS);
1764 kr = vm32_vm_region_recurse(VM_MAP_NULL, &throwaway_addr32_u, &throwaway_size32_u, &throwaway_depth, vm_throwaway_region_recurse_info, &read_data_size);
1765 assert(kr != KERN_SUCCESS);
1766 kr = vm32_vm_region_recurse_64(VM_MAP_NULL, &throwaway_addr32_u, &throwaway_size32_u, &throwaway_depth, vm_throwaway_region_recurse_info_64, &read_data_size);
1767 assert(kr != KERN_SUCCESS);
1768
1769 kr = mach_vm_page_info(VM_MAP_NULL, alloced_addr, VM_PAGE_INFO_BASIC, page_info, &read_data_size);
1770 assert(kr != KERN_SUCCESS);
1771 kr = mach_vm_page_query(VM_MAP_NULL, alloced_addr, &throwaway_state, &throwaway_state);
1772 assert(kr != KERN_SUCCESS);
1773 kr = vm_map_page_query(VM_MAP_NULL, vm_throwaway_addr, &throwaway_state, &throwaway_state);
1774 assert(kr != KERN_SUCCESS);
1775 kr = vm32_vm_map_page_query(VM_MAP_NULL, throwaway_addr32_u, &throwaway_state, &throwaway_state);
1776 assert(kr != KERN_SUCCESS);
1777
1778 /*
1779 * Cleanup our allocations and maps
1780 */
1781 kr = mach_vm_deallocate(map64, alloced_addr, size_16kb);
1782 assert(kr == KERN_SUCCESS);
1783 kr = vm32_vm_deallocate(map32, alloced_addr32, size32_16kb);
1784 assert(kr == KERN_SUCCESS);
1785
1786 cleanup_map(&map64);
1787 cleanup_map(&map32);
1788
1789 /*
1790 * If we made it far without crashing, the test works.
1791 */
1792
1793 *out = 1;
1794 return 0;
1795 }
1796 SYSCTL_TEST_REGISTER(vm_map_null, vm_map_null_tests);
1797
1798
1799 static int
vm_map_copyio_test(__unused int64_t in,int64_t * out)1800 vm_map_copyio_test(__unused int64_t in, int64_t *out)
1801 {
1802 /* Test is not supported */
1803 *out = ENOTSUP;
1804 return 0;
1805 }
1806 SYSCTL_TEST_REGISTER(vm_map_copyio, vm_map_copyio_test);
1807
1808 static int
vm_page_relocate_test(__unused int64_t in,int64_t * out)1809 vm_page_relocate_test(__unused int64_t in, int64_t *out)
1810 {
1811 /* Test is not supported */
1812 *out = ENOTSUP;
1813 return 0;
1814 }
1815 SYSCTL_TEST_REGISTER(vm_page_relocate, vm_page_relocate_test);
1816
1817 #define PAGE_SHIFT_4K 12
1818 #define PAGE_SHIFT_16K 14
1819 static int
vm_map_copy_entry_subrange_test(__unused int64_t in,int64_t * out)1820 vm_map_copy_entry_subrange_test(__unused int64_t in, int64_t *out)
1821 {
1822 mach_vm_size_t size_4kb, size_16kb;
1823 vm_map_t map_4k, map_16k;
1824 mach_vm_address_t alloced_addr, mapped_addr;
1825 mach_vm_size_t entry_size;
1826 mach_port_t entry_handle;
1827 mach_vm_size_t mapped_size;
1828 vm_region_basic_info_data_64_t region_info;
1829 mach_msg_type_number_t region_info_count;
1830
1831 kern_return_t kr;
1832
1833 size_4kb = 4 * 1024;
1834 size_16kb = 16 * 1024;
1835
1836 map_4k = create_map(0, vm_compute_max_offset(true));
1837 kr = vm_map_set_page_shift(map_4k, PAGE_SHIFT_4K);
1838 map_16k = create_map(0, vm_compute_max_offset(true));
1839 kr = vm_map_set_page_shift(map_16k, PAGE_SHIFT_16K);
1840
1841 /*
1842 * Test mapping a portion of a copy entry from a 4k map to a 16k one.
1843 * The result size should be aligned to the destination's page size (16k).
1844 */
1845 // Get a copy entry to map into the system
1846 kr = mach_vm_allocate(map_4k, &alloced_addr, size_16kb, VM_FLAGS_ANYWHERE);
1847 assert(kr == KERN_SUCCESS);
1848
1849 entry_size = size_16kb;
1850 kr = mach_make_memory_entry_64(map_4k, &entry_size, alloced_addr,
1851 MAP_MEM_VM_COPY | MAP_MEM_USE_DATA_ADDR | VM_PROT_DEFAULT,
1852 &entry_handle, MACH_PORT_NULL);
1853 assert(kr == KERN_SUCCESS);
1854 assert(entry_size == size_16kb);
1855
1856 // Attempt to map a portion of the entry into the 16k map
1857 kr = mach_vm_map(map_16k, &mapped_addr, size_4kb, 0, VM_FLAGS_ANYWHERE,
1858 entry_handle, 0, true, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
1859 VM_INHERIT_DEFAULT);
1860 assert(kr == KERN_SUCCESS);
1861
1862 // Ensure the entry is actually mapped whole
1863 region_info_count = VM_REGION_BASIC_INFO_COUNT_64;
1864 kr = mach_vm_region(map_16k, &mapped_addr, &mapped_size, VM_REGION_BASIC_INFO_64,
1865 (vm_region_info_t) ®ion_info, ®ion_info_count, NULL);
1866 assert(kr == KERN_SUCCESS);
1867 assert(mapped_size == entry_size);
1868
1869 // Cleanup
1870 mach_memory_entry_port_release(entry_handle);
1871 kr = mach_vm_deallocate(map_16k, mapped_addr, size_16kb);
1872 assert(kr == KERN_SUCCESS);
1873 kr = mach_vm_deallocate(map_4k, alloced_addr, size_16kb);
1874 assert(kr == KERN_SUCCESS);
1875 cleanup_map(&map_4k);
1876 cleanup_map(&map_16k);
1877
1878 *out = 1;
1879 return 0;
1880 }
1881 SYSCTL_TEST_REGISTER(vm_map_copy_entry_subrange, vm_map_copy_entry_subrange_test);
1882
1883
1884 static int
vm_memory_entry_map_size_null_test(__unused int64_t in,int64_t * out)1885 vm_memory_entry_map_size_null_test(__unused int64_t in, int64_t *out)
1886 {
1887 mach_vm_size_t size_16kb, map_size;
1888 vm_map_t map;
1889
1890 kern_return_t kr;
1891
1892 map = create_map(0, vm_compute_max_offset(true));
1893 size_16kb = 16 * 1024;
1894
1895 map_size = 0xdeadbeef;
1896 kr = mach_memory_entry_map_size(MACH_PORT_NULL, map, 0, size_16kb, &map_size);
1897 assert(kr == KERN_INVALID_ARGUMENT);
1898 assert(map_size == 0);
1899
1900 cleanup_map(&map);
1901
1902 *out = 1;
1903 return 0;
1904 }
1905 SYSCTL_TEST_REGISTER(vm_memory_entry_map_size_null, vm_memory_entry_map_size_null_test);
1906
1907 static int
vm_memory_entry_map_size_overflow_tests(__unused int64_t in,int64_t * out)1908 vm_memory_entry_map_size_overflow_tests(__unused int64_t in, int64_t *out)
1909 {
1910 mach_vm_size_t size_16kb, entry_size, map_size;
1911 vm_map_t map;
1912 mach_port_t parent_handle, entry_handle;
1913 mach_vm_address_t alloced_addr;
1914 vm_map_offset_t entry_offset;
1915 memory_object_offset_t maximum_offset;
1916
1917 kern_return_t kr;
1918
1919 size_16kb = 16 * 1024;
1920 map = create_map(0, vm_compute_max_offset(true));
1921 /*
1922 * (1) Attempt to overflow offset + mem_entry->offset
1923 */
1924 // Setup - create an entry with nonzero offset
1925 kr = mach_memory_object_memory_entry_64((host_t) 1, 1,
1926 size_16kb * 2, VM_PROT_DEFAULT, 0, &parent_handle);
1927 assert(kr == KERN_SUCCESS);
1928
1929 entry_size = size_16kb;
1930 kr = mach_make_memory_entry_64(map, &entry_size, size_16kb,
1931 VM_PROT_DEFAULT, &entry_handle, parent_handle);
1932 assert(kr == KERN_SUCCESS);
1933
1934 // Pass in maximum offset to attempt overflow
1935 maximum_offset = (memory_object_offset_t) -1;
1936 kr = mach_memory_entry_map_size(entry_handle, map, maximum_offset, size_16kb,
1937 &map_size);
1938 assert(kr == KERN_INVALID_ARGUMENT);
1939
1940 // Cleanup
1941 mach_memory_entry_port_release(parent_handle);
1942 mach_memory_entry_port_release(entry_handle);
1943
1944 /*
1945 * (2) Attempt to overflow offset + mem_entry->data_offset
1946 */
1947 // Setup - create an entry with nonzero data_offset
1948 kr = mach_vm_allocate(map, &alloced_addr, 2 * size_16kb, VM_FLAGS_ANYWHERE);
1949 assert(kr == KERN_SUCCESS);
1950
1951 entry_size = size_16kb;
1952 entry_offset = alloced_addr + (size_16kb / 2);
1953 kr = mach_make_memory_entry_64(map, &entry_size, entry_offset,
1954 MAP_MEM_VM_COPY | MAP_MEM_USE_DATA_ADDR | VM_PROT_DEFAULT,
1955 &entry_handle, MACH_PORT_NULL);
1956 assert(kr == KERN_SUCCESS);
1957
1958 // Pass in maximum offset to attempt overflow
1959 kr = mach_memory_entry_map_size(entry_handle, map, maximum_offset, size_16kb,
1960 &map_size);
1961 assert(kr == KERN_INVALID_ARGUMENT);
1962
1963 // Cleanup
1964 mach_memory_entry_port_release(entry_handle);
1965 kr = mach_vm_deallocate(map, alloced_addr, 2 * size_16kb);
1966 assert(kr == KERN_SUCCESS);
1967 cleanup_map(&map);
1968
1969 *out = 1;
1970 return 0;
1971 }
1972 SYSCTL_TEST_REGISTER(vm_memory_entry_map_size_overflow, vm_memory_entry_map_size_overflow_tests);
1973
1974 static int
vm_memory_entry_map_size_copy_tests(__unused int64_t in,int64_t * out)1975 vm_memory_entry_map_size_copy_tests(__unused int64_t in, int64_t *out)
1976 {
1977 mach_vm_size_t size_2kb, size_4kb, size_16kb;
1978 mach_vm_size_t entry_size_4k, entry_size_16k;
1979 mach_vm_size_t map_size;
1980 vm_map_t map_4k, map_16k;
1981 mach_port_t entry_4k, entry_16k;
1982 mach_vm_address_t alloced_addr_4k, alloced_addr_16k;
1983
1984 kern_return_t kr;
1985
1986 size_2kb = 2 * 1024;
1987 size_4kb = 4 * 1024;
1988 size_16kb = 16 * 1024;
1989
1990 /*
1991 * Setup - initialize maps and create copy entries for each
1992 */
1993 // 4k map and entry
1994 map_4k = create_map(0, vm_compute_max_offset(true));
1995 kr = vm_map_set_page_shift(map_4k, PAGE_SHIFT_4K);
1996 assert(kr == KERN_SUCCESS);
1997
1998 kr = mach_vm_allocate(map_4k, &alloced_addr_4k, size_16kb, VM_FLAGS_ANYWHERE);
1999 assert(kr == KERN_SUCCESS);
2000
2001 entry_size_4k = size_16kb;
2002 kr = mach_make_memory_entry_64(map_4k, &entry_size_4k, alloced_addr_4k,
2003 MAP_MEM_VM_COPY | VM_PROT_DEFAULT, &entry_4k, MACH_PORT_NULL);
2004 assert(kr == KERN_SUCCESS);
2005 assert(entry_size_4k == size_16kb);
2006
2007 // 16k map and entry
2008 map_16k = create_map(0, vm_compute_max_offset(true));
2009 kr = vm_map_set_page_shift(map_16k, PAGE_SHIFT_16K);
2010 assert(kr == KERN_SUCCESS);
2011
2012 kr = mach_vm_allocate(map_16k, &alloced_addr_16k, size_16kb, VM_FLAGS_ANYWHERE);
2013 assert(kr == KERN_SUCCESS);
2014
2015 entry_size_16k = size_16kb;
2016 kr = mach_make_memory_entry_64(map_16k, &entry_size_16k, alloced_addr_16k,
2017 MAP_MEM_VM_COPY | VM_PROT_DEFAULT, &entry_16k, MACH_PORT_NULL);
2018 assert(kr == KERN_SUCCESS);
2019 assert(entry_size_16k == size_16kb);
2020
2021 /*
2022 * (1) Test 4k map with 4k entry and 16k map with 16k entry. Page-aligned
2023 * ranges should have no size adjustment.
2024 */
2025 for (mach_vm_size_t i = 1; i <= 4; i++) {
2026 kr = mach_memory_entry_map_size(entry_4k, map_4k, 0, i * size_4kb, &map_size);
2027 assert(kr == KERN_SUCCESS);
2028 assert(map_size == (i * size_4kb));
2029 }
2030 kr = mach_memory_entry_map_size(entry_16k, map_16k, 0, size_16kb, &map_size);
2031 assert(kr == KERN_SUCCESS);
2032 assert(map_size == size_16kb);
2033
2034 /*
2035 * (2) Test 4k map with 16k entry. Since we have a 4k map, we should be able
2036 * to map a 4k range of the entry, but to map a 2k range we will need to map
2037 * a full 4k page.
2038 */
2039 kr = mach_memory_entry_map_size(entry_16k, map_4k, 0, size_16kb, &map_size);
2040 assert(kr == KERN_SUCCESS);
2041 assert(map_size == size_16kb);
2042 kr = mach_memory_entry_map_size(entry_16k, map_4k, 0, size_4kb, &map_size);
2043 assert(kr == KERN_SUCCESS);
2044 assert(map_size == size_4kb);
2045 kr = mach_memory_entry_map_size(entry_16k, map_4k, 0, size_2kb, &map_size);
2046 assert(kr == KERN_SUCCESS);
2047 assert(map_size == size_4kb);
2048
2049 /*
2050 * (3) Test 16k map with 4k entry. Since we have a 16k map, we will need to
2051 * map the whole 16kb memory entry even if a smaller range is requested.
2052 */
2053 kr = mach_memory_entry_map_size(entry_4k, map_16k, 0, size_16kb, &map_size);
2054 assert(kr == KERN_SUCCESS);
2055 assert(map_size == size_16kb);
2056 kr = mach_memory_entry_map_size(entry_4k, map_16k, 0, size_4kb, &map_size);
2057 assert(kr == KERN_SUCCESS);
2058 assert(map_size == size_16kb);
2059 kr = mach_memory_entry_map_size(entry_4k, map_16k, 0, size_2kb, &map_size);
2060 assert(kr == KERN_SUCCESS);
2061 assert(map_size == size_16kb);
2062
2063 /*
2064 * (4) Detect error in the case where the size requested is too large.
2065 */
2066 map_size = 0xdeadbeef;
2067 kr = mach_memory_entry_map_size(entry_4k, map_16k, 0, 2 * size_16kb, &map_size);
2068 assert(kr == KERN_INVALID_ARGUMENT);
2069 assert(map_size == 0);
2070
2071 /*
2072 * Clean up memory entries, allocations, and maps
2073 */
2074 mach_memory_entry_port_release(entry_4k);
2075 mach_memory_entry_port_release(entry_16k);
2076 kr = mach_vm_deallocate(map_4k, alloced_addr_4k, size_16kb);
2077 assert(kr == KERN_SUCCESS);
2078 kr = mach_vm_deallocate(map_16k, alloced_addr_16k, size_16kb);
2079 assert(kr == KERN_SUCCESS);
2080 cleanup_map(&map_4k);
2081 cleanup_map(&map_16k);
2082
2083 *out = 1;
2084 return 0;
2085 }
2086 SYSCTL_TEST_REGISTER(vm_memory_entry_map_size_copy, vm_memory_entry_map_size_copy_tests);
2087
2088 static int
vm_memory_entry_parent_submap_tests(__unused int64_t in,int64_t * out)2089 vm_memory_entry_parent_submap_tests(__unused int64_t in, int64_t *out)
2090 {
2091 vm_shared_region_t shared_region;
2092 mach_port_t parent_handle, entry_handle;
2093 vm_named_entry_t parent_entry;
2094 mach_vm_size_t entry_size;
2095 vm_prot_t vmflags;
2096
2097 kern_return_t kr;
2098
2099 /*
2100 * Use shared region to get a named_entry which refers to a submap
2101 */
2102 shared_region = vm_shared_region_get(current_task());
2103 parent_handle = shared_region->sr_mem_entry;
2104 assert(parent_handle != NULL);
2105 parent_entry = mach_memory_entry_from_port(parent_handle);
2106 assert(parent_entry->is_sub_map);
2107
2108 /*
2109 * We should be able to create an entry using the submap entry as the parent
2110 */
2111 entry_size = parent_entry->size;
2112 vmflags = VM_PROT_DEFAULT;
2113 kr = mach_make_memory_entry_64(VM_MAP_NULL, &entry_size, 0, vmflags,
2114 &entry_handle, parent_handle);
2115 assert(kr == KERN_SUCCESS);
2116 mach_memory_entry_port_release(entry_handle);
2117
2118 /*
2119 * Should fail if using mach_make_memory_entry_mem_only since the parent
2120 * entry is not an object
2121 */
2122 vmflags |= MAP_MEM_ONLY;
2123 kr = mach_make_memory_entry_64(VM_MAP_NULL, &entry_size, 0, vmflags,
2124 &entry_handle, parent_handle);
2125 assert(kr == KERN_INVALID_ARGUMENT);
2126
2127 /*
2128 * Cleanup
2129 */
2130 vm_shared_region_deallocate(shared_region);
2131
2132 *out = 1;
2133 return 0;
2134 }
2135 SYSCTL_TEST_REGISTER(vm_memory_entry_parent_submap, vm_memory_entry_parent_submap_tests);
2136
2137 static int
vm_cpu_map_pageout_test(int64_t in,int64_t * out)2138 vm_cpu_map_pageout_test(int64_t in, int64_t *out)
2139 {
2140 /* Test is not supported */
2141 (void)in;
2142 *out = ENOTSUP;
2143 return 0;
2144 }
2145 SYSCTL_TEST_REGISTER(vm_cpu_map_pageout, vm_cpu_map_pageout_test);
2146
2147 static int
vm_get_wimg_mode(int64_t in,int64_t * out)2148 vm_get_wimg_mode(int64_t in, int64_t *out)
2149 {
2150 mach_vm_offset_t addr = (mach_vm_offset_t)in;
2151 vm_map_entry_t entry;
2152 vm_map_t map = current_map();
2153 vm_map_lock_read(map);
2154 bool map_contains_addr = vm_map_lookup_entry(map, addr, &entry);
2155 if (!map_contains_addr) {
2156 vm_map_unlock_read(map);
2157 return EINVAL;
2158 }
2159
2160 if (entry->is_sub_map) {
2161 vm_map_unlock_read(map);
2162 return ENOTSUP;
2163 }
2164
2165 *out = 0;
2166 vm_object_t obj = VME_OBJECT(entry);
2167 if (obj != VM_OBJECT_NULL) {
2168 *out = obj->wimg_bits;
2169 }
2170
2171 vm_map_unlock_read(map);
2172 return 0;
2173 }
2174 SYSCTL_TEST_REGISTER(vm_get_wimg_mode, vm_get_wimg_mode);
2175
2176 /*
2177 * Make sure copies from 4k->16k maps doesn't lead to address space holes
2178 */
2179 static int
vm_map_4k_16k_test(int64_t in,int64_t * out)2180 vm_map_4k_16k_test(int64_t in, int64_t *out)
2181 {
2182 #if PMAP_CREATE_FORCE_4K_PAGES
2183 const mach_vm_size_t alloc_size = (36 * 1024);
2184 assert((alloc_size % FOURK_PAGE_SHIFT) == 0);
2185 assert((alloc_size % SIXTEENK_PAGE_SHIFT) != 0);
2186 assert(alloc_size > msg_ool_size_small); // avoid kernel buffer copy optimization
2187
2188 /* initialize maps */
2189 pmap_t pmap_4k, pmap_16k;
2190 vm_map_t map_4k, map_16k;
2191 pmap_4k = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT | PMAP_CREATE_FORCE_4K_PAGES);
2192 assert(pmap_4k);
2193 map_4k = vm_map_create_options(pmap_4k, MACH_VM_MIN_ADDRESS, MACH_VM_MAX_ADDRESS, VM_MAP_CREATE_PAGEABLE);
2194 assert(map_4k != VM_MAP_NULL);
2195 vm_map_set_page_shift(map_4k, FOURK_PAGE_SHIFT);
2196
2197 pmap_16k = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT);
2198 assert(pmap_16k);
2199 map_16k = vm_map_create_options(pmap_16k, MACH_VM_MIN_ADDRESS, MACH_VM_MAX_ADDRESS, VM_MAP_CREATE_PAGEABLE);
2200 assert(map_16k != VM_MAP_NULL);
2201 assert(VM_MAP_PAGE_SHIFT(map_16k) == SIXTEENK_PAGE_SHIFT);
2202
2203 /* create mappings in 4k map */
2204 /* allocate space */
2205 vm_address_t address_4k;
2206 kern_return_t kr = vm_allocate_external(map_4k, &address_4k, alloc_size, VM_FLAGS_ANYWHERE);
2207 assert3u(kr, ==, KERN_SUCCESS); /* reserve space for 4k entries in 4k map */
2208
2209 /* overwrite with a bunch of 4k entries */
2210 for (mach_vm_address_t addr = address_4k; addr < (address_4k + alloc_size); addr += FOURK_PAGE_SIZE) {
2211 /* allocate 128MB objects, so that they don't get coalesced, preventing entry simplification */
2212 vm_object_t object = vm_object_allocate(ANON_CHUNK_SIZE, map_4k->serial_id);
2213 kr = vm_map_enter(map_4k, &addr, FOURK_PAGE_SIZE, /* mask */ 0,
2214 VM_MAP_KERNEL_FLAGS_FIXED(.vmf_overwrite = TRUE), object, /* offset */ 0,
2215 /* copy */ false, VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
2216 assert3u(kr, ==, KERN_SUCCESS); /* overwrite the 4k chunk at addr with its own entry */
2217 }
2218
2219 /* set up vm_map_copy_t */
2220 vm_map_copy_t copy;
2221 kr = vm_map_copyin(map_4k, address_4k, alloc_size, true, ©);
2222 assert3u(kr, ==, KERN_SUCCESS); /* copyin from 4k map succeeds */
2223
2224 /* write out the vm_map_copy_t to the 16k map */
2225 vm_address_t address_16k;
2226 if (in == 0) {
2227 /* vm_map_copyout */
2228 vm_map_address_t tmp_address;
2229 kr = vm_map_copyout(map_16k, &tmp_address, copy);
2230 assert3u(kr, ==, KERN_SUCCESS); /* copyout into 16k map suceeds */
2231 address_16k = (vm_address_t)tmp_address;
2232 } else if (in == 1) {
2233 /* vm_map_copy_overwrite */
2234 /* reserve space */
2235 kr = vm_allocate_external(map_16k, &address_16k, alloc_size, VM_FLAGS_ANYWHERE);
2236 assert3u(kr, ==, KERN_SUCCESS); /* reserve space in 16k map succeeds */
2237
2238 /* do the overwrite */
2239 kr = vm_map_copy_overwrite(map_16k, address_16k, copy, alloc_size,
2240 true);
2241 assert3u(kr, ==, KERN_SUCCESS); /* copy_overwrite into 16k map succeds */
2242 } else {
2243 panic("invalid vm_map_4k_16k_test variant: %lld", in);
2244 }
2245
2246 /* validate that everything is combined into one large 16k-aligned entry */
2247 mach_vm_size_t expected_size = VM_MAP_ROUND_PAGE(alloc_size, SIXTEENK_PAGE_MASK);
2248 vm_map_lock_read(map_16k);
2249 vm_map_entry_t entry;
2250 bool address_in_map = vm_map_lookup_entry(map_16k, address_16k, &entry);
2251 assert(address_in_map); /* address_16k found in map_16k */
2252 assert3u((entry->vme_end - entry->vme_start), ==, expected_size); /* 4k entries combined into a single 16k entry */
2253 vm_map_unlock_read(map_16k);
2254 #else /* !PMAP_CREATE_FORCE_4K_PAGES */
2255 (void)in;
2256 #endif /* !PMAP_CREATE_FORCE_4K_PAGES */
2257 *out = 1;
2258 return 0;
2259 }
2260 SYSCTL_TEST_REGISTER(vm_map_4k_16k, vm_map_4k_16k_test);
2261
2262 static int
vm_vector_upl_test(int64_t in,int64_t * out)2263 vm_vector_upl_test(int64_t in, int64_t *out)
2264 {
2265 extern upl_t vector_upl_create(vm_offset_t, uint32_t);
2266 extern boolean_t vector_upl_set_subupl(upl_t, upl_t, uint32_t);
2267
2268 upl_t vector_upl = NULL;
2269 vm_address_t kva = 0;
2270
2271 *out = 0;
2272
2273 struct {
2274 uint64_t iov;
2275 uint16_t iovcnt;
2276 } args;
2277
2278 struct {
2279 uint64_t base;
2280 uint32_t len;
2281 } *iov;
2282
2283 size_t iovsize = 0;
2284 iov = NULL;
2285
2286 int error = copyin((user_addr_t)in, &args, sizeof(args));
2287 if ((error != 0) || (args.iovcnt == 0)) {
2288 goto vector_upl_test_done;
2289 }
2290
2291 iovsize = sizeof(*iov) * args.iovcnt;
2292
2293 iov = kalloc_data(iovsize, Z_WAITOK_ZERO);
2294 if (iov == NULL) {
2295 error = ENOMEM;
2296 goto vector_upl_test_done;
2297 }
2298
2299 error = copyin((user_addr_t)args.iov, iov, iovsize);
2300 if (error != 0) {
2301 goto vector_upl_test_done;
2302 }
2303
2304 vector_upl = vector_upl_create(iov->base & PAGE_MASK, args.iovcnt);
2305 upl_size_t vector_upl_size = 0;
2306
2307 /* Create each sub-UPL and append it to the top-level vector UPL. */
2308 for (uint16_t i = 0; i < args.iovcnt; i++) {
2309 upl_t subupl;
2310 upl_size_t upl_size = iov[i].len;
2311 unsigned int upl_count = 0;
2312 upl_control_flags_t upl_flags = UPL_SET_IO_WIRE | UPL_SET_LITE | UPL_WILL_MODIFY | UPL_SET_INTERNAL;
2313 kern_return_t kr = vm_map_create_upl(current_map(),
2314 (vm_map_offset_t)iov[i].base,
2315 &upl_size,
2316 &subupl,
2317 NULL,
2318 &upl_count,
2319 &upl_flags,
2320 VM_KERN_MEMORY_DIAG);
2321 if (kr != KERN_SUCCESS) {
2322 printf("vm_map_create_upl[%d](%p, 0x%lx) returned 0x%x\n",
2323 (int)i, (void*)iov[i].base, (unsigned long)iov[i].len, kr);
2324 error = EIO;
2325 goto vector_upl_test_done;
2326 }
2327 /* This effectively transfers our reference to subupl over to vector_upl. */
2328 vector_upl_set_subupl(vector_upl, subupl, upl_size);
2329 vector_upl_set_iostate(vector_upl, subupl, vector_upl_size, upl_size);
2330 vector_upl_size += upl_size;
2331 }
2332
2333 /* Map the vector UPL as a single KVA region and modify the page contents by adding 1 to each char. */
2334 kern_return_t kr = vm_upl_map(kernel_map, vector_upl, &kva);
2335 if (kr != KERN_SUCCESS) {
2336 error = ENOMEM;
2337 goto vector_upl_test_done;
2338 }
2339
2340 char *buf = (char*)kva;
2341 for (upl_size_t i = 0; i < vector_upl_size; i++) {
2342 buf[i] = buf[i] + 1;
2343 }
2344 *out = (int64_t)vector_upl_size;
2345
2346 vector_upl_test_done:
2347
2348 if (kva != 0) {
2349 vm_upl_unmap(kernel_map, vector_upl);
2350 }
2351
2352 if (vector_upl != NULL) {
2353 /* Committing the vector UPL will release and deallocate each of its sub-UPLs. */
2354 upl_commit(vector_upl, NULL, 0);
2355 upl_deallocate(vector_upl);
2356 }
2357
2358 if (iov != NULL) {
2359 kfree_data(iov, iovsize);
2360 }
2361
2362 return error;
2363 }
2364 SYSCTL_TEST_REGISTER(vm_vector_upl, vm_vector_upl_test);
2365
2366 /*
2367 * Test that wiring copy delay memory pushes pages to its copy object
2368 */
2369 static int
vm_map_wire_copy_delay_memory_test(__unused int64_t in,int64_t * out)2370 vm_map_wire_copy_delay_memory_test(__unused int64_t in, int64_t *out)
2371 {
2372 kern_return_t kr;
2373 vm_map_t map;
2374 mach_vm_address_t address_a, address_b, address_c;
2375 vm_prot_t cur_prot, max_prot;
2376 vm_map_entry_t entry;
2377 vm_object_t object;
2378 vm_page_t m;
2379 bool result;
2380
2381 T_BEGIN("vm_map_wire_copy_delay_memory_test");
2382 map = create_map(0x100000000ULL, 0x200000000ULL);
2383
2384 address_a = 0;
2385 kr = mach_vm_allocate(
2386 map,
2387 &address_a,
2388 /* size */ PAGE_SIZE,
2389 VM_FLAGS_ANYWHERE);
2390 T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "mach_vm_allocate A");
2391
2392 address_b = 0;
2393 kr = mach_vm_remap(
2394 map,
2395 &address_b,
2396 /* size */ PAGE_SIZE,
2397 /* mask */ 0,
2398 VM_FLAGS_ANYWHERE,
2399 map,
2400 address_a,
2401 /* copy */ FALSE,
2402 &cur_prot,
2403 &max_prot,
2404 VM_INHERIT_NONE);
2405 T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "mach_vm_remap A->B");
2406
2407 address_c = 0;
2408 kr = mach_vm_remap(
2409 map,
2410 &address_c,
2411 /* size */ PAGE_SIZE,
2412 /* mask */ 0,
2413 VM_FLAGS_ANYWHERE,
2414 map,
2415 address_b,
2416 /* copy */ TRUE,
2417 &cur_prot,
2418 &max_prot,
2419 VM_INHERIT_NONE);
2420 T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "mach_vm_remap B->C");
2421
2422 kr = mach_vm_protect(
2423 map,
2424 address_c,
2425 /* size */ PAGE_SIZE,
2426 /* set_max */ FALSE,
2427 VM_PROT_READ);
2428 T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "mach_vm_protect C");
2429
2430 kr = vm_map_wire_kernel(
2431 map,
2432 /* begin */ address_b,
2433 /* end */ address_b + PAGE_SIZE,
2434 VM_PROT_NONE,
2435 VM_KERN_MEMORY_OSFMK,
2436 false);
2437 T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "vm_map_wire_kernel B");
2438
2439 vm_map_lock(map);
2440 result = vm_map_lookup_entry(map, address_c, &entry);
2441 T_ASSERT_EQ_INT(result, true, "vm_map_lookup_entry");
2442
2443 object = VME_OBJECT(entry);
2444 T_ASSERT_NOTNULL(object, "C's object should not be null");
2445 vm_object_lock(object);
2446
2447 m = vm_page_lookup(object, /* offset */ 0);
2448 T_ASSERT_NOTNULL(m, "C should have a page pushed to it");
2449
2450 /* cleanup */
2451 vm_object_unlock(object);
2452 vm_map_unlock(map);
2453 cleanup_map(&map);
2454
2455 T_END;
2456 *out = 1;
2457 return 0;
2458 }
2459 SYSCTL_TEST_REGISTER(vm_map_wire_copy_delay_memory, vm_map_wire_copy_delay_memory_test);
2460
2461
2462 /*
2463 * Compare the contents of an original userspace buffer with that kernel mapping of a UPL created
2464 * against that userspace buffer. Also validate that the physical pages in the UPL's page list
2465 * match the physical pages backing the kernel mapping at the pmap layer. Furthermore, if UPL creation
2466 * was expected to copy the original buffer, validate that the backing pages for the userspace buffer
2467 * don't match the kernel/UPL pages, otherwise validate that they do match.
2468 */
2469 static int
upl_buf_compare(user_addr_t src,upl_t upl,const void * upl_buf,upl_size_t size,bool copy_expected)2470 upl_buf_compare(user_addr_t src, upl_t upl, const void *upl_buf, upl_size_t size, bool copy_expected)
2471 {
2472 int error = 0;
2473 void *temp = kalloc_data(PAGE_SIZE, Z_WAITOK);
2474
2475 upl_size_t i = 0;
2476 while (i < size) {
2477 size_t bytes = MIN(size - i, PAGE_SIZE);
2478 error = copyin(src + i, temp, bytes);
2479 if (!error && (memcmp(temp, (const void*)((uintptr_t)upl_buf + i), bytes) != 0)) {
2480 printf("%s: memcmp(%p, %p, %zu) failed, src[0] = 0x%llx, buf[0] = 0x%llx\n",
2481 __func__, (void*)(src + i), (const void*)((uintptr_t)upl_buf + i), bytes, *((unsigned long long*)temp), *((unsigned long long*)((uintptr_t)upl_buf + i)));
2482 error = EINVAL;
2483 }
2484 if (!error) {
2485 ppnum_t user_pa = pmap_find_phys(current_map()->pmap, (addr64_t)src + i);
2486 ppnum_t upl_pa = pmap_find_phys(kernel_pmap, (addr64_t)upl_buf + i);
2487 if ((upl_pa == 0) || /* UPL is wired, PA should always be valid */
2488 (!copy_expected && (upl_pa != user_pa)) ||
2489 (copy_expected && (upl_pa == user_pa)) ||
2490 (upl_pa != (upl->page_list[i >> PAGE_SHIFT].phys_addr))) {
2491 printf("%s: PA verification[%u] failed: copy=%u, upl_pa = 0x%lx, user_pa = 0x%lx, page list PA = 0x%lx\n",
2492 __func__, (unsigned)i, (unsigned)copy_expected, (unsigned long)upl_pa, (unsigned long)user_pa,
2493 (unsigned long)upl->page_list[i].phys_addr);
2494 error = EFAULT;
2495 }
2496 }
2497 if (error) {
2498 break;
2499 }
2500 i += bytes;
2501 }
2502
2503 kfree_data(temp, PAGE_SIZE);
2504
2505 return error;
2506 }
2507
2508 static int
vm_upl_test(int64_t in,int64_t * out __unused)2509 vm_upl_test(int64_t in, int64_t *out __unused)
2510 {
2511 upl_t upl = NULL;
2512 vm_address_t kva = 0;
2513
2514 struct {
2515 uint64_t ptr; /* Base address of buffer in userspace */
2516 uint32_t size; /* Size of userspace buffer (in bytes) */
2517 char test_pattern; /* Starting char of test pattern we should write (if applicable) */
2518 bool copy_expected; /* Is UPL creation expected to create a copy of the original buffer? */
2519 bool should_fail; /* Is UPL creation expected to fail due to permissions checking? */
2520 bool upl_rw; /* Should the UPL be created RW (!UPL_COPYOUT_FROM) instead of RO? */
2521 } args;
2522 int error = copyin((user_addr_t)in, &args, sizeof(args));
2523 if ((error != 0) || (args.size == 0)) {
2524 goto upl_test_done;
2525 }
2526
2527 upl_size_t upl_size = args.size;
2528 unsigned int upl_count = 0;
2529 upl_control_flags_t upl_flags = UPL_SET_IO_WIRE | UPL_SET_LITE | UPL_SET_INTERNAL;
2530 if (!args.upl_rw) {
2531 upl_flags |= UPL_COPYOUT_FROM;
2532 } else {
2533 upl_flags |= UPL_WILL_MODIFY;
2534 }
2535 kern_return_t kr = vm_map_create_upl(current_map(),
2536 (vm_map_offset_t)args.ptr,
2537 &upl_size,
2538 &upl,
2539 NULL,
2540 &upl_count,
2541 &upl_flags,
2542 VM_KERN_MEMORY_DIAG);
2543 if (args.should_fail && (kr == KERN_PROTECTION_FAILURE)) {
2544 goto upl_test_done;
2545 } else if (args.should_fail && (kr == KERN_SUCCESS)) {
2546 printf("%s: vm_map_create_upl(%p, 0x%lx) did not fail as expected\n",
2547 __func__, (void*)args.ptr, (unsigned long)args.size);
2548 error = EIO;
2549 goto upl_test_done;
2550 } else if (kr != KERN_SUCCESS) {
2551 printf("%s: vm_map_create_upl(%p, 0x%lx) returned 0x%x\n",
2552 __func__, (void*)args.ptr, (unsigned long)args.size, kr);
2553 error = kr;
2554 goto upl_test_done;
2555 }
2556
2557 kr = vm_upl_map(kernel_map, upl, &kva);
2558 if (kr != KERN_SUCCESS) {
2559 error = kr;
2560 printf("%s: vm_upl_map() returned 0x%x\n", __func__, kr);
2561 goto upl_test_done;
2562 }
2563
2564 /* Ensure the mapped UPL contents match the original user buffer contents */
2565 error = upl_buf_compare((user_addr_t)args.ptr, upl, (void*)kva, upl_size, args.copy_expected);
2566
2567 if (error) {
2568 printf("%s: upl_buf_compare(%p, %p, %zu) failed\n",
2569 __func__, (void*)args.ptr, (void*)kva, (size_t)upl_size);
2570 }
2571
2572 if (!error && args.upl_rw) {
2573 /*
2574 * If the UPL is writable, update the contents so that userspace can
2575 * validate that it sees the updates.
2576 */
2577 for (unsigned int i = 0; i < (upl_size / sizeof(unsigned int)); i++) {
2578 ((unsigned int*)kva)[i] = (unsigned int)args.test_pattern + i;
2579 }
2580 }
2581
2582 upl_test_done:
2583
2584 if (kva != 0) {
2585 vm_upl_unmap(kernel_map, upl);
2586 }
2587
2588 if (upl != NULL) {
2589 upl_commit(upl, NULL, 0);
2590 upl_deallocate(upl);
2591 }
2592
2593 return error;
2594 }
2595 SYSCTL_TEST_REGISTER(vm_upl, vm_upl_test);
2596
2597 static int
vm_upl_submap_test(int64_t in,int64_t * out __unused)2598 vm_upl_submap_test(int64_t in, int64_t *out __unused)
2599 {
2600 vm_map_address_t start = 0x180000000ULL;
2601 vm_map_address_t end = start + 0x180000000ULL;
2602
2603 upl_t upl = NULL;
2604 vm_address_t kva = 0;
2605 int error = 0;
2606
2607 /*
2608 * Create temporary pmap and VM map for nesting our submap.
2609 * We can't directly nest our submap into the current user map, because it will
2610 * have already nested the shared region, and our security model doesn't allow
2611 * multiple nested pmaps.
2612 */
2613 pmap_t temp_pmap = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT);
2614
2615 vm_map_t temp_map = VM_MAP_NULL;
2616 if (temp_pmap != PMAP_NULL) {
2617 temp_map = vm_map_create_options(temp_pmap, 0, 0xfffffffffffff, 0);
2618 }
2619
2620 /* Now create the pmap and VM map that will back the submap entry in 'temp_map'. */
2621 pmap_t nested_pmap = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT | PMAP_CREATE_NESTED);
2622
2623 vm_map_t nested_map = VM_MAP_NULL;
2624 if (nested_pmap != PMAP_NULL) {
2625 #if defined(__arm64__)
2626 pmap_set_nested(nested_pmap);
2627 #endif /* defined(__arm64__) */
2628 #if CODE_SIGNING_MONITOR
2629 csm_setup_nested_address_space(nested_pmap, start, end - start);
2630 #endif
2631 nested_map = vm_map_create_options(nested_pmap, 0, end - start, 0);
2632 }
2633
2634 if (temp_map == VM_MAP_NULL || nested_map == VM_MAP_NULL) {
2635 error = ENOMEM;
2636 printf("%s: failed to create VM maps\n", __func__);
2637 goto upl_submap_test_done;
2638 }
2639
2640 nested_map->is_nested_map = TRUE;
2641 nested_map->vmmap_sealed = VM_MAP_WILL_BE_SEALED;
2642
2643 struct {
2644 uint64_t ptr; /* Base address of original buffer in userspace */
2645 uint64_t upl_base; /* Base address in 'temp_map' against which UPL should be created */
2646 uint32_t size; /* Size of userspace buffer in bytes */
2647 uint32_t upl_size; /* Size of UPL to create in bytes */
2648 bool upl_rw; /* Should the UPL be created RW (!UPL_COPYOUT_FROM) instead of RO? */
2649 } args;
2650 error = copyin((user_addr_t)in, &args, sizeof(args));
2651 if ((error != 0) || (args.size == 0) || (args.upl_size == 0)) {
2652 goto upl_submap_test_done;
2653 }
2654
2655 /*
2656 * Remap the original userspace buffer into the nested map, with CoW protection.
2657 * This will not actually instantiate new mappings in 'nested_pmap', but will instead create
2658 * new shadow object of the original object for the userspace buffer in the nested map.
2659 * Mappings would only be created in 'nested_pmap' upon a later non-CoW fault of the nested region,
2660 * which we aren't doing here. That's fine, as we're not testing pmap functionality here; we
2661 * only care that UPL creation produces the expected results at the VM map/entry level.
2662 */
2663 mach_vm_offset_t submap_start = 0;
2664
2665 vm_prot_ut remap_cur_prot = vm_sanitize_wrap_prot(VM_PROT_READ);
2666 vm_prot_ut remap_max_prot = vm_sanitize_wrap_prot(VM_PROT_READ);
2667
2668 kern_return_t kr = mach_vm_remap_new_kernel(nested_map, (mach_vm_offset_ut*)&submap_start, args.size, 0,
2669 VM_MAP_KERNEL_FLAGS_FIXED(.vm_tag = VM_KERN_MEMORY_OSFMK), current_map(), args.ptr, TRUE,
2670 &remap_cur_prot, &remap_max_prot, VM_INHERIT_NONE);
2671 if (kr != KERN_SUCCESS) {
2672 printf("%s: failed to remap source buffer to nested map: 0x%x\n", __func__, kr);
2673 error = kr;
2674 goto upl_submap_test_done;
2675 }
2676
2677 vm_map_seal(nested_map, true);
2678 pmap_set_shared_region(temp_pmap, nested_pmap, start, end - start);
2679
2680 /* Do the actual nesting. */
2681 vm_map_reference(nested_map);
2682 kr = vm_map_enter(temp_map, &start, end - start, 0,
2683 VM_MAP_KERNEL_FLAGS_FIXED(.vmkf_submap = TRUE, .vmkf_nested_pmap = TRUE), (vm_object_t)(uintptr_t) nested_map, 0,
2684 true, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_DEFAULT);
2685
2686 if (kr != KERN_SUCCESS) {
2687 error = kr;
2688 printf("%s: failed to enter nested map in test map: 0x%x\n", __func__, kr);
2689 vm_map_deallocate(nested_map);
2690 goto upl_submap_test_done;
2691 }
2692
2693 /* Validate that the nesting operation produced the expected submap entry in 'temp_map'. */
2694 vm_map_entry_t submap_entry;
2695 if (!vm_map_lookup_entry(temp_map, args.upl_base, &submap_entry) || !submap_entry->is_sub_map) {
2696 error = ENOENT;
2697 printf("%s: did not find submap entry at beginning up UPL region\n", __func__);
2698 goto upl_submap_test_done;
2699 }
2700
2701 upl_size_t upl_size = args.upl_size;
2702 unsigned int upl_count = 0;
2703 upl_control_flags_t upl_flags = UPL_SET_IO_WIRE | UPL_SET_LITE | UPL_SET_INTERNAL;
2704 if (!args.upl_rw) {
2705 upl_flags |= UPL_COPYOUT_FROM;
2706 }
2707 kr = vm_map_create_upl(temp_map,
2708 (vm_map_offset_t)args.upl_base,
2709 &upl_size,
2710 &upl,
2711 NULL,
2712 &upl_count,
2713 &upl_flags,
2714 VM_KERN_MEMORY_DIAG);
2715
2716 if (kr != KERN_SUCCESS) {
2717 error = kr;
2718 printf("%s: failed to create UPL for submap: 0x%x\n", __func__, kr);
2719 goto upl_submap_test_done;
2720 }
2721
2722 /* Validate that UPL creation unnested a portion of the submap entry. */
2723 if (!vm_map_lookup_entry(temp_map, args.upl_base, &submap_entry) || submap_entry->is_sub_map) {
2724 error = ENOENT;
2725 printf("%s: did not find non-submap entry at beginning up UPL region\n", __func__);
2726 goto upl_submap_test_done;
2727 }
2728
2729 kr = vm_upl_map(kernel_map, upl, &kva);
2730 if (kr != KERN_SUCCESS) {
2731 error = kr;
2732 goto upl_submap_test_done;
2733 }
2734
2735 /*
2736 * Compare the original userspace buffer to the ultimate kernel mapping of the UPL.
2737 * The unnesting and CoW faulting performed as part of UPL creation should have copied the original buffer
2738 * pages, so we expect the two buffers to be backed by different pages.
2739 */
2740 error = upl_buf_compare((user_addr_t)args.ptr + (args.upl_base - start), upl, (void*)kva, upl_size, true);
2741
2742 if (!error) {
2743 /*
2744 * Now validate that the nested region in 'temp_map' matches the original buffer.
2745 * The unnesting and CoW faulting performed as part of UPL creation should have acted directly
2746 * upon 'temp_map', so the backing pages should be the same here.
2747 */
2748 vm_map_switch_context_t switch_ctx = vm_map_switch_to(temp_map);
2749 error = upl_buf_compare((user_addr_t)args.upl_base, upl, (void*)kva, upl_size, false);
2750 vm_map_switch_back(switch_ctx);
2751 }
2752
2753 upl_submap_test_done:
2754
2755 if (kva != 0) {
2756 vm_upl_unmap(kernel_map, upl);
2757 }
2758
2759 if (upl != NULL) {
2760 upl_commit(upl, NULL, 0);
2761 upl_deallocate(upl);
2762 }
2763
2764 if (temp_map != VM_MAP_NULL) {
2765 vm_map_deallocate(temp_map);
2766 temp_pmap = PMAP_NULL;
2767 }
2768 if (nested_map != VM_MAP_NULL) {
2769 vm_map_deallocate(nested_map);
2770 nested_pmap = PMAP_NULL;
2771 }
2772
2773 if (temp_pmap != PMAP_NULL) {
2774 pmap_destroy(temp_pmap);
2775 }
2776 if (nested_pmap != PMAP_NULL) {
2777 pmap_destroy(nested_pmap);
2778 }
2779
2780 return error;
2781 }
2782 SYSCTL_TEST_REGISTER(vm_upl_submap, vm_upl_submap_test);
2783
2784 #if CONFIG_SPTM
2785
2786 static void
page_clean_timeout(thread_call_param_t param0,__unused thread_call_param_t param1)2787 page_clean_timeout(thread_call_param_t param0, __unused thread_call_param_t param1)
2788 {
2789 vm_page_t m = (vm_page_t)param0;
2790 vm_object_t object = VM_PAGE_OBJECT(m);
2791 vm_object_lock(object);
2792 m->vmp_cleaning = false;
2793 vm_page_wakeup(object, m);
2794 vm_object_unlock(object);
2795 }
2796
2797 /**
2798 * This sysctl is meant to exercise very specific functionality that can't be exercised through
2799 * the normal vm_map_create_upl() path. It operates directly against the vm_object backing
2800 * the specified address range, and does not take any locks against the VM map to guarantee
2801 * stability of the specified address range. It is therefore meant to be used against
2802 * VM regions directly allocated by the userspace caller and guaranteed to not be altered by
2803 * other threads. The regular vm_upl/vm_upl_submap sysctls should be preferred over this
2804 * if at all possible.
2805 */
2806 static int
vm_upl_object_test(int64_t in,int64_t * out __unused)2807 vm_upl_object_test(int64_t in, int64_t *out __unused)
2808 {
2809 upl_t upl = NULL;
2810
2811 struct {
2812 uint64_t ptr; /* Base address of buffer in userspace */
2813 uint32_t size; /* Size of userspace buffer (in bytes) */
2814 bool upl_rw;
2815 bool should_fail; /* Is UPL creation expected to fail due to permissions checking? */
2816 bool exec_fault;
2817 } args;
2818 int error = copyin((user_addr_t)in, &args, sizeof(args));
2819 if ((error != 0) || (args.size == 0)) {
2820 goto upl_object_test_done;
2821 }
2822
2823 upl_size_t upl_size = args.size;
2824 unsigned int upl_count = 0;
2825 upl_control_flags_t upl_flags = UPL_SET_IO_WIRE | UPL_SET_LITE | UPL_SET_INTERNAL;
2826 if (!args.upl_rw) {
2827 upl_flags |= UPL_COPYOUT_FROM;
2828 } else {
2829 upl_flags |= UPL_WILL_MODIFY;
2830 }
2831
2832 vm_map_entry_t entry;
2833 vm_object_t object;
2834 vm_page_t m __unused;
2835
2836 if (!vm_map_lookup_entry(current_map(), args.ptr, &entry) || entry->is_sub_map) {
2837 error = ENOENT;
2838 printf("%s: did not find entry at beginning up UPL region\n", __func__);
2839 goto upl_object_test_done;
2840 }
2841
2842 object = VME_OBJECT(entry);
2843 if (object == VM_OBJECT_NULL) {
2844 error = ENOENT;
2845 printf("%s: No VM object associated with entry at beginning of UPL region\n", __func__);
2846 goto upl_object_test_done;
2847 }
2848
2849 vm_object_reference(object);
2850
2851 kern_return_t kr = vm_object_iopl_request(object,
2852 (vm_object_offset_t)(args.ptr - entry->vme_start + VME_OFFSET(entry)),
2853 upl_size,
2854 &upl,
2855 NULL,
2856 &upl_count,
2857 upl_flags,
2858 VM_KERN_MEMORY_DIAG);
2859
2860 if (args.exec_fault) {
2861 /*
2862 * The page may have already been retyped to its "final" executable type by a prior fault,
2863 * so simulate a page recycle operation in order to ensure that our simulated exec fault below
2864 * will attempt to retype it.
2865 */
2866 vm_object_lock(object);
2867 m = vm_page_lookup(object, (VME_OFFSET(entry) + ((vm_map_address_t)args.ptr - entry->vme_start)));
2868 assert(m != VM_PAGE_NULL);
2869 assert(m->vmp_iopl_wired);
2870 ppnum_t pn = VM_PAGE_GET_PHYS_PAGE(m);
2871 pmap_disconnect(pn);
2872 pmap_lock_phys_page(pn);
2873 pmap_recycle_page(pn);
2874 pmap_unlock_phys_page(pn);
2875 assertf(pmap_will_retype(current_map()->pmap, (vm_map_address_t)args.ptr, VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_EXECUTE | VM_PROT_READ, 0, PMAP_MAPPING_TYPE_INFER),
2876 "pmap will not retype for vm_page_t %p", m);
2877 vm_object_unlock(object);
2878 }
2879
2880 if (args.should_fail && (kr == KERN_PROTECTION_FAILURE)) {
2881 goto upl_object_test_done;
2882 } else if (args.should_fail && (kr == KERN_SUCCESS)) {
2883 printf("%s: vm_object_iopl_request(%p, 0x%lx) did not fail as expected\n",
2884 __func__, (void*)args.ptr, (unsigned long)args.size);
2885 error = EIO;
2886 goto upl_object_test_done;
2887 } else if (kr != KERN_SUCCESS) {
2888 printf("%s: vm_object_iopl_request(%p, 0x%lx) returned 0x%x\n",
2889 __func__, (void*)args.ptr, (unsigned long)args.size, kr);
2890 error = kr;
2891 goto upl_object_test_done;
2892 }
2893
2894 if (args.exec_fault) {
2895 kr = vm_fault(current_map(),
2896 (vm_map_address_t)args.ptr,
2897 VM_PROT_EXECUTE | VM_PROT_READ,
2898 FALSE,
2899 VM_KERN_MEMORY_NONE,
2900 THREAD_UNINT,
2901 NULL,
2902 0);
2903 /* Exec page retype attempt with in-flight IOPL should be forbidden. */
2904 if (kr != KERN_PROTECTION_FAILURE) {
2905 printf("%s: vm_fault(%p) did not fail as expected\n", __func__, (void*)args.ptr);
2906 error = ((kr == KERN_SUCCESS) ? EIO : kr);
2907 goto upl_object_test_done;
2908 }
2909 assertf(pmap_will_retype(current_map()->pmap, (vm_map_address_t)args.ptr, VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_EXECUTE | VM_PROT_READ, 0, PMAP_MAPPING_TYPE_INFER),
2910 "pmap will not retype for vm_page_t %p", m);
2911 }
2912
2913 upl_object_test_done:
2914
2915 if (upl != NULL) {
2916 upl_commit(upl, NULL, 0);
2917 upl_deallocate(upl);
2918 }
2919
2920 if ((error == 0) && args.exec_fault) {
2921 /*
2922 * Exec page retype attempt without in-flight IOPL should ultimately succeed, but should
2923 * block if the page is being cleaned. Simulate that scenario with a thread call to "finish"
2924 * the clean operation and wake up the waiting fault handler after 1s.
2925 */
2926 vm_object_lock(object);
2927 assert(!m->vmp_iopl_wired);
2928 m->vmp_cleaning = true;
2929 vm_object_unlock(object);
2930 thread_call_t page_clean_timer_call = thread_call_allocate(page_clean_timeout, m);
2931 uint64_t deadline;
2932 clock_interval_to_deadline(1, NSEC_PER_SEC, &deadline);
2933 thread_call_enter_delayed(page_clean_timer_call, deadline);
2934 kr = vm_fault(current_map(),
2935 (vm_map_address_t)args.ptr,
2936 VM_PROT_EXECUTE | VM_PROT_READ,
2937 FALSE,
2938 VM_KERN_MEMORY_NONE,
2939 THREAD_UNINT,
2940 NULL,
2941 0);
2942 /*
2943 * Thread call should no longer be active, as its expiry should have been the thing that
2944 * unblocked the fault above.
2945 */
2946 assert(!thread_call_isactive(page_clean_timer_call));
2947 thread_call_free(page_clean_timer_call);
2948 if (kr != KERN_SUCCESS) {
2949 printf("%s: vm_fault(%p) did not succeed as expected\n", __func__, (void*)args.ptr);
2950 error = kr;
2951 }
2952 }
2953
2954 if (object != VM_OBJECT_NULL) {
2955 vm_object_deallocate(object);
2956 }
2957
2958 return error;
2959 }
2960 SYSCTL_TEST_REGISTER(vm_upl_object, vm_upl_object_test);
2961
2962 #endif /* CONFIG_SPTM */
2963