1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/memory_object.h>
33 #include <mach/vm_map.h>
34
35 #include <kern/ledger.h>
36
37 #include <device/device_port.h>
38 #include <vm/memory_object.h>
39 #include <vm/vm_fault.h>
40 #include <vm/vm_map_internal.h>
41 #include <vm/vm_object.h>
42 #include <vm/vm_pageout.h>
43 #include <vm/vm_protos.h>
44
45 #include <mach/mach_vm.h>
46
47 #include <sys/errno.h> /* for the sysctl tests */
48
49 extern ledger_template_t task_ledger_template;
50
51 extern kern_return_t
52 vm_map_copy_adjust_to_target(
53 vm_map_copy_t copy_map,
54 vm_map_offset_t offset,
55 vm_map_size_t size,
56 vm_map_t target_map,
57 boolean_t copy,
58 vm_map_copy_t *target_copy_map_p,
59 vm_map_offset_t *overmap_start_p,
60 vm_map_offset_t *overmap_end_p,
61 vm_map_offset_t *trimmed_start_p);
62
63 #define VM_TEST_COLLAPSE_COMPRESSOR 0
64 #define VM_TEST_WIRE_AND_EXTRACT 0
65 #define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0
66 #if __arm64__
67 #define VM_TEST_KERNEL_OBJECT_FAULT 0
68 #endif /* __arm64__ */
69 #define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG)
70
71 #if VM_TEST_COLLAPSE_COMPRESSOR
72 extern boolean_t vm_object_collapse_compressor_allowed;
73 #include <IOKit/IOLib.h>
74 static void
vm_test_collapse_compressor(void)75 vm_test_collapse_compressor(void)
76 {
77 vm_object_size_t backing_size, top_size;
78 vm_object_t backing_object, top_object;
79 vm_map_offset_t backing_offset, top_offset;
80 unsigned char *backing_address, *top_address;
81 kern_return_t kr;
82
83 printf("VM_TEST_COLLAPSE_COMPRESSOR:\n");
84
85 /* create backing object */
86 backing_size = 15 * PAGE_SIZE;
87 backing_object = vm_object_allocate(backing_size);
88 assert(backing_object != VM_OBJECT_NULL);
89 printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
90 backing_object);
91 /* map backing object */
92 backing_offset = 0;
93 kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
94 VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_DATA,
95 backing_object, 0, FALSE,
96 VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
97 assert(kr == KERN_SUCCESS);
98 backing_address = (unsigned char *) backing_offset;
99 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
100 "mapped backing object %p at 0x%llx\n",
101 backing_object, (uint64_t) backing_offset);
102 /* populate with pages to be compressed in backing object */
103 backing_address[0x1 * PAGE_SIZE] = 0xB1;
104 backing_address[0x4 * PAGE_SIZE] = 0xB4;
105 backing_address[0x7 * PAGE_SIZE] = 0xB7;
106 backing_address[0xa * PAGE_SIZE] = 0xBA;
107 backing_address[0xd * PAGE_SIZE] = 0xBD;
108 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
109 "populated pages to be compressed in "
110 "backing_object %p\n", backing_object);
111 /* compress backing object */
112 vm_object_pageout(backing_object);
113 printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
114 backing_object);
115 /* wait for all the pages to be gone */
116 while (*(volatile int *)&backing_object->resident_page_count != 0) {
117 IODelay(10);
118 }
119 printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
120 backing_object);
121 /* populate with pages to be resident in backing object */
122 backing_address[0x0 * PAGE_SIZE] = 0xB0;
123 backing_address[0x3 * PAGE_SIZE] = 0xB3;
124 backing_address[0x6 * PAGE_SIZE] = 0xB6;
125 backing_address[0x9 * PAGE_SIZE] = 0xB9;
126 backing_address[0xc * PAGE_SIZE] = 0xBC;
127 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
128 "populated pages to be resident in "
129 "backing_object %p\n", backing_object);
130 /* leave the other pages absent */
131 /* mess with the paging_offset of the backing_object */
132 assert(backing_object->paging_offset == 0);
133 backing_object->paging_offset = 3 * PAGE_SIZE;
134
135 /* create top object */
136 top_size = 9 * PAGE_SIZE;
137 top_object = vm_object_allocate(top_size);
138 assert(top_object != VM_OBJECT_NULL);
139 printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
140 top_object);
141 /* map top object */
142 top_offset = 0;
143 kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
144 VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_DATA,
145 top_object, 0, FALSE,
146 VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
147 assert(kr == KERN_SUCCESS);
148 top_address = (unsigned char *) top_offset;
149 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
150 "mapped top object %p at 0x%llx\n",
151 top_object, (uint64_t) top_offset);
152 /* populate with pages to be compressed in top object */
153 top_address[0x3 * PAGE_SIZE] = 0xA3;
154 top_address[0x4 * PAGE_SIZE] = 0xA4;
155 top_address[0x5 * PAGE_SIZE] = 0xA5;
156 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
157 "populated pages to be compressed in "
158 "top_object %p\n", top_object);
159 /* compress top object */
160 vm_object_pageout(top_object);
161 printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
162 top_object);
163 /* wait for all the pages to be gone */
164 while (top_object->resident_page_count != 0) {
165 IODelay(10);
166 }
167 printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
168 top_object);
169 /* populate with pages to be resident in top object */
170 top_address[0x0 * PAGE_SIZE] = 0xA0;
171 top_address[0x1 * PAGE_SIZE] = 0xA1;
172 top_address[0x2 * PAGE_SIZE] = 0xA2;
173 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
174 "populated pages to be resident in "
175 "top_object %p\n", top_object);
176 /* leave the other pages absent */
177
178 /* link the 2 objects */
179 vm_object_reference(backing_object);
180 top_object->shadow = backing_object;
181 top_object->vo_shadow_offset = 3 * PAGE_SIZE;
182 printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
183 top_object, backing_object);
184
185 /* unmap backing object */
186 vm_map_remove(kernel_map,
187 backing_offset,
188 backing_offset + backing_size,
189 VM_MAP_REMOVE_NO_FLAGS);
190 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
191 "unmapped backing_object %p [0x%llx:0x%llx]\n",
192 backing_object,
193 (uint64_t) backing_offset,
194 (uint64_t) (backing_offset + backing_size));
195
196 /* collapse */
197 printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
198 vm_object_lock(top_object);
199 vm_object_collapse(top_object, 0, FALSE);
200 vm_object_unlock(top_object);
201 printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object);
202
203 /* did it work? */
204 if (top_object->shadow != VM_OBJECT_NULL) {
205 printf("VM_TEST_COLLAPSE_COMPRESSOR: not collapsed\n");
206 printf("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
207 if (vm_object_collapse_compressor_allowed) {
208 panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL");
209 }
210 } else {
211 /* check the contents of the mapping */
212 unsigned char expect[9] =
213 { 0xA0, 0xA1, 0xA2, /* resident in top */
214 0xA3, 0xA4, 0xA5, /* compressed in top */
215 0xB9, /* resident in backing + shadow_offset */
216 0xBD, /* compressed in backing + shadow_offset + paging_offset */
217 0x00 }; /* absent in both */
218 unsigned char actual[9];
219 unsigned int i, errors;
220
221 errors = 0;
222 for (i = 0; i < sizeof(actual); i++) {
223 actual[i] = (unsigned char) top_address[i * PAGE_SIZE];
224 if (actual[i] != expect[i]) {
225 errors++;
226 }
227 }
228 printf("VM_TEST_COLLAPSE_COMPRESSOR: "
229 "actual [%x %x %x %x %x %x %x %x %x] "
230 "expect [%x %x %x %x %x %x %x %x %x] "
231 "%d errors\n",
232 actual[0], actual[1], actual[2], actual[3],
233 actual[4], actual[5], actual[6], actual[7],
234 actual[8],
235 expect[0], expect[1], expect[2], expect[3],
236 expect[4], expect[5], expect[6], expect[7],
237 expect[8],
238 errors);
239 if (errors) {
240 panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL");
241 } else {
242 printf("VM_TEST_COLLAPSE_COMPRESSOR: PASS\n");
243 }
244 }
245 }
246 #else /* VM_TEST_COLLAPSE_COMPRESSOR */
247 #define vm_test_collapse_compressor()
248 #endif /* VM_TEST_COLLAPSE_COMPRESSOR */
249
250 #if VM_TEST_WIRE_AND_EXTRACT
251 extern ppnum_t vm_map_get_phys_page(vm_map_t map,
252 vm_offset_t offset);
253 static void
vm_test_wire_and_extract(void)254 vm_test_wire_and_extract(void)
255 {
256 ledger_t ledger;
257 vm_map_t user_map, wire_map;
258 mach_vm_address_t user_addr, wire_addr;
259 mach_vm_size_t user_size, wire_size;
260 mach_vm_offset_t cur_offset;
261 vm_prot_t cur_prot, max_prot;
262 ppnum_t user_ppnum, wire_ppnum;
263 kern_return_t kr;
264
265 ledger = ledger_instantiate(task_ledger_template,
266 LEDGER_CREATE_ACTIVE_ENTRIES);
267 pmap_t user_pmap = pmap_create_options(ledger, 0, PMAP_CREATE_64BIT);
268 assert(user_pmap);
269 user_map = vm_map_create_options(user_pmap,
270 0x100000000ULL,
271 0x200000000ULL,
272 VM_MAP_CREATE_PAGEABLE);
273 wire_map = vm_map_create_options(NULL,
274 0x100000000ULL,
275 0x200000000ULL,
276 VM_MAP_CREATE_PAGEABLE);
277 user_addr = 0;
278 user_size = 0x10000;
279 kr = mach_vm_allocate(user_map,
280 &user_addr,
281 user_size,
282 VM_FLAGS_ANYWHERE);
283 assert(kr == KERN_SUCCESS);
284 wire_addr = 0;
285 wire_size = user_size;
286 kr = mach_vm_remap(wire_map,
287 &wire_addr,
288 wire_size,
289 0,
290 VM_FLAGS_ANYWHERE,
291 user_map,
292 user_addr,
293 FALSE,
294 &cur_prot,
295 &max_prot,
296 VM_INHERIT_NONE);
297 assert(kr == KERN_SUCCESS);
298 for (cur_offset = 0;
299 cur_offset < wire_size;
300 cur_offset += PAGE_SIZE) {
301 kr = vm_map_wire_and_extract(wire_map,
302 wire_addr + cur_offset,
303 VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
304 TRUE,
305 &wire_ppnum);
306 assert(kr == KERN_SUCCESS);
307 user_ppnum = vm_map_get_phys_page(user_map,
308 user_addr + cur_offset);
309 printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x "
310 "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
311 kr,
312 user_map, user_addr + cur_offset, user_ppnum,
313 wire_map, wire_addr + cur_offset, wire_ppnum);
314 if (kr != KERN_SUCCESS ||
315 wire_ppnum == 0 ||
316 wire_ppnum != user_ppnum) {
317 panic("VM_TEST_WIRE_AND_EXTRACT: FAIL");
318 }
319 }
320 cur_offset -= PAGE_SIZE;
321 kr = vm_map_wire_and_extract(wire_map,
322 wire_addr + cur_offset,
323 VM_PROT_DEFAULT,
324 TRUE,
325 &wire_ppnum);
326 assert(kr == KERN_SUCCESS);
327 printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
328 "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
329 kr,
330 user_map, user_addr + cur_offset, user_ppnum,
331 wire_map, wire_addr + cur_offset, wire_ppnum);
332 if (kr != KERN_SUCCESS ||
333 wire_ppnum == 0 ||
334 wire_ppnum != user_ppnum) {
335 panic("VM_TEST_WIRE_AND_EXTRACT: FAIL");
336 }
337
338 printf("VM_TEST_WIRE_AND_EXTRACT: PASS\n");
339 }
340 #else /* VM_TEST_WIRE_AND_EXTRACT */
341 #define vm_test_wire_and_extract()
342 #endif /* VM_TEST_WIRE_AND_EXTRACT */
343
344 #if VM_TEST_PAGE_WIRE_OVERFLOW_PANIC
345 static void
vm_test_page_wire_overflow_panic(void)346 vm_test_page_wire_overflow_panic(void)
347 {
348 vm_object_t object;
349 vm_page_t page;
350
351 printf("VM_TEST_PAGE_WIRE_OVERFLOW_PANIC: starting...\n");
352
353 object = vm_object_allocate(PAGE_SIZE);
354 vm_object_lock(object);
355 page = vm_page_alloc(object, 0x0);
356 vm_page_lock_queues();
357 do {
358 vm_page_wire(page, 1, FALSE);
359 } while (page->wire_count != 0);
360 vm_page_unlock_queues();
361 vm_object_unlock(object);
362 panic("FBDP(%p,%p): wire_count overflow not detected",
363 object, page);
364 }
365 #else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
366 #define vm_test_page_wire_overflow_panic()
367 #endif /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
368
369 #if __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT
370 extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit);
371 static void
vm_test_kernel_object_fault(void)372 vm_test_kernel_object_fault(void)
373 {
374 vm_offset_t stack;
375 uintptr_t frameb[2];
376 int ret;
377
378 kmem_alloc(kernel_map, &stack,
379 kernel_stack_size + ptoa(2),
380 KMA_NOFAIL | KMA_KSTACK | KMA_KOBJECT |
381 KMA_GUARD_FIRST | KMA_GUARD_LAST,
382 VM_KERN_MEMORY_STACK);
383
384 ret = copyinframe((uintptr_t)stack, (char *)frameb, TRUE);
385 if (ret != 0) {
386 printf("VM_TEST_KERNEL_OBJECT_FAULT: PASS\n");
387 } else {
388 printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n");
389 }
390
391 kmem_free(kernel_map, stack, kernel_stack_size + ptoa(2));
392 stack = 0;
393 }
394 #else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
395 #define vm_test_kernel_object_fault()
396 #endif /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
397
398 #if VM_TEST_DEVICE_PAGER_TRANSPOSE
399 static void
vm_test_device_pager_transpose(void)400 vm_test_device_pager_transpose(void)
401 {
402 memory_object_t device_pager;
403 vm_object_t anon_object, device_object;
404 vm_size_t size;
405 vm_map_offset_t device_mapping;
406 kern_return_t kr;
407
408 size = 3 * PAGE_SIZE;
409 anon_object = vm_object_allocate(size);
410 assert(anon_object != VM_OBJECT_NULL);
411 device_pager = device_pager_setup(NULL, 0, size, 0);
412 assert(device_pager != NULL);
413 device_object = memory_object_to_vm_object(device_pager);
414 assert(device_object != VM_OBJECT_NULL);
415 #if 0
416 /*
417 * Can't actually map this, since another thread might do a
418 * vm_map_enter() that gets coalesced into this object, which
419 * would cause the test to fail.
420 */
421 vm_map_offset_t anon_mapping = 0;
422 kr = vm_map_enter(kernel_map, &anon_mapping, size, 0,
423 VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
424 anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
425 VM_INHERIT_DEFAULT);
426 assert(kr == KERN_SUCCESS);
427 #endif
428 device_mapping = 0;
429 kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0,
430 VM_FLAGS_ANYWHERE,
431 VM_MAP_KERNEL_FLAGS_DATA,
432 VM_KERN_MEMORY_NONE,
433 (void *)device_pager, 0, FALSE,
434 VM_PROT_DEFAULT, VM_PROT_ALL,
435 VM_INHERIT_DEFAULT);
436 assert(kr == KERN_SUCCESS);
437 memory_object_deallocate(device_pager);
438
439 vm_object_lock(anon_object);
440 vm_object_activity_begin(anon_object);
441 anon_object->blocked_access = TRUE;
442 vm_object_unlock(anon_object);
443 vm_object_lock(device_object);
444 vm_object_activity_begin(device_object);
445 device_object->blocked_access = TRUE;
446 vm_object_unlock(device_object);
447
448 assert(anon_object->ref_count == 1);
449 assert(!anon_object->named);
450 assert(device_object->ref_count == 2);
451 assert(device_object->named);
452
453 kr = vm_object_transpose(device_object, anon_object, size);
454 assert(kr == KERN_SUCCESS);
455
456 vm_object_lock(anon_object);
457 vm_object_activity_end(anon_object);
458 anon_object->blocked_access = FALSE;
459 vm_object_unlock(anon_object);
460 vm_object_lock(device_object);
461 vm_object_activity_end(device_object);
462 device_object->blocked_access = FALSE;
463 vm_object_unlock(device_object);
464
465 assert(anon_object->ref_count == 2);
466 assert(anon_object->named);
467 #if 0
468 kr = vm_deallocate(kernel_map, anon_mapping, size);
469 assert(kr == KERN_SUCCESS);
470 #endif
471 assert(device_object->ref_count == 1);
472 assert(!device_object->named);
473 kr = vm_deallocate(kernel_map, device_mapping, size);
474 assert(kr == KERN_SUCCESS);
475
476 printf("VM_TEST_DEVICE_PAGER_TRANSPOSE: PASS\n");
477 }
478 #else /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
479 #define vm_test_device_pager_transpose()
480 #endif /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
481
482 #if PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT
483 extern kern_return_t vm_allocate_external(vm_map_t map,
484 vm_offset_t *addr,
485 vm_size_t size,
486 int flags);
487 extern kern_return_t vm_remap_external(vm_map_t target_map,
488 vm_offset_t *address,
489 vm_size_t size,
490 vm_offset_t mask,
491 int flags,
492 vm_map_t src_map,
493 vm_offset_t memory_address,
494 boolean_t copy,
495 vm_prot_t *cur_protection,
496 vm_prot_t *max_protection,
497 vm_inherit_t inheritance);
498 extern int debug4k_panic_on_misaligned_sharing;
499
500 void vm_test_4k(void);
501 void
vm_test_4k(void)502 vm_test_4k(void)
503 {
504 pmap_t test_pmap;
505 vm_map_t test_map;
506 kern_return_t kr;
507 vm_address_t expected_addr;
508 vm_address_t alloc1_addr, alloc2_addr, alloc3_addr, alloc4_addr;
509 vm_address_t alloc5_addr, dealloc_addr, remap_src_addr, remap_dst_addr;
510 vm_size_t alloc1_size, alloc2_size, alloc3_size, alloc4_size;
511 vm_size_t alloc5_size, remap_src_size;
512 vm_address_t fault_addr;
513 vm_prot_t cur_prot, max_prot;
514 int saved_debug4k_panic_on_misaligned_sharing;
515
516 printf("\n\n\nVM_TEST_4K:%d creating 4K map...\n", __LINE__);
517 test_pmap = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT | PMAP_CREATE_FORCE_4K_PAGES);
518 assert(test_pmap != NULL);
519 test_map = vm_map_create_options(test_pmap,
520 MACH_VM_MIN_ADDRESS,
521 MACH_VM_MAX_ADDRESS,
522 VM_MAP_CREATE_PAGEABLE);
523 assert(test_map != VM_MAP_NULL);
524 vm_map_set_page_shift(test_map, FOURK_PAGE_SHIFT);
525 printf("VM_TEST_4K:%d map %p pmap %p page_size 0x%x\n", __LINE__, test_map, test_pmap, VM_MAP_PAGE_SIZE(test_map));
526
527 alloc1_addr = 0;
528 alloc1_size = 1 * FOURK_PAGE_SIZE;
529 expected_addr = 0x1000;
530 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
531 kr = vm_allocate_external(test_map,
532 &alloc1_addr,
533 alloc1_size,
534 VM_FLAGS_ANYWHERE);
535 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
536 assertf(alloc1_addr == expected_addr, "alloc1_addr = 0x%lx expected 0x%lx", alloc1_addr, expected_addr);
537 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
538 expected_addr += alloc1_size;
539
540 printf("VM_TEST_4K:%d vm_deallocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
541 kr = vm_deallocate(test_map, alloc1_addr, alloc1_size);
542 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
543 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
544
545 alloc1_addr = 0;
546 alloc1_size = 1 * FOURK_PAGE_SIZE;
547 expected_addr = 0x1000;
548 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc1_addr, alloc1_size);
549 kr = vm_allocate_external(test_map,
550 &alloc1_addr,
551 alloc1_size,
552 VM_FLAGS_ANYWHERE);
553 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
554 assertf(alloc1_addr == expected_addr, "alloc1_addr = 0x%lx expected 0x%lx", alloc1_addr, expected_addr);
555 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc1_addr);
556 expected_addr += alloc1_size;
557
558 alloc2_addr = 0;
559 alloc2_size = 3 * FOURK_PAGE_SIZE;
560 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc2_addr, alloc2_size);
561 kr = vm_allocate_external(test_map,
562 &alloc2_addr,
563 alloc2_size,
564 VM_FLAGS_ANYWHERE);
565 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
566 assertf(alloc2_addr == expected_addr, "alloc2_addr = 0x%lx expected 0x%lx", alloc2_addr, expected_addr);
567 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc2_addr);
568 expected_addr += alloc2_size;
569
570 alloc3_addr = 0;
571 alloc3_size = 18 * FOURK_PAGE_SIZE;
572 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc3_addr, alloc3_size);
573 kr = vm_allocate_external(test_map,
574 &alloc3_addr,
575 alloc3_size,
576 VM_FLAGS_ANYWHERE);
577 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
578 assertf(alloc3_addr == expected_addr, "alloc3_addr = 0x%lx expected 0x%lx\n", alloc3_addr, expected_addr);
579 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc3_addr);
580 expected_addr += alloc3_size;
581
582 alloc4_addr = 0;
583 alloc4_size = 1 * FOURK_PAGE_SIZE;
584 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc4_addr, alloc4_size);
585 kr = vm_allocate_external(test_map,
586 &alloc4_addr,
587 alloc4_size,
588 VM_FLAGS_ANYWHERE);
589 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
590 assertf(alloc4_addr == expected_addr, "alloc4_addr = 0x%lx expected 0x%lx", alloc4_addr, expected_addr);
591 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc3_addr);
592 expected_addr += alloc4_size;
593
594 printf("VM_TEST_4K:%d vm_protect(%p, 0x%lx, 0x%lx, READ)...\n", __LINE__, test_map, alloc2_addr, (1UL * FOURK_PAGE_SIZE));
595 kr = vm_protect(test_map,
596 alloc2_addr,
597 (1UL * FOURK_PAGE_SIZE),
598 FALSE,
599 VM_PROT_READ);
600 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
601
602 for (fault_addr = alloc1_addr;
603 fault_addr < alloc4_addr + alloc4_size + (2 * FOURK_PAGE_SIZE);
604 fault_addr += FOURK_PAGE_SIZE) {
605 printf("VM_TEST_4K:%d write fault at 0x%lx...\n", __LINE__, fault_addr);
606 kr = vm_fault(test_map,
607 fault_addr,
608 VM_PROT_WRITE,
609 FALSE,
610 VM_KERN_MEMORY_NONE,
611 THREAD_UNINT,
612 NULL,
613 0);
614 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
615 if (fault_addr == alloc2_addr) {
616 assertf(kr == KERN_PROTECTION_FAILURE, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_PROTECTION_FAILURE);
617 printf("VM_TEST_4K:%d read fault at 0x%lx...\n", __LINE__, fault_addr);
618 kr = vm_fault(test_map,
619 fault_addr,
620 VM_PROT_READ,
621 FALSE,
622 VM_KERN_MEMORY_NONE,
623 THREAD_UNINT,
624 NULL,
625 0);
626 assertf(kr == KERN_SUCCESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_SUCCESS);
627 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
628 } else if (fault_addr >= alloc4_addr + alloc4_size) {
629 assertf(kr == KERN_INVALID_ADDRESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_INVALID_ADDRESS);
630 } else {
631 assertf(kr == KERN_SUCCESS, "fault_addr = 0x%lx kr = 0x%x expected 0x%x", fault_addr, kr, KERN_SUCCESS);
632 }
633 }
634
635 alloc5_addr = 0;
636 alloc5_size = 7 * FOURK_PAGE_SIZE;
637 printf("VM_TEST_4K:%d vm_allocate(%p, 0x%lx, 0x%lx)...\n", __LINE__, test_map, alloc5_addr, alloc5_size);
638 kr = vm_allocate_external(test_map,
639 &alloc5_addr,
640 alloc5_size,
641 VM_FLAGS_ANYWHERE);
642 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
643 assertf(alloc5_addr == expected_addr, "alloc5_addr = 0x%lx expected 0x%lx", alloc5_addr, expected_addr);
644 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, alloc5_addr);
645 expected_addr += alloc5_size;
646
647 dealloc_addr = vm_map_round_page(alloc5_addr, PAGE_SHIFT);
648 dealloc_addr += FOURK_PAGE_SIZE;
649 printf("VM_TEST_4K:%d vm_deallocate(%p, 0x%lx, 0x%x)...\n", __LINE__, test_map, dealloc_addr, FOURK_PAGE_SIZE);
650 kr = vm_deallocate(test_map, dealloc_addr, FOURK_PAGE_SIZE);
651 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
652 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
653
654 remap_src_addr = vm_map_round_page(alloc3_addr, PAGE_SHIFT);
655 remap_src_addr += FOURK_PAGE_SIZE;
656 remap_src_size = 2 * FOURK_PAGE_SIZE;
657 remap_dst_addr = 0;
658 printf("VM_TEST_4K:%d vm_remap(%p, 0x%lx, 0x%lx, 0x%lx, copy=0)...\n", __LINE__, test_map, remap_dst_addr, remap_src_size, remap_src_addr);
659 kr = vm_remap_external(test_map,
660 &remap_dst_addr,
661 remap_src_size,
662 0, /* mask */
663 VM_FLAGS_ANYWHERE,
664 test_map,
665 remap_src_addr,
666 FALSE, /* copy */
667 &cur_prot,
668 &max_prot,
669 VM_INHERIT_DEFAULT);
670 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
671 assertf(remap_dst_addr == expected_addr, "remap_dst_addr = 0x%lx expected 0x%lx", remap_dst_addr, expected_addr);
672 printf("VM_TEST_4K:%d -> 0x%lx\n", __LINE__, remap_dst_addr);
673 expected_addr += remap_src_size;
674
675 for (fault_addr = remap_dst_addr;
676 fault_addr < remap_dst_addr + remap_src_size;
677 fault_addr += 4096) {
678 printf("VM_TEST_4K:%d write fault at 0x%lx...\n", __LINE__, fault_addr);
679 kr = vm_fault(test_map,
680 fault_addr,
681 VM_PROT_WRITE,
682 FALSE,
683 VM_KERN_MEMORY_NONE,
684 THREAD_UNINT,
685 NULL,
686 0);
687 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
688 printf("VM_TEST_4K:%d -> 0x%x\n", __LINE__, kr);
689 }
690
691 printf("VM_TEST_4K:\n");
692 remap_dst_addr = 0;
693 remap_src_addr = alloc3_addr + 0xc000;
694 remap_src_size = 0x5000;
695 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=0) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
696 kr = vm_remap_external(kernel_map,
697 &remap_dst_addr,
698 remap_src_size,
699 0, /* mask */
700 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
701 test_map,
702 remap_src_addr,
703 FALSE, /* copy */
704 &cur_prot,
705 &max_prot,
706 VM_INHERIT_DEFAULT);
707 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
708 printf("VM_TEST_4K: -> remapped (shared) in map %p at addr 0x%lx\n", kernel_map, remap_dst_addr);
709
710 printf("VM_TEST_4K:\n");
711 remap_dst_addr = 0;
712 remap_src_addr = alloc3_addr + 0xc000;
713 remap_src_size = 0x5000;
714 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=1) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
715 kr = vm_remap_external(kernel_map,
716 &remap_dst_addr,
717 remap_src_size,
718 0, /* mask */
719 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
720 test_map,
721 remap_src_addr,
722 TRUE, /* copy */
723 &cur_prot,
724 &max_prot,
725 VM_INHERIT_DEFAULT);
726 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
727 printf("VM_TEST_4K: -> remapped (COW) in map %p at addr 0x%lx\n", kernel_map, remap_dst_addr);
728
729 printf("VM_TEST_4K:\n");
730 saved_debug4k_panic_on_misaligned_sharing = debug4k_panic_on_misaligned_sharing;
731 debug4k_panic_on_misaligned_sharing = 0;
732 remap_dst_addr = 0;
733 remap_src_addr = alloc1_addr;
734 remap_src_size = alloc1_size + alloc2_size;
735 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=0) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
736 kr = vm_remap_external(kernel_map,
737 &remap_dst_addr,
738 remap_src_size,
739 0, /* mask */
740 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
741 test_map,
742 remap_src_addr,
743 FALSE, /* copy */
744 &cur_prot,
745 &max_prot,
746 VM_INHERIT_DEFAULT);
747 assertf(kr != KERN_SUCCESS, "kr = 0x%x", kr);
748 printf("VM_TEST_4K: -> remap (SHARED) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
749 debug4k_panic_on_misaligned_sharing = saved_debug4k_panic_on_misaligned_sharing;
750
751 printf("VM_TEST_4K:\n");
752 remap_dst_addr = 0;
753 remap_src_addr = alloc1_addr;
754 remap_src_size = alloc1_size + alloc2_size;
755 printf("VM_TEST_4K: vm_remap(%p, 0x%lx, 0x%lx, %p, copy=1) from 4K to 16K\n", test_map, remap_src_addr, remap_src_size, kernel_map);
756 kr = vm_remap_external(kernel_map,
757 &remap_dst_addr,
758 remap_src_size,
759 0, /* mask */
760 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
761 test_map,
762 remap_src_addr,
763 TRUE, /* copy */
764 &cur_prot,
765 &max_prot,
766 VM_INHERIT_DEFAULT);
767 #if 000
768 assertf(kr != KERN_SUCCESS, "kr = 0x%x", kr);
769 printf("VM_TEST_4K: -> remap (COPY) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
770 #else /* 000 */
771 assertf(kr == KERN_SUCCESS, "kr = 0x%x", kr);
772 printf("VM_TEST_4K: -> remap (COPY) in map %p at addr 0x%lx kr=0x%x\n", kernel_map, remap_dst_addr, kr);
773 #endif /* 000 */
774
775
776 #if 00
777 printf("VM_TEST_4K:%d vm_map_remove(%p, 0x%llx, 0x%llx)...\n", __LINE__, test_map, test_map->min_offset, test_map->max_offset);
778 vm_map_remove(test_map, test_map->min_offset, test_map->max_offset);
779 #endif
780
781 printf("VM_TEST_4K: PASS\n\n\n\n");
782 }
783 #endif /* PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT */
784
785 #if MACH_ASSERT
786 static void
vm_test_map_copy_adjust_to_target_one(vm_map_copy_t copy_map,vm_map_t target_map)787 vm_test_map_copy_adjust_to_target_one(
788 vm_map_copy_t copy_map,
789 vm_map_t target_map)
790 {
791 kern_return_t kr;
792 vm_map_copy_t target_copy;
793 vm_map_offset_t overmap_start, overmap_end, trimmed_start;
794
795 target_copy = VM_MAP_COPY_NULL;
796 /* size is 2 (4k) pages but range covers 3 pages */
797 kr = vm_map_copy_adjust_to_target(copy_map,
798 0x0 + 0xfff,
799 0x1002,
800 target_map,
801 FALSE,
802 &target_copy,
803 &overmap_start,
804 &overmap_end,
805 &trimmed_start);
806 assert(kr == KERN_SUCCESS);
807 assert(overmap_start == 0);
808 assert(overmap_end == 0);
809 assert(trimmed_start == 0);
810 assertf(target_copy->size == 0x3000,
811 "target_copy %p size 0x%llx\n",
812 target_copy, (uint64_t)target_copy->size);
813 vm_map_copy_discard(target_copy);
814
815 /* 1. adjust_to_target() for bad offset -> error */
816 /* 2. adjust_to_target() for bad size -> error */
817 /* 3. adjust_to_target() for the whole thing -> unchanged */
818 /* 4. adjust_to_target() to trim start by less than 1 page */
819 /* 5. adjust_to_target() to trim end by less than 1 page */
820 /* 6. adjust_to_target() to trim start and end by less than 1 page */
821 /* 7. adjust_to_target() to trim start by more than 1 page */
822 /* 8. adjust_to_target() to trim end by more than 1 page */
823 /* 9. adjust_to_target() to trim start and end by more than 1 page */
824 /* 10. adjust_to_target() to trim start by more than 1 entry */
825 /* 11. adjust_to_target() to trim start by more than 1 entry */
826 /* 12. adjust_to_target() to trim start and end by more than 1 entry */
827 /* 13. adjust_to_target() to trim start and end down to 1 entry */
828 }
829
830 static void
vm_test_map_copy_adjust_to_target(void)831 vm_test_map_copy_adjust_to_target(void)
832 {
833 kern_return_t kr;
834 vm_map_t map4k, map16k;
835 vm_object_t obj1, obj2, obj3, obj4;
836 vm_map_offset_t addr4k, addr16k;
837 vm_map_size_t size4k, size16k;
838 vm_map_copy_t copy4k, copy16k;
839 vm_prot_t curprot, maxprot;
840
841 /* create a 4k map */
842 map4k = vm_map_create_options(PMAP_NULL, 0, (uint32_t)-1,
843 VM_MAP_CREATE_PAGEABLE);
844 vm_map_set_page_shift(map4k, 12);
845
846 /* create a 16k map */
847 map16k = vm_map_create_options(PMAP_NULL, 0, (uint32_t)-1,
848 VM_MAP_CREATE_PAGEABLE);
849 vm_map_set_page_shift(map16k, 14);
850
851 /* create 4 VM objects */
852 obj1 = vm_object_allocate(0x100000);
853 obj2 = vm_object_allocate(0x100000);
854 obj3 = vm_object_allocate(0x100000);
855 obj4 = vm_object_allocate(0x100000);
856
857 /* map objects in 4k map */
858 vm_object_reference(obj1);
859 addr4k = 0x1000;
860 size4k = 0x3000;
861 kr = vm_map_enter(map4k, &addr4k, size4k, 0, VM_FLAGS_ANYWHERE,
862 VM_MAP_KERNEL_FLAGS_DATA, 0, obj1, 0,
863 FALSE, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
864 VM_INHERIT_DEFAULT);
865 assert(kr == KERN_SUCCESS);
866 assert(addr4k == 0x1000);
867
868 /* map objects in 16k map */
869 vm_object_reference(obj1);
870 addr16k = 0x4000;
871 size16k = 0x8000;
872 kr = vm_map_enter(map16k, &addr16k, size16k, 0, VM_FLAGS_ANYWHERE,
873 VM_MAP_KERNEL_FLAGS_DATA, 0, obj1, 0,
874 FALSE, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
875 VM_INHERIT_DEFAULT);
876 assert(kr == KERN_SUCCESS);
877 assert(addr16k == 0x4000);
878
879 /* test for <rdar://60959809> */
880 ipc_port_t mem_entry;
881 memory_object_size_t mem_entry_size;
882 mach_vm_size_t map_size;
883 mem_entry_size = 0x1002;
884 mem_entry = IPC_PORT_NULL;
885 kr = mach_make_memory_entry_64(map16k, &mem_entry_size, addr16k + 0x2fff,
886 MAP_MEM_VM_SHARE | MAP_MEM_USE_DATA_ADDR | VM_PROT_READ,
887 &mem_entry, IPC_PORT_NULL);
888 assertf(kr == KERN_SUCCESS, "kr 0x%x\n", kr);
889 assertf(mem_entry_size == 0x5001, "mem_entry_size 0x%llx\n", (uint64_t) mem_entry_size);
890 map_size = 0;
891 kr = mach_memory_entry_map_size(mem_entry, map4k, 0, 0x1002, &map_size);
892 assertf(kr == KERN_SUCCESS, "kr 0x%x\n", kr);
893 assertf(map_size == 0x3000, "mem_entry %p map_size 0x%llx\n", mem_entry, (uint64_t)map_size);
894 mach_memory_entry_port_release(mem_entry);
895
896 /* create 4k copy map */
897 curprot = VM_PROT_NONE;
898 maxprot = VM_PROT_NONE;
899 kr = vm_map_copy_extract(map4k, addr4k, 0x3000,
900 FALSE, ©4k, &curprot, &maxprot,
901 VM_INHERIT_DEFAULT, VM_MAP_KERNEL_FLAGS_NONE);
902 assert(kr == KERN_SUCCESS);
903 assert(copy4k->size == 0x3000);
904
905 /* create 16k copy map */
906 curprot = VM_PROT_NONE;
907 maxprot = VM_PROT_NONE;
908 kr = vm_map_copy_extract(map16k, addr16k, 0x4000,
909 FALSE, ©16k, &curprot, &maxprot,
910 VM_INHERIT_DEFAULT, VM_MAP_KERNEL_FLAGS_NONE);
911 assert(kr == KERN_SUCCESS);
912 assert(copy16k->size == 0x4000);
913
914 /* test each combination */
915 // vm_test_map_copy_adjust_to_target_one(copy4k, map4k);
916 // vm_test_map_copy_adjust_to_target_one(copy16k, map16k);
917 // vm_test_map_copy_adjust_to_target_one(copy4k, map16k);
918 vm_test_map_copy_adjust_to_target_one(copy16k, map4k);
919
920 /* assert 1 ref on 4k map */
921 assert(os_ref_get_count_raw(&map4k->map_refcnt) == 1);
922 /* release 4k map */
923 vm_map_deallocate(map4k);
924 /* assert 1 ref on 16k map */
925 assert(os_ref_get_count_raw(&map16k->map_refcnt) == 1);
926 /* release 16k map */
927 vm_map_deallocate(map16k);
928 /* deallocate copy maps */
929 vm_map_copy_discard(copy4k);
930 vm_map_copy_discard(copy16k);
931 /* assert 1 ref on all VM objects */
932 assert(obj1->ref_count == 1);
933 assert(obj2->ref_count == 1);
934 assert(obj3->ref_count == 1);
935 assert(obj4->ref_count == 1);
936 /* release all VM objects */
937 vm_object_deallocate(obj1);
938 vm_object_deallocate(obj2);
939 vm_object_deallocate(obj3);
940 vm_object_deallocate(obj4);
941 }
942 #endif /* MACH_ASSERT */
943
944 #if __arm64__ && !KASAN
945 __attribute__((noinline))
946 static void
vm_test_per_mapping_internal_accounting(void)947 vm_test_per_mapping_internal_accounting(void)
948 {
949 ledger_t ledger;
950 pmap_t user_pmap;
951 vm_map_t user_map;
952 kern_return_t kr;
953 ledger_amount_t balance;
954 mach_vm_address_t user_addr, user_remap;
955 vm_map_offset_t device_addr;
956 mach_vm_size_t user_size;
957 vm_prot_t cur_prot, max_prot;
958 upl_size_t upl_size;
959 upl_t upl;
960 unsigned int upl_count;
961 upl_control_flags_t upl_flags;
962 upl_page_info_t *pl;
963 ppnum_t ppnum;
964 vm_object_t device_object;
965 vm_map_offset_t map_start, map_end;
966 int pmap_flags;
967
968 pmap_flags = 0;
969 if (sizeof(vm_map_offset_t) == 4) {
970 map_start = 0x100000000ULL;
971 map_end = 0x200000000ULL;
972 pmap_flags |= PMAP_CREATE_64BIT;
973 } else {
974 map_start = 0x10000000;
975 map_end = 0x20000000;
976 }
977 /* create a user address space */
978 ledger = ledger_instantiate(task_ledger_template,
979 LEDGER_CREATE_ACTIVE_ENTRIES);
980 assert(ledger);
981 user_pmap = pmap_create_options(ledger, 0, pmap_flags);
982 assert(user_pmap);
983 user_map = vm_map_create(user_pmap,
984 map_start,
985 map_end,
986 TRUE);
987 assert(user_map);
988 /* check ledger */
989 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
990 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
991 assertf(balance == 0, "balance=0x%llx", balance);
992 /* allocate 1 page in that address space */
993 user_addr = 0;
994 user_size = PAGE_SIZE;
995 kr = mach_vm_allocate(user_map,
996 &user_addr,
997 user_size,
998 VM_FLAGS_ANYWHERE);
999 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1000 /* check ledger */
1001 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1002 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1003 assertf(balance == 0, "balance=0x%llx", balance);
1004 /* remap the original mapping */
1005 user_remap = 0;
1006 kr = mach_vm_remap(user_map,
1007 &user_remap,
1008 PAGE_SIZE,
1009 0,
1010 VM_FLAGS_ANYWHERE,
1011 user_map,
1012 user_addr,
1013 FALSE, /* copy */
1014 &cur_prot,
1015 &max_prot,
1016 VM_INHERIT_DEFAULT);
1017 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1018 /* check ledger */
1019 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1020 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1021 assertf(balance == 0, "balance=0x%llx", balance);
1022 /* create a UPL from the original mapping */
1023 upl_size = PAGE_SIZE;
1024 upl = NULL;
1025 upl_count = 0;
1026 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1027 kr = vm_map_create_upl(user_map,
1028 (vm_map_offset_t)user_addr,
1029 &upl_size,
1030 &upl,
1031 NULL,
1032 &upl_count,
1033 &upl_flags,
1034 VM_KERN_MEMORY_DIAG);
1035 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1036 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
1037 assert(upl_page_present(pl, 0));
1038 ppnum = upl_phys_page(pl, 0);
1039 /* check ledger */
1040 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1041 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1042 assertf(balance == 0, "balance=0x%llx", balance);
1043 device_object = vm_object_allocate(PAGE_SIZE);
1044 assert(device_object);
1045 device_object->private = TRUE;
1046 device_object->phys_contiguous = TRUE;
1047 kr = vm_object_populate_with_private(device_object, 0,
1048 ppnum, PAGE_SIZE);
1049 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1050
1051 /* check ledger */
1052 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1053 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1054 assertf(balance == 0, "balance=0x%llx", balance);
1055 /* deallocate the original mapping */
1056 kr = mach_vm_deallocate(user_map, user_addr, PAGE_SIZE);
1057 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1058 /* map the device_object in the kernel */
1059 device_addr = 0;
1060 vm_object_reference(device_object);
1061 kr = vm_map_enter(kernel_map,
1062 &device_addr,
1063 PAGE_SIZE,
1064 0,
1065 VM_FLAGS_ANYWHERE,
1066 VM_MAP_KERNEL_FLAGS_DATA,
1067 0,
1068 device_object,
1069 0,
1070 FALSE, /* copy */
1071 VM_PROT_DEFAULT,
1072 VM_PROT_DEFAULT,
1073 VM_INHERIT_NONE);
1074 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1075 /* access the device pager mapping */
1076 *(char *)device_addr = 'x';
1077 printf("%s:%d 0x%llx: 0x%x\n", __FUNCTION__, __LINE__, (uint64_t)device_addr, *(uint32_t *)device_addr);
1078 /* check ledger */
1079 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1080 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1081 assertf(balance == 0, "balance=0x%llx", balance);
1082 /* fault in the remap addr */
1083 kr = vm_fault(user_map, (vm_map_offset_t)user_remap, VM_PROT_READ,
1084 FALSE, 0, TRUE, NULL, 0);
1085 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1086 /* check ledger */
1087 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1088 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1089 assertf(balance == PAGE_SIZE, "balance=0x%llx", balance);
1090 /* deallocate remapping */
1091 kr = mach_vm_deallocate(user_map, user_remap, PAGE_SIZE);
1092 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1093 /* check ledger */
1094 kr = ledger_get_balance(ledger, task_ledgers.internal, &balance);
1095 assertf(kr == KERN_SUCCESS, "kr=0x%x", kr);
1096 assertf(balance == 0, "balance=0x%llx", balance);
1097 /* TODO: cleanup... */
1098 printf("%s:%d PASS\n", __FUNCTION__, __LINE__);
1099 }
1100 #endif /* __arm64__ && !KASAN */
1101
1102 static void
vm_test_kernel_tag_accounting_kma(kma_flags_t base,kma_flags_t bit)1103 vm_test_kernel_tag_accounting_kma(kma_flags_t base, kma_flags_t bit)
1104 {
1105 vm_tag_t tag = VM_KERN_MEMORY_REASON; /* unused during POST */
1106 uint64_t init_size = vm_tag_get_size(tag);
1107 uint64_t final_size = init_size + PAGE_SIZE;
1108 vm_address_t address;
1109 kern_return_t kr;
1110
1111 /*
1112 * Test the matrix of:
1113 * - born with or without bit
1114 * - bit flipped or not
1115 * - dies with or without bit
1116 */
1117 for (uint32_t i = 0; i < 4; i++) {
1118 kma_flags_t flags1 = base | ((i & 1) ? bit : KMA_NONE);
1119 kma_flags_t flags2 = base | ((i & 2) ? bit : KMA_NONE);
1120
1121 kr = kmem_alloc(kernel_map, &address, PAGE_SIZE, flags1, tag);
1122 assert3u(kr, ==, KERN_SUCCESS);
1123
1124 if (flags1 & (KMA_VAONLY | KMA_PAGEABLE)) {
1125 assert3u(init_size, ==, vm_tag_get_size(tag));
1126 } else {
1127 assert3u(final_size, ==, vm_tag_get_size(tag));
1128 }
1129
1130 if ((flags1 ^ flags2) == KMA_VAONLY) {
1131 if (flags1 & KMA_VAONLY) {
1132 kernel_memory_populate(address, PAGE_SIZE,
1133 KMA_KOBJECT | KMA_NOFAIL, tag);
1134 } else {
1135 kernel_memory_depopulate(address, PAGE_SIZE,
1136 KMA_KOBJECT, tag);
1137 }
1138 }
1139
1140 if ((flags1 ^ flags2) == KMA_PAGEABLE) {
1141 if (flags1 & KMA_PAGEABLE) {
1142 kr = vm_map_wire_kernel(kernel_map,
1143 address, address + PAGE_SIZE,
1144 VM_PROT_DEFAULT, tag, false);
1145 assert3u(kr, ==, KERN_SUCCESS);
1146 } else {
1147 kr = vm_map_unwire(kernel_map,
1148 address, address + PAGE_SIZE, false);
1149 assert3u(kr, ==, KERN_SUCCESS);
1150 }
1151 }
1152
1153 if (flags2 & (KMA_VAONLY | KMA_PAGEABLE)) {
1154 assert3u(init_size, ==, vm_tag_get_size(tag));
1155 } else {
1156 assert3u(final_size, ==, vm_tag_get_size(tag));
1157 }
1158
1159 kmem_free(kernel_map, address, PAGE_SIZE);
1160 assert3u(init_size, ==, vm_tag_get_size(tag));
1161 }
1162 }
1163
1164 __attribute__((noinline))
1165 static void
vm_test_kernel_tag_accounting(void)1166 vm_test_kernel_tag_accounting(void)
1167 {
1168 printf("%s: test running\n", __func__);
1169
1170 printf("%s: account (KMA_KOBJECT + populate)...\n", __func__);
1171 vm_test_kernel_tag_accounting_kma(KMA_KOBJECT, KMA_VAONLY);
1172 printf("%s: PASS\n", __func__);
1173
1174 printf("%s: account (regular object + wiring)...\n", __func__);
1175 vm_test_kernel_tag_accounting_kma(KMA_NONE, KMA_PAGEABLE);
1176 printf("%s: PASS\n", __func__);
1177
1178 printf("%s: test passed\n", __func__);
1179
1180 #undef if_bit
1181 }
1182
1183 __attribute__((noinline))
1184 static void
vm_test_collapse_overflow(void)1185 vm_test_collapse_overflow(void)
1186 {
1187 vm_object_t object, backing_object;
1188 vm_object_size_t size;
1189 vm_page_t m;
1190
1191 /* create an object for which (int)(size>>PAGE_SHIFT) = 0 */
1192 size = 0x400000000000ULL;
1193 assert((int)(size >> PAGE_SHIFT) == 0);
1194 backing_object = vm_object_allocate(size);
1195 assert(backing_object);
1196 vm_object_reference(backing_object);
1197 /* insert a page */
1198 m = VM_PAGE_NULL;
1199 while (m == VM_PAGE_NULL) {
1200 m = vm_page_grab();
1201 if (m == VM_PAGE_NULL) {
1202 VM_PAGE_WAIT();
1203 }
1204 }
1205 assert(m);
1206 vm_object_lock(backing_object);
1207 vm_page_insert(m, backing_object, 0);
1208 vm_object_unlock(backing_object);
1209 /* make it back another object */
1210 object = vm_object_allocate(size);
1211 assert(object);
1212 vm_object_reference(object);
1213 object->shadow = backing_object;
1214 vm_object_reference(backing_object);
1215 /* trigger a bypass */
1216 vm_object_lock(object);
1217 vm_object_collapse(object, 0, TRUE);
1218 if (object->shadow != backing_object) {
1219 panic("%s:%d FAIL\n", __FUNCTION__, __LINE__);
1220 }
1221 vm_object_unlock(object);
1222 /* cleanup */
1223 vm_object_deallocate(object);
1224 vm_object_deallocate(backing_object);
1225
1226 printf("%s:%d PASS\n", __FUNCTION__, __LINE__);
1227 }
1228
1229 __attribute__((noinline))
1230 static void
vm_test_physical_size_overflow(void)1231 vm_test_physical_size_overflow(void)
1232 {
1233 vm_map_address_t start;
1234 mach_vm_size_t size;
1235 kern_return_t kr;
1236 mach_vm_size_t phys_size;
1237 bool fail;
1238 int failures = 0;
1239
1240 /* size == 0 */
1241 start = 0x100000;
1242 size = 0x0;
1243 kr = vm_map_range_physical_size(kernel_map,
1244 start,
1245 size,
1246 &phys_size);
1247 fail = (kr != KERN_SUCCESS || phys_size != 0);
1248 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1249 __FUNCTION__, __LINE__,
1250 (fail ? "FAIL" : "PASS"),
1251 (uint64_t)start, size, kr, phys_size);
1252 failures += fail;
1253
1254 /* plain wraparound */
1255 start = 0x100000;
1256 size = 0xffffffffffffffff - 0x10000;
1257 kr = vm_map_range_physical_size(kernel_map,
1258 start,
1259 size,
1260 &phys_size);
1261 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1262 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1263 __FUNCTION__, __LINE__,
1264 (fail ? "FAIL" : "PASS"),
1265 (uint64_t)start, size, kr, phys_size);
1266 failures += fail;
1267
1268 /* wraparound after rounding */
1269 start = 0xffffffffffffff00;
1270 size = 0xf0;
1271 kr = vm_map_range_physical_size(kernel_map,
1272 start,
1273 size,
1274 &phys_size);
1275 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1276 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1277 __FUNCTION__, __LINE__,
1278 (fail ? "FAIL" : "PASS"),
1279 (uint64_t)start, size, kr, phys_size);
1280 failures += fail;
1281
1282 /* wraparound to start after rounding */
1283 start = 0x100000;
1284 size = 0xffffffffffffffff;
1285 kr = vm_map_range_physical_size(kernel_map,
1286 start,
1287 size,
1288 &phys_size);
1289 fail = (kr != KERN_INVALID_ARGUMENT || phys_size != 0);
1290 printf("%s:%d %s start=0x%llx size=0x%llx -> kr=%d phys_size=0x%llx\n",
1291 __FUNCTION__, __LINE__,
1292 (fail ? "FAIL" : "PASS"),
1293 (uint64_t)start, size, kr, phys_size);
1294 failures += fail;
1295
1296 if (failures) {
1297 panic("%s: FAIL (failures=%d)", __FUNCTION__, failures);
1298 }
1299 printf("%s: PASS\n", __FUNCTION__);
1300 }
1301
1302 boolean_t vm_tests_in_progress = FALSE;
1303
1304 kern_return_t
vm_tests(void)1305 vm_tests(void)
1306 {
1307 vm_tests_in_progress = TRUE;
1308
1309 vm_test_collapse_compressor();
1310 vm_test_wire_and_extract();
1311 vm_test_page_wire_overflow_panic();
1312 vm_test_kernel_object_fault();
1313 vm_test_device_pager_transpose();
1314 #if MACH_ASSERT
1315 vm_test_map_copy_adjust_to_target();
1316 #endif /* MACH_ASSERT */
1317 #if PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT
1318 vm_test_4k();
1319 #endif /* PMAP_CREATE_FORCE_4K_PAGES && MACH_ASSERT */
1320 #if __arm64__ && !KASAN
1321 vm_test_per_mapping_internal_accounting();
1322 #endif /* __arm64__ && !KASAN */
1323 vm_test_kernel_tag_accounting();
1324 vm_test_collapse_overflow();
1325 vm_test_physical_size_overflow();
1326
1327 vm_tests_in_progress = FALSE;
1328
1329 return KERN_SUCCESS;
1330 }
1331
1332 /*
1333 * Checks that vm_map_delete() can deal with map unaligned entries.
1334 * rdar://88969652
1335 */
1336 static int
vm_map_non_aligned_test(__unused int64_t in,int64_t * out)1337 vm_map_non_aligned_test(__unused int64_t in, int64_t *out)
1338 {
1339 vm_map_t map = current_map();
1340 mach_vm_size_t size = 2 * VM_MAP_PAGE_SIZE(map);
1341 mach_vm_address_t addr;
1342 vm_map_entry_t entry;
1343 kern_return_t kr;
1344
1345 if (VM_MAP_PAGE_SHIFT(map) > PAGE_SHIFT) {
1346 kr = mach_vm_allocate(map, &addr, size, VM_FLAGS_ANYWHERE);
1347 if (kr != KERN_SUCCESS) {
1348 return ENOMEM;
1349 }
1350
1351 vm_map_lock(map);
1352 if (!vm_map_lookup_entry(map, addr, &entry)) {
1353 panic("couldn't find the entry we just made: "
1354 "map:%p addr:0x%0llx", map, addr);
1355 }
1356
1357 /*
1358 * Now break the entry into:
1359 * 2 * 4k
1360 * 2 * 4k
1361 * 1 * 16k
1362 */
1363 vm_map_clip_end(map, entry, addr + VM_MAP_PAGE_SIZE(map));
1364 entry->map_aligned = FALSE;
1365 vm_map_clip_end(map, entry, addr + PAGE_SIZE * 2);
1366 vm_map_unlock(map);
1367
1368 kr = mach_vm_deallocate(map, addr, size);
1369 assert(kr == KERN_SUCCESS);
1370 }
1371
1372 *out = 1;
1373 return 0;
1374 }
1375 SYSCTL_TEST_REGISTER(vm_map_non_aligned, vm_map_non_aligned_test);
1376