1 /*
2 * Copyright (c) 2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_page.h>
30 #include <vm/pmap.h>
31 #include <kern/ledger.h>
32 #include <kern/thread.h>
33 #if defined(__arm64__)
34 #include <pexpert/arm64/board_config.h>
35 #if CONFIG_SPTM
36 #include <arm64/sptm/pmap/pmap_pt_geometry.h>
37 #else /* CONFIG_SPTM */
38 #include <arm/pmap/pmap_pt_geometry.h>
39 #endif /* CONFIG_SPTM */
40 #endif /* defined(__arm64__) */
41 #include <vm/vm_map.h>
42
43 extern void read_random(void* buffer, u_int numBytes);
44
45 extern ledger_template_t task_ledger_template;
46
47 extern boolean_t arm_force_fast_fault(ppnum_t, vm_prot_t, int, void*);
48 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
49
50 kern_return_t test_pmap_enter_disconnect(unsigned int num_loops);
51 kern_return_t test_pmap_compress_remove(unsigned int num_loops);
52 kern_return_t test_pmap_exec_remove(unsigned int num_loops);
53 kern_return_t test_pmap_nesting(unsigned int num_loops);
54 kern_return_t test_pmap_iommu_disconnect(void);
55 kern_return_t test_pmap_extended(void);
56 void test_pmap_call_overhead(unsigned int num_loops);
57 uint64_t test_pmap_page_protect_overhead(unsigned int num_loops, unsigned int num_aliases);
58 #if CONFIG_SPTM
59 kern_return_t test_pmap_huge_pv_list(unsigned int num_loops, unsigned int num_mappings);
60 #endif
61
62 #define PMAP_TEST_VA (0xDEADULL << PAGE_SHIFT)
63
64 typedef struct {
65 pmap_t pmap;
66 vm_map_address_t va;
67 processor_t proc;
68 ppnum_t pn;
69 volatile boolean_t stop;
70 } pmap_test_thread_args;
71
72
73 /**
74 * Helper for creating a new pmap to be used for testing.
75 *
76 * @param flags Flags to pass to pmap_create_options()
77 *
78 * @return The newly-allocated pmap, or NULL if allocation fails.
79 */
80 static pmap_t
pmap_create_wrapper(unsigned int flags)81 pmap_create_wrapper(unsigned int flags)
82 {
83 pmap_t new_pmap = NULL;
84 ledger_t ledger;
85 assert(task_ledger_template != NULL);
86 if ((ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES)) == NULL) {
87 return NULL;
88 }
89 new_pmap = pmap_create_options(ledger, 0, flags);
90 ledger_dereference(ledger);
91 return new_pmap;
92 }
93
94 /**
95 * Helper for allocating a wired VM page to be used for testing.
96 *
97 * @note The allocated page will be wired with the VM_KERN_MEMORY_PTE tag,
98 * which will attribute the page to the pmap module.
99 *
100 * @return the newly-allocated vm_page_t, or NULL if allocation fails.
101 */
102 static vm_page_t
pmap_test_alloc_vm_page(void)103 pmap_test_alloc_vm_page(void)
104 {
105 vm_page_t m = vm_page_grab();
106 if (m != VM_PAGE_NULL) {
107 vm_page_lock_queues();
108 vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
109 vm_page_unlock_queues();
110 }
111 return m;
112 }
113
114 /**
115 * Helper for freeing a VM page previously allocated by pmap_test_alloc_vm_page().
116 *
117 * @param m The page to free. This may be NULL, in which case this function will
118 * do nothing.
119 */
120 static void
pmap_test_free_vm_page(vm_page_t m)121 pmap_test_free_vm_page(vm_page_t m)
122 {
123 if (m != VM_PAGE_NULL) {
124 vm_page_lock_queues();
125 vm_page_free(m);
126 vm_page_unlock_queues();
127 }
128 }
129
130 static void
pmap_disconnect_thread(void * arg,wait_result_t __unused wres)131 pmap_disconnect_thread(void *arg, wait_result_t __unused wres)
132 {
133 pmap_test_thread_args *args = arg;
134 do {
135 pmap_disconnect(args->pn);
136 } while (!args->stop);
137 thread_wakeup((event_t)args);
138 }
139
140 kern_return_t
test_pmap_enter_disconnect(unsigned int num_loops)141 test_pmap_enter_disconnect(unsigned int num_loops)
142 {
143 kern_return_t kr = KERN_SUCCESS;
144 thread_t disconnect_thread;
145 pmap_t new_pmap = pmap_create_wrapper(0);
146 if (new_pmap == NULL) {
147 return KERN_FAILURE;
148 }
149 vm_page_t m = pmap_test_alloc_vm_page();
150 if (m == VM_PAGE_NULL) {
151 pmap_destroy(new_pmap);
152 return KERN_FAILURE;
153 }
154 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
155 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .pn = phys_page};
156 kern_return_t res = kernel_thread_start_priority(pmap_disconnect_thread,
157 &args, thread_kern_get_pri(current_thread()), &disconnect_thread);
158 if (res) {
159 pmap_destroy(new_pmap);
160 pmap_test_free_vm_page(m);
161 return res;
162 }
163 thread_deallocate(disconnect_thread);
164
165 while (num_loops-- != 0) {
166 kr = pmap_enter(new_pmap, PMAP_TEST_VA, phys_page,
167 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
168 assert(kr == KERN_SUCCESS);
169 }
170
171 assert_wait((event_t)&args, THREAD_UNINT);
172 args.stop = TRUE;
173 thread_block(THREAD_CONTINUE_NULL);
174
175 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
176 pmap_test_free_vm_page(m);
177 pmap_destroy(new_pmap);
178 return KERN_SUCCESS;
179 }
180
181 static void
pmap_remove_thread(void * arg,wait_result_t __unused wres)182 pmap_remove_thread(void *arg, wait_result_t __unused wres)
183 {
184 pmap_test_thread_args *args = arg;
185 do {
186 kern_return_t kr = pmap_enter_options(args->pmap, args->va, args->pn,
187 VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_OPTIONS_INTERNAL, NULL, PMAP_MAPPING_TYPE_INFER);
188 assert(kr == KERN_SUCCESS);
189 pmap_remove(args->pmap, args->va, args->va + PAGE_SIZE);
190 } while (!args->stop);
191 thread_wakeup((event_t)args);
192 }
193
194 /**
195 * Test that a mapping to a physical page can be concurrently removed while
196 * the page is being compressed, without triggering accounting panics.
197 *
198 * @param num_loops The number of test loops to run
199 *
200 * @return KERN_SUCCESS if the test runs to completion, otherwise an
201 * appropriate error code.
202 */
203 kern_return_t
test_pmap_compress_remove(unsigned int num_loops)204 test_pmap_compress_remove(unsigned int num_loops)
205 {
206 thread_t remove_thread;
207 pmap_t new_pmap = pmap_create_wrapper(0);
208 if (new_pmap == NULL) {
209 return KERN_FAILURE;
210 }
211 vm_page_t m = pmap_test_alloc_vm_page();
212 if (m == VM_PAGE_NULL) {
213 pmap_destroy(new_pmap);
214 return KERN_FAILURE;
215 }
216 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
217 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .va = PMAP_TEST_VA, .pn = phys_page};
218 kern_return_t res = kernel_thread_start_priority(pmap_remove_thread,
219 &args, thread_kern_get_pri(current_thread()), &remove_thread);
220 if (res) {
221 pmap_destroy(new_pmap);
222 pmap_test_free_vm_page(m);
223 return res;
224 }
225 thread_deallocate(remove_thread);
226
227 while (num_loops-- != 0) {
228 pmap_disconnect_options(phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
229 }
230
231 assert_wait((event_t)&args, THREAD_UNINT);
232 args.stop = TRUE;
233 thread_block(THREAD_CONTINUE_NULL);
234
235 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
236 pmap_destroy(new_pmap);
237 pmap_test_free_vm_page(m);
238 return KERN_SUCCESS;
239 }
240
241
242 kern_return_t
test_pmap_exec_remove(unsigned int num_loops __unused)243 test_pmap_exec_remove(unsigned int num_loops __unused)
244 {
245 return KERN_NOT_SUPPORTED;
246 }
247
248
249 #if defined(__arm64__)
250
251 static const vm_map_address_t nesting_start = SHARED_REGION_BASE;
252 static const vm_map_address_t nesting_size = 16 * ARM_16K_TT_L2_SIZE;
253
254 static void
pmap_nest_thread(void * arg,wait_result_t __unused wres)255 pmap_nest_thread(void *arg, wait_result_t __unused wres)
256 {
257 const pmap_test_thread_args *args = arg;
258 pmap_t main_pmap = pmap_create_wrapper(0);
259 kern_return_t kr;
260
261 thread_bind(args->proc);
262 thread_block(THREAD_CONTINUE_NULL);
263
264 /**
265 * Exercise nesting and unnesting while bound to the specified CPU (if non-NULL).
266 * The unnesting size here should match the unnesting size used in the first
267 * unnesting step of the main thread, in order to avoid concurrently unnesting
268 * beyond that region and violating the checks against over-unnesting performed
269 * in the main thread.
270 */
271 if (main_pmap != NULL) {
272 kr = pmap_nest(main_pmap, args->pmap, nesting_start, nesting_size);
273 assert(kr == KERN_SUCCESS);
274
275 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
276 assert(kr == KERN_SUCCESS);
277 }
278
279 thread_bind(PROCESSOR_NULL);
280 thread_block(THREAD_CONTINUE_NULL);
281
282 assert_wait((event_t)(uintptr_t)&(args->stop), THREAD_UNINT);
283 if (!args->stop) {
284 thread_block(THREAD_CONTINUE_NULL);
285 } else {
286 clear_wait(current_thread(), THREAD_AWAKENED);
287 }
288
289 /* Unnest all remaining mappings so that we can safely destroy our pmap. */
290 if (main_pmap != NULL) {
291 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
292 assert(kr == KERN_SUCCESS);
293 pmap_destroy(main_pmap);
294 }
295
296 thread_wakeup((event_t)arg);
297 }
298
299 /**
300 * Test that pmap_nest() and pmap_unnest() work correctly when executed concurrently from
301 * multiple threads. Spawn some worker threads at elevated priority and bound to the
302 * same CPU in order to provoke preemption of the nest/unnest operation.
303 *
304 * @param num_loops The number of nest/unnest loops to perform. This should be kept to
305 * a small number because each cycle is expensive and may consume a global shared
306 * region ID.
307 *
308 * @return KERN_SUCCESS if all tests succeed, an appropriate error code otherwise.
309 */
310 kern_return_t
test_pmap_nesting(unsigned int num_loops)311 test_pmap_nesting(unsigned int num_loops)
312 {
313 kern_return_t kr = KERN_SUCCESS;
314
315 vm_page_t m1 = VM_PAGE_NULL, m2 = VM_PAGE_NULL;
316
317 m1 = pmap_test_alloc_vm_page();
318 m2 = pmap_test_alloc_vm_page();
319 if ((m1 == VM_PAGE_NULL) || (m2 == VM_PAGE_NULL)) {
320 kr = KERN_FAILURE;
321 goto test_nesting_cleanup;
322 }
323 const ppnum_t pp1 = VM_PAGE_GET_PHYS_PAGE(m1);
324 const ppnum_t pp2 = VM_PAGE_GET_PHYS_PAGE(m2);
325 for (unsigned int i = 0; (i < num_loops) && (kr == KERN_SUCCESS); i++) {
326 pmap_t nested_pmap = pmap_create_wrapper(0);
327 pmap_t main_pmap = pmap_create_wrapper(0);
328 if ((nested_pmap == NULL) || (main_pmap == NULL)) {
329 pmap_destroy(main_pmap);
330 pmap_destroy(nested_pmap);
331 kr = KERN_FAILURE;
332 break;
333 }
334 pmap_set_nested(nested_pmap);
335 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
336 uint8_t rand;
337 read_random(&rand, sizeof(rand));
338 uint8_t rand_mod = rand % 3;
339 if (rand_mod == 0) {
340 continue;
341 }
342 kr = pmap_enter(nested_pmap, va, (rand_mod == 1) ? pp1 : pp2, VM_PROT_READ,
343 VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
344 assert(kr == KERN_SUCCESS);
345 }
346 kr = pmap_nest(main_pmap, nested_pmap, nesting_start, nesting_size);
347 assert(kr == KERN_SUCCESS);
348
349 /* Validate the initial nest operation produced global mappings within the nested pmap. */
350 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
351 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
352 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
353 if (nested_pte != main_pte) {
354 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx",
355 __func__, nested_pte, main_pte, (unsigned long long)va);
356 }
357 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
358 panic("%s: nested_pte (%p) is not global for va 0x%llx",
359 __func__, nested_pte, (unsigned long long)va);
360 }
361 }
362
363 /* Now kick off various worker threads to concurrently nest and unnest. */
364 const processor_t nest_proc = current_processor();
365 thread_bind(nest_proc);
366 thread_block(THREAD_CONTINUE_NULL);
367
368 /**
369 * Avoid clogging the CPUs with high-priority kernel threads on older devices.
370 * Testing has shown this may provoke a userspace watchdog timeout.
371 */
372 #define TEST_NEST_THREADS 4
373 #if TEST_NEST_THREADS >= MAX_CPUS
374 #undef TEST_NEST_THREADS
375 #define TEST_NEST_THREADS MAX_CPUS - 1
376 #endif
377 thread_t nest_threads[TEST_NEST_THREADS];
378 kern_return_t thread_krs[TEST_NEST_THREADS];
379 pmap_test_thread_args args[TEST_NEST_THREADS];
380 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
381 args[j].pmap = nested_pmap;
382 args[j].stop = FALSE;
383 /**
384 * Spawn the worker threads at various priorities at the high end of the kernel range,
385 * and bind every other thread to the same CPU as this thread to provoke preemption,
386 * while also allowing some threads to run concurrently on other CPUs.
387 */
388 args[j].proc = ((j % 2) ? PROCESSOR_NULL : nest_proc);
389 thread_krs[j] = kernel_thread_start_priority(pmap_nest_thread, &args[j], MAXPRI_KERNEL - (j % 4), &nest_threads[j]);
390 if (thread_krs[j] == KERN_SUCCESS) {
391 thread_set_thread_name(nest_threads[j], "pmap_nest_thread");
392 }
393 }
394
395 /* Unnest the bulk of the nested region and validate that it produced the expected PTE contents. */
396 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
397 assert(kr == KERN_SUCCESS);
398
399 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size - ARM_16K_TT_L2_SIZE); va += PAGE_SIZE) {
400 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
401 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
402
403 if (main_pte != NULL) {
404 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
405 __func__, main_pte, (unsigned long long)va);
406 }
407 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
408 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
409 __func__, nested_pte, (unsigned long long)va);
410 }
411 }
412
413 /* Validate that the prior unnest did not unnest too much. */
414 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
415 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
416 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
417 if (nested_pte != main_pte) {
418 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx following adjacent unnest",
419 __func__, nested_pte, main_pte, (unsigned long long)va);
420 }
421 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
422 panic("%s: nested_pte (%p) is not global for va 0x%llx following adjacent unnest",
423 __func__, nested_pte, (unsigned long long)va);
424 }
425 }
426
427 /* Now unnest the remainder. */
428 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
429 assert(kr == KERN_SUCCESS);
430
431 thread_bind(PROCESSOR_NULL);
432 thread_block(THREAD_CONTINUE_NULL);
433
434 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
435 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
436 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
437
438 if (main_pte != NULL) {
439 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
440 __func__, main_pte, (unsigned long long)va);
441 }
442 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
443 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
444 __func__, nested_pte, (unsigned long long)va);
445 }
446 }
447
448 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
449 if (thread_krs[j] == KERN_SUCCESS) {
450 assert_wait((event_t)&args[j], THREAD_UNINT);
451 args[j].stop = TRUE;
452 thread_wakeup((event_t)(uintptr_t)&(args[j].stop));
453 thread_block(THREAD_CONTINUE_NULL);
454 } else {
455 kr = thread_krs[j];
456 }
457 }
458 pmap_remove(nested_pmap, nesting_start, nesting_start + nesting_size);
459 pmap_destroy(main_pmap);
460 pmap_destroy(nested_pmap);
461 }
462
463 test_nesting_cleanup:
464 pmap_test_free_vm_page(m1);
465 pmap_test_free_vm_page(m2);
466
467 return kr;
468 }
469
470 #else /* defined(__arm64__) */
471
472 kern_return_t
test_pmap_nesting(unsigned int num_loops __unused)473 test_pmap_nesting(unsigned int num_loops __unused)
474 {
475 return KERN_NOT_SUPPORTED;
476 }
477
478 #endif /* defined(__arm64__) */
479
480 kern_return_t
test_pmap_iommu_disconnect(void)481 test_pmap_iommu_disconnect(void)
482 {
483 return KERN_SUCCESS;
484 }
485
486
487 kern_return_t
test_pmap_extended(void)488 test_pmap_extended(void)
489 {
490 #if !CONFIG_SPTM /* SPTM TODO: remove this condition once the SPTM supports 4K and stage-2 mappings */
491 #endif /* !CONFIG_SPTM */
492 return KERN_SUCCESS;
493 }
494
495 void
test_pmap_call_overhead(unsigned int num_loops __unused)496 test_pmap_call_overhead(unsigned int num_loops __unused)
497 {
498 #if defined(__arm64__)
499 pmap_t pmap = current_thread()->map->pmap;
500 for (unsigned int i = 0; i < num_loops; ++i) {
501 pmap_nop(pmap);
502 }
503 #endif
504 }
505
506 uint64_t
test_pmap_page_protect_overhead(unsigned int num_loops __unused,unsigned int num_aliases __unused)507 test_pmap_page_protect_overhead(unsigned int num_loops __unused, unsigned int num_aliases __unused)
508 {
509 uint64_t duration = 0;
510 #if defined(__arm64__)
511 pmap_t new_pmap = pmap_create_wrapper(0);
512 vm_page_t m = pmap_test_alloc_vm_page();
513 kern_return_t kr = KERN_SUCCESS;
514
515 if ((new_pmap == NULL) || (m == VM_PAGE_NULL)) {
516 goto ppo_cleanup;
517 }
518
519 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
520
521 for (unsigned int loop = 0; loop < num_loops; ++loop) {
522 for (unsigned int alias = 0; alias < num_aliases; ++alias) {
523 kr = pmap_enter(new_pmap, PMAP_TEST_VA + (PAGE_SIZE * alias), phys_page,
524 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
525 assert(kr == KERN_SUCCESS);
526 }
527
528 uint64_t start_time = mach_absolute_time();
529
530 pmap_page_protect_options(phys_page, VM_PROT_READ, 0, NULL);
531
532 duration += (mach_absolute_time() - start_time);
533
534 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + (num_aliases * PAGE_SIZE));
535 }
536
537 ppo_cleanup:
538 pmap_test_free_vm_page(m);
539 if (new_pmap != NULL) {
540 pmap_destroy(new_pmap);
541 }
542 #endif
543 return duration;
544 }
545
546 #if CONFIG_SPTM
547
548 typedef struct {
549 pmap_test_thread_args args;
550 unsigned int num_mappings;
551 volatile unsigned int nthreads;
552 thread_call_t panic_callout;
553 } pmap_hugepv_test_thread_args;
554
555 /**
556 * Worker thread that exercises pmap_remove() and pmap_enter() with a huge PV list.
557 * This thread relies on the fact that PV lists are structured with newer PTEs at
558 * the beginning of the list, so it maximizes PV list traversal time by removing
559 * mappings sequentially starting with the beginning VA of the mapping region
560 * (thus the oldest mapping), and then re-entering that removed mapping at the
561 * beginning of the list.
562 *
563 * @param arg Thread argument parameter, actually of type pmap_hugepv_test_thread_args*
564 * @param wres Wait result, currently unused.
565 */
566 static void
hugepv_remove_enter_thread(void * arg,wait_result_t __unused wres)567 hugepv_remove_enter_thread(void *arg, wait_result_t __unused wres)
568 {
569 unsigned int mapping = 0;
570 pmap_hugepv_test_thread_args *args = arg;
571 do {
572 vm_map_address_t va = args->args.va + ((vm_offset_t)mapping << PAGE_SHIFT);
573 pmap_remove(args->args.pmap, va, va + PAGE_SIZE);
574 kern_return_t kr = pmap_enter_options(args->args.pmap, va, args->args.pn,
575 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_OPTIONS_INTERNAL,
576 NULL, PMAP_MAPPING_TYPE_INFER);
577 assert(kr == KERN_SUCCESS);
578 if (++mapping == args->num_mappings) {
579 mapping = 0;
580 }
581 } while (!args->args.stop);
582 /* Ensure the update of nthreads is not speculated ahead of checking the stop flag. */
583 os_atomic_thread_fence(acquire);
584 if (os_atomic_dec(&args->nthreads, relaxed) == 0) {
585 thread_wakeup((event_t)args);
586 }
587 }
588
589 /**
590 * Worker thread to exercise fast-fault behavior with a huge PV list.
591 * This thread first removes permissions from all mappings for the page, which
592 * does not actually remove the mappings but rather clears their AF bit.
593 * It then simulates a fast fault on one random mapping in the list, which
594 * also clears the fast-fault state for the first 64 mappings in the list.
595 *
596 * @param arg Thread argument parameter, actually of type pmap_hugepv_test_thread_args*
597 * @param wres Wait result, currently unused.
598 */
599 static void
hugepv_fast_fault_thread(void * arg,wait_result_t __unused wres)600 hugepv_fast_fault_thread(void *arg, wait_result_t __unused wres)
601 {
602 pmap_hugepv_test_thread_args *args = arg;
603 do {
604 boolean_t success = arm_force_fast_fault(args->args.pn, VM_PROT_NONE, 0, NULL);
605 assert(success);
606 unsigned int rand;
607 read_random(&rand, sizeof(rand));
608 unsigned int mapping = rand % args->num_mappings;
609 arm_fast_fault(args->args.pmap, args->args.va + ((vm_offset_t)mapping << PAGE_SHIFT), VM_PROT_READ, false, FALSE);
610 } while (!args->args.stop);
611 /* Ensure the update of nthreads is not speculated ahead of checking the stop flag. */
612 os_atomic_thread_fence(acquire);
613 if (os_atomic_dec(&args->nthreads, relaxed) == 0) {
614 thread_wakeup((event_t)args);
615 }
616 }
617
618 /**
619 * Worker thread for updating cacheability of a physical page with a huge PV list.
620 * This thread simply twiddles all mappings between write-combined and normal (write-back)
621 * cacheability.
622 *
623 * @param arg Thread argument parameter, actually of type pmap_hugepv_test_thread_args*
624 * @param wres Wait result, currently unused.
625 */
626 static void
hugepv_cache_attr_thread(void * arg,wait_result_t __unused wres)627 hugepv_cache_attr_thread(void *arg, wait_result_t __unused wres)
628 {
629 pmap_hugepv_test_thread_args *args = arg;
630 do {
631 pmap_set_cache_attributes(args->args.pn, VM_WIMG_WCOMB);
632 pmap_set_cache_attributes(args->args.pn, VM_WIMG_DEFAULT);
633 } while (!args->args.stop);
634 /* Ensure the update of nthreads is not speculated ahead of checking the stop flag. */
635 os_atomic_thread_fence(acquire);
636 if (os_atomic_dec(&args->nthreads, relaxed) == 0) {
637 thread_wakeup((event_t)args);
638 }
639 }
640
641 /**
642 * Helper function for starting the 2.5-minute panic timer to ensure that we
643 * don't get stuck during test teardown.
644 *
645 * @param panic_callout The timer call to use for the panic callout.
646 */
647 static inline void
huge_pv_start_panic_timer(thread_call_t panic_callout)648 huge_pv_start_panic_timer(thread_call_t panic_callout)
649 {
650 uint64_t deadline;
651 clock_interval_to_deadline(150, NSEC_PER_SEC, &deadline);
652 thread_call_enter_delayed(panic_callout, deadline);
653 }
654
655 /**
656 * Timer callout that executes in case the huge PV test incurs excessive (>= 5min)
657 * runtime, which can happen due to unlucky scheduling of the main thread. In this
658 * case we simply set the "stop" flag and expect the worker threads to exit gracefully.
659 *
660 * @param param0 The pmap_hugepv_test_thread_args used to control the test, cast
661 * as thread_call_param_t.
662 * @param param1 Unused argument.
663 */
664 static void
huge_pv_test_timeout(thread_call_param_t param0,__unused thread_call_param_t param1)665 huge_pv_test_timeout(thread_call_param_t param0, __unused thread_call_param_t param1)
666 {
667 pmap_hugepv_test_thread_args *args = (pmap_hugepv_test_thread_args*)param0;
668 args->args.stop = TRUE;
669 huge_pv_start_panic_timer(args->panic_callout);
670 }
671
672 /**
673 * Timer callout that executes in case the huge PV test was canceled by
674 * huge_pv_test_timeout above, but failed to terminate within 2.5 minutes.
675 * This callout simply panics to allow inspection of the resultant coredump,
676 * as it should never be reached under correct operation.
677 *
678 * @param param0 Unused argument.
679 * @param param1 Unused argument.
680 */
681 static void __attribute__((noreturn))
huge_pv_test_panic(__unused thread_call_param_t param0,__unused thread_call_param_t param1)682 huge_pv_test_panic(__unused thread_call_param_t param0, __unused thread_call_param_t param1)
683 {
684 panic("%s: test timed out", __func__);
685 }
686
687 /**
688 * Main test thread for exercising contention on a massive physical-to-virtual
689 * mapping list in the pmap. This thread creates a large number of mappings
690 * (as requested by the caller) to the same physical page, spawns the above
691 * worker threads to do different operations on that physical page, then while
692 * that is going on it repeatedly calls pmap_page_protect_options() on the page,
693 * for the number of loops specified by the caller.
694 *
695 * @param num_loops Number of iterations to execute in the main thread before
696 * stopping the workers.
697 * @param num_mappings The number of alias mappings to create for the same
698 * physical page.
699 *
700 * @return KERN_SUCCESS if the test succeeds, KERN_FAILURE if it encounters
701 * an unexpected setup failure. Any failed integrity check during
702 * the actual execution of the worker threads will panic.
703 */
704 kern_return_t
test_pmap_huge_pv_list(unsigned int num_loops,unsigned int num_mappings)705 test_pmap_huge_pv_list(unsigned int num_loops, unsigned int num_mappings)
706 {
707 kern_return_t kr = KERN_SUCCESS;
708 thread_t remove_enter_thread, fast_fault_thread, cache_attr_thread;
709 if ((num_loops == 0) || (num_mappings == 0)) {
710 /**
711 * If num_mappings is 0, we'll get into a case in which the
712 * remove_enter_thread leaves a single dangling mapping, triggering
713 * a panic when we free the page. This isn't a valid test
714 * configuration anyway.
715 */
716 return KERN_SUCCESS;
717 }
718 pmap_t new_pmap = pmap_create_wrapper(0);
719 if (new_pmap == NULL) {
720 return KERN_FAILURE;
721 }
722 vm_page_t m = pmap_test_alloc_vm_page();
723 if (m == VM_PAGE_NULL) {
724 pmap_destroy(new_pmap);
725 return KERN_FAILURE;
726 }
727
728 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
729
730 for (unsigned int mapping = 0; mapping < num_mappings; ++mapping) {
731 kr = pmap_enter(new_pmap, PMAP_TEST_VA + ((vm_offset_t)mapping << PAGE_SHIFT), phys_page,
732 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
733 assert(kr == KERN_SUCCESS);
734 }
735
736 thread_call_t huge_pv_panic_call = thread_call_allocate(huge_pv_test_panic, NULL);
737
738 pmap_hugepv_test_thread_args args = {
739 .args = {.pmap = new_pmap, .stop = FALSE, .va = PMAP_TEST_VA, .pn = phys_page},
740 .nthreads = 0, .num_mappings = num_mappings, .panic_callout = huge_pv_panic_call
741 };
742
743 thread_call_t huge_pv_timer_call = thread_call_allocate(huge_pv_test_timeout, &args);
744
745 kr = kernel_thread_start_priority(hugepv_remove_enter_thread,
746 &args, thread_kern_get_pri(current_thread()), &remove_enter_thread);
747 if (kr != KERN_SUCCESS) {
748 goto hugepv_cleanup;
749 }
750 ++args.nthreads;
751 thread_deallocate(remove_enter_thread);
752
753 kr = kernel_thread_start_priority(hugepv_fast_fault_thread, &args,
754 thread_kern_get_pri(current_thread()), &fast_fault_thread);
755 if (kr != KERN_SUCCESS) {
756 goto hugepv_cleanup;
757 }
758 ++args.nthreads;
759 thread_deallocate(fast_fault_thread);
760
761 kr = kernel_thread_start_priority(hugepv_cache_attr_thread, &args,
762 thread_kern_get_pri(current_thread()), &cache_attr_thread);
763 if (kr != KERN_SUCCESS) {
764 goto hugepv_cleanup;
765 }
766 ++args.nthreads;
767 thread_deallocate(cache_attr_thread);
768
769 /**
770 * Set up a 5 minute timer to gracefully halt the test upon expiry.
771 * Ordinarily the test should complete in well less than 5 minutes,
772 * but it can run longer and hit the 10 minute BATS timeout if this
773 * thread is really unlucky w.r.t. scheduling (which can happen if
774 * it is repeatedly preempted and starved by the other threads
775 * contending on the PVH lock).
776 */
777 uint64_t deadline;
778 clock_interval_to_deadline(300, NSEC_PER_SEC, &deadline);
779 thread_call_enter_delayed(huge_pv_timer_call, deadline);
780
781 for (unsigned int i = 0; (i < num_loops) && !args.args.stop; i++) {
782 pmap_page_protect_options(phys_page, VM_PROT_READ, 0, NULL);
783 /**
784 * Yield briefly to give the other workers a chance to get through
785 * more iterations.
786 */
787 __builtin_arm_wfe();
788 }
789
790 pmap_disconnect_options(phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
791
792 hugepv_cleanup:
793 thread_call_cancel_wait(huge_pv_timer_call);
794 thread_call_free(huge_pv_timer_call);
795
796 if (__improbable(args.args.stop)) {
797 /**
798 * If stop is already set, we hit the timeout, so we can't safely block waiting for
799 * the workers to terminate as they may already be doing so. Spin in a WFE loop
800 * instead.
801 */
802 while (os_atomic_load_exclusive(&args.nthreads, relaxed) != 0) {
803 __builtin_arm_wfe();
804 }
805 os_atomic_clear_exclusive();
806 } else if (args.nthreads > 0) {
807 /* Ensure prior stores to nthreads are visible before the update to args.args.stop. */
808 os_atomic_thread_fence(release);
809 huge_pv_start_panic_timer(huge_pv_panic_call);
810 assert_wait((event_t)&args, THREAD_UNINT);
811 args.args.stop = TRUE;
812 thread_block(THREAD_CONTINUE_NULL);
813 assert(args.nthreads == 0);
814 }
815
816 thread_call_cancel_wait(huge_pv_panic_call);
817 thread_call_free(huge_pv_panic_call);
818
819 if (new_pmap != NULL) {
820 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + ((vm_offset_t)num_mappings << PAGE_SHIFT));
821 }
822
823 pmap_test_free_vm_page(m);
824 if (new_pmap != NULL) {
825 pmap_destroy(new_pmap);
826 }
827
828 return kr;
829 }
830
831 #endif /* CONFIG_SPTM */
832