1 /*
2 * Copyright (c) 2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_page.h>
30 #include <vm/pmap.h>
31 #include <kern/ledger.h>
32 #include <kern/thread.h>
33 #if defined(__arm64__)
34 #include <pexpert/arm64/board_config.h>
35 #include <arm/pmap/pmap_pt_geometry.h>
36 #endif /* defined(__arm64__) */
37 #include <vm/vm_map.h>
38
39 extern void read_random(void* buffer, u_int numBytes);
40
41 extern ledger_template_t task_ledger_template;
42
43 extern boolean_t arm_force_fast_fault(ppnum_t, vm_prot_t, int, void*);
44 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
45
46 kern_return_t test_pmap_enter_disconnect(unsigned int num_loops);
47 kern_return_t test_pmap_compress_remove(unsigned int num_loops);
48 kern_return_t test_pmap_exec_remove(unsigned int num_loops);
49 kern_return_t test_pmap_nesting(unsigned int num_loops);
50 kern_return_t test_pmap_iommu_disconnect(void);
51 kern_return_t test_pmap_extended(void);
52 void test_pmap_call_overhead(unsigned int num_loops);
53 uint64_t test_pmap_page_protect_overhead(unsigned int num_loops, unsigned int num_aliases);
54
55 #define PMAP_TEST_VA (0xDEAD << PAGE_SHIFT)
56
57 typedef struct {
58 pmap_t pmap;
59 vm_map_address_t va;
60 processor_t proc;
61 ppnum_t pn;
62 volatile boolean_t stop;
63 } pmap_test_thread_args;
64
65 static pmap_t
pmap_create_wrapper(unsigned int flags)66 pmap_create_wrapper(unsigned int flags)
67 {
68 pmap_t new_pmap = NULL;
69 ledger_t ledger;
70 assert(task_ledger_template != NULL);
71 if ((ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES)) == NULL) {
72 return NULL;
73 }
74 new_pmap = pmap_create_options(ledger, 0, flags);
75 ledger_dereference(ledger);
76 return new_pmap;
77 }
78
79 static void
pmap_disconnect_thread(void * arg,wait_result_t __unused wres)80 pmap_disconnect_thread(void *arg, wait_result_t __unused wres)
81 {
82 pmap_test_thread_args *args = arg;
83 do {
84 pmap_disconnect(args->pn);
85 } while (!args->stop);
86 thread_wakeup((event_t)args);
87 }
88
89 kern_return_t
test_pmap_enter_disconnect(unsigned int num_loops)90 test_pmap_enter_disconnect(unsigned int num_loops)
91 {
92 kern_return_t kr = KERN_SUCCESS;
93 thread_t disconnect_thread;
94 pmap_t new_pmap = pmap_create_wrapper(0);
95 if (new_pmap == NULL) {
96 return KERN_FAILURE;
97 }
98 vm_page_t m = vm_page_grab();
99 if (m == VM_PAGE_NULL) {
100 pmap_destroy(new_pmap);
101 return KERN_FAILURE;
102 }
103 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
104 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .pn = phys_page};
105 kern_return_t res = kernel_thread_start_priority(pmap_disconnect_thread,
106 &args, thread_kern_get_pri(current_thread()), &disconnect_thread);
107 if (res) {
108 pmap_destroy(new_pmap);
109 vm_page_lock_queues();
110 vm_page_free(m);
111 vm_page_unlock_queues();
112 return res;
113 }
114 thread_deallocate(disconnect_thread);
115
116 while (num_loops-- != 0) {
117 kr = pmap_enter(new_pmap, PMAP_TEST_VA, phys_page,
118 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
119 assert(kr == KERN_SUCCESS);
120 }
121
122 assert_wait((event_t)&args, THREAD_UNINT);
123 args.stop = TRUE;
124 thread_block(THREAD_CONTINUE_NULL);
125
126 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
127 vm_page_lock_queues();
128 vm_page_free(m);
129 vm_page_unlock_queues();
130 pmap_destroy(new_pmap);
131 return KERN_SUCCESS;
132 }
133
134 static void
pmap_remove_thread(void * arg,wait_result_t __unused wres)135 pmap_remove_thread(void *arg, wait_result_t __unused wres)
136 {
137 pmap_test_thread_args *args = arg;
138 do {
139 kern_return_t kr = pmap_enter_options(args->pmap, args->va, args->pn,
140 VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_OPTIONS_INTERNAL, NULL, PMAP_MAPPING_TYPE_INFER);
141 assert(kr == KERN_SUCCESS);
142 pmap_remove(args->pmap, args->va, args->va + PAGE_SIZE);
143 } while (!args->stop);
144 thread_wakeup((event_t)args);
145 }
146
147 /**
148 * Test that a mapping to a physical page can be concurrently removed while
149 * the page is being compressed, without triggering accounting panics.
150 *
151 * @param num_loops The number of test loops to run
152 *
153 * @return KERN_SUCCESS if the test runs to completion, otherwise an
154 * appropriate error code.
155 */
156 kern_return_t
test_pmap_compress_remove(unsigned int num_loops)157 test_pmap_compress_remove(unsigned int num_loops)
158 {
159 thread_t remove_thread;
160 pmap_t new_pmap = pmap_create_wrapper(0);
161 if (new_pmap == NULL) {
162 return KERN_FAILURE;
163 }
164 vm_page_t m = vm_page_grab();
165 if (m == VM_PAGE_NULL) {
166 pmap_destroy(new_pmap);
167 return KERN_FAILURE;
168 }
169 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
170 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .va = PMAP_TEST_VA, .pn = phys_page};
171 kern_return_t res = kernel_thread_start_priority(pmap_remove_thread,
172 &args, thread_kern_get_pri(current_thread()), &remove_thread);
173 if (res) {
174 pmap_destroy(new_pmap);
175 vm_page_lock_queues();
176 vm_page_free(m);
177 vm_page_unlock_queues();
178 return res;
179 }
180 thread_deallocate(remove_thread);
181
182 while (num_loops-- != 0) {
183 pmap_disconnect_options(phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
184 }
185
186 assert_wait((event_t)&args, THREAD_UNINT);
187 args.stop = TRUE;
188 thread_block(THREAD_CONTINUE_NULL);
189
190 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
191 pmap_destroy(new_pmap);
192 vm_page_lock_queues();
193 vm_page_free(m);
194 vm_page_unlock_queues();
195 return KERN_SUCCESS;
196 }
197
198
199 kern_return_t
test_pmap_exec_remove(unsigned int num_loops __unused)200 test_pmap_exec_remove(unsigned int num_loops __unused)
201 {
202 return KERN_NOT_SUPPORTED;
203 }
204
205
206 #if defined(__arm64__)
207
208 static const vm_map_address_t nesting_start = SHARED_REGION_BASE;
209 static const vm_map_address_t nesting_size = 16 * ARM_16K_TT_L2_SIZE;
210
211 static void
pmap_nest_thread(void * arg,wait_result_t __unused wres)212 pmap_nest_thread(void *arg, wait_result_t __unused wres)
213 {
214 const pmap_test_thread_args *args = arg;
215 pmap_t main_pmap = pmap_create_wrapper(0);
216 kern_return_t kr;
217
218 thread_bind(args->proc);
219 thread_block(THREAD_CONTINUE_NULL);
220
221 /**
222 * Exercise nesting and unnesting while bound to the specified CPU (if non-NULL).
223 * The unnesting size here should match the unnesting size used in the first
224 * unnesting step of the main thread, in order to avoid concurrently unnesting
225 * beyond that region and violating the checks against over-unnesting performed
226 * in the main thread.
227 */
228 if (main_pmap != NULL) {
229 kr = pmap_nest(main_pmap, args->pmap, nesting_start, nesting_size);
230 assert(kr == KERN_SUCCESS);
231
232 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
233 assert(kr == KERN_SUCCESS);
234 }
235
236 thread_bind(PROCESSOR_NULL);
237 thread_block(THREAD_CONTINUE_NULL);
238
239 assert_wait((event_t)(uintptr_t)&(args->stop), THREAD_UNINT);
240 if (!args->stop) {
241 thread_block(THREAD_CONTINUE_NULL);
242 } else {
243 clear_wait(current_thread(), THREAD_AWAKENED);
244 }
245
246 /* Unnest all remaining mappings so that we can safely destroy our pmap. */
247 if (main_pmap != NULL) {
248 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
249 assert(kr == KERN_SUCCESS);
250 pmap_destroy(main_pmap);
251 }
252
253 thread_wakeup((event_t)arg);
254 }
255
256 /**
257 * Test that pmap_nest() and pmap_unnest() work correctly when executed concurrently from
258 * multiple threads. Spawn some worker threads at elevated priority and bound to the
259 * same CPU in order to provoke preemption of the nest/unnest operation.
260 *
261 * @param num_loops The number of nest/unnest loops to perform. This should be kept to
262 * a small number because each cycle is expensive and may consume a global shared
263 * region ID.
264 *
265 * @return KERN_SUCCESS if all tests succeed, an appropriate error code otherwise.
266 */
267 kern_return_t
test_pmap_nesting(unsigned int num_loops)268 test_pmap_nesting(unsigned int num_loops)
269 {
270 kern_return_t kr = KERN_SUCCESS;
271
272 vm_page_t m1 = VM_PAGE_NULL, m2 = VM_PAGE_NULL;
273
274 m1 = vm_page_grab();
275 m2 = vm_page_grab();
276 if ((m1 == VM_PAGE_NULL) || (m2 == VM_PAGE_NULL)) {
277 kr = KERN_FAILURE;
278 goto test_nesting_cleanup;
279 }
280 const ppnum_t pp1 = VM_PAGE_GET_PHYS_PAGE(m1);
281 const ppnum_t pp2 = VM_PAGE_GET_PHYS_PAGE(m2);
282 for (unsigned int i = 0; (i < num_loops) && (kr == KERN_SUCCESS); i++) {
283 pmap_t nested_pmap = pmap_create_wrapper(0);
284 pmap_t main_pmap = pmap_create_wrapper(0);
285 if ((nested_pmap == NULL) || (main_pmap == NULL)) {
286 pmap_destroy(main_pmap);
287 pmap_destroy(nested_pmap);
288 kr = KERN_FAILURE;
289 break;
290 }
291 pmap_set_nested(nested_pmap);
292 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
293 uint8_t rand;
294 read_random(&rand, sizeof(rand));
295 uint8_t rand_mod = rand % 3;
296 if (rand_mod == 0) {
297 continue;
298 }
299 kr = pmap_enter(nested_pmap, va, (rand_mod == 1) ? pp1 : pp2, VM_PROT_READ,
300 VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
301 assert(kr == KERN_SUCCESS);
302 }
303 kr = pmap_nest(main_pmap, nested_pmap, nesting_start, nesting_size);
304 assert(kr == KERN_SUCCESS);
305
306 /* Validate the initial nest operation produced global mappings within the nested pmap. */
307 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
308 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
309 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
310 if (nested_pte != main_pte) {
311 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx",
312 __func__, nested_pte, main_pte, (unsigned long long)va);
313 }
314 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
315 panic("%s: nested_pte (%p) is not global for va 0x%llx",
316 __func__, nested_pte, (unsigned long long)va);
317 }
318 }
319
320 /* Now kick off various worker threads to concurrently nest and unnest. */
321 const processor_t nest_proc = current_processor();
322 thread_bind(nest_proc);
323 thread_block(THREAD_CONTINUE_NULL);
324
325 /**
326 * Avoid clogging the CPUs with high-priority kernel threads on older devices.
327 * Testing has shown this may provoke a userspace watchdog timeout.
328 */
329 #define TEST_NEST_THREADS 4
330 #if TEST_NEST_THREADS >= MAX_CPUS
331 #undef TEST_NEST_THREADS
332 #define TEST_NEST_THREADS MAX_CPUS - 1
333 #endif
334 thread_t nest_threads[TEST_NEST_THREADS];
335 kern_return_t thread_krs[TEST_NEST_THREADS];
336 pmap_test_thread_args args[TEST_NEST_THREADS];
337 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
338 args[j].pmap = nested_pmap;
339 args[j].stop = FALSE;
340 /**
341 * Spawn the worker threads at various priorities at the high end of the kernel range,
342 * and bind every other thread to the same CPU as this thread to provoke preemption,
343 * while also allowing some threads to run concurrently on other CPUs.
344 */
345 args[j].proc = ((j % 2) ? PROCESSOR_NULL : nest_proc);
346 thread_krs[j] = kernel_thread_start_priority(pmap_nest_thread, &args[j], MAXPRI_KERNEL - (j % 4), &nest_threads[j]);
347 if (thread_krs[j] == KERN_SUCCESS) {
348 thread_set_thread_name(nest_threads[j], "pmap_nest_thread");
349 }
350 }
351
352 /* Unnest the bulk of the nested region and validate that it produced the expected PTE contents. */
353 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
354 assert(kr == KERN_SUCCESS);
355
356 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size - ARM_16K_TT_L2_SIZE); va += PAGE_SIZE) {
357 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
358 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
359
360 if (main_pte != NULL) {
361 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
362 __func__, main_pte, (unsigned long long)va);
363 }
364 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
365 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
366 __func__, nested_pte, (unsigned long long)va);
367 }
368 }
369
370 /* Validate that the prior unnest did not unnest too much. */
371 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
372 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
373 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
374 if (nested_pte != main_pte) {
375 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx following adjacent unnest",
376 __func__, nested_pte, main_pte, (unsigned long long)va);
377 }
378 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
379 panic("%s: nested_pte (%p) is not global for va 0x%llx following adjacent unnest",
380 __func__, nested_pte, (unsigned long long)va);
381 }
382 }
383
384 /* Now unnest the remainder. */
385 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
386 assert(kr == KERN_SUCCESS);
387
388 thread_bind(PROCESSOR_NULL);
389 thread_block(THREAD_CONTINUE_NULL);
390
391 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
392 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
393 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
394
395 if (main_pte != NULL) {
396 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
397 __func__, main_pte, (unsigned long long)va);
398 }
399 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
400 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
401 __func__, nested_pte, (unsigned long long)va);
402 }
403 }
404
405 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
406 if (thread_krs[j] == KERN_SUCCESS) {
407 assert_wait((event_t)&args[j], THREAD_UNINT);
408 args[j].stop = TRUE;
409 thread_wakeup((event_t)(uintptr_t)&(args[j].stop));
410 thread_block(THREAD_CONTINUE_NULL);
411 } else {
412 kr = thread_krs[j];
413 }
414 }
415 pmap_remove(nested_pmap, nesting_start, nesting_start + nesting_size);
416 pmap_destroy(main_pmap);
417 pmap_destroy(nested_pmap);
418 }
419
420 test_nesting_cleanup:
421 vm_page_lock_queues();
422 if (m1 != VM_PAGE_NULL) {
423 vm_page_free(m1);
424 }
425 if (m2 != VM_PAGE_NULL) {
426 vm_page_free(m2);
427 }
428 vm_page_unlock_queues();
429
430 return kr;
431 }
432
433 #else /* defined(__arm64__) */
434
435 kern_return_t
test_pmap_nesting(unsigned int num_loops __unused)436 test_pmap_nesting(unsigned int num_loops __unused)
437 {
438 return KERN_NOT_SUPPORTED;
439 }
440
441 #endif /* defined(__arm64__) */
442
443 kern_return_t
test_pmap_iommu_disconnect(void)444 test_pmap_iommu_disconnect(void)
445 {
446 return KERN_SUCCESS;
447 }
448
449
450 kern_return_t
test_pmap_extended(void)451 test_pmap_extended(void)
452 {
453 return KERN_SUCCESS;
454 }
455
456 void
test_pmap_call_overhead(unsigned int num_loops __unused)457 test_pmap_call_overhead(unsigned int num_loops __unused)
458 {
459 #if defined(__arm64__)
460 pmap_t pmap = current_thread()->map->pmap;
461 for (unsigned int i = 0; i < num_loops; ++i) {
462 pmap_nop(pmap);
463 }
464 #endif
465 }
466
467 uint64_t
test_pmap_page_protect_overhead(unsigned int num_loops __unused,unsigned int num_aliases __unused)468 test_pmap_page_protect_overhead(unsigned int num_loops __unused, unsigned int num_aliases __unused)
469 {
470 uint64_t duration = 0;
471 #if defined(__arm64__)
472 pmap_t new_pmap = pmap_create_wrapper(0);
473 vm_page_t m = vm_page_grab();
474 kern_return_t kr = KERN_SUCCESS;
475
476 vm_page_lock_queues();
477 if (m != VM_PAGE_NULL) {
478 vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
479 }
480 vm_page_unlock_queues();
481
482 if ((new_pmap == NULL) || (m == VM_PAGE_NULL)) {
483 goto ppo_cleanup;
484 }
485
486 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
487
488 for (unsigned int loop = 0; loop < num_loops; ++loop) {
489 for (unsigned int alias = 0; alias < num_aliases; ++alias) {
490 kr = pmap_enter(new_pmap, PMAP_TEST_VA + (PAGE_SIZE * alias), phys_page,
491 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
492 assert(kr == KERN_SUCCESS);
493 }
494
495 uint64_t start_time = mach_absolute_time();
496
497 pmap_page_protect_options(phys_page, VM_PROT_READ, 0, NULL);
498
499 duration += (mach_absolute_time() - start_time);
500
501 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + (num_aliases * PAGE_SIZE));
502 }
503
504 ppo_cleanup:
505 vm_page_lock_queues();
506 if (m != VM_PAGE_NULL) {
507 vm_page_free(m);
508 }
509 vm_page_unlock_queues();
510 if (new_pmap != NULL) {
511 pmap_destroy(new_pmap);
512 }
513 #endif
514 return duration;
515 }
516