1 /*
2 * Copyright (c) 2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_page.h>
30 #include <vm/pmap.h>
31 #include <kern/ledger.h>
32 #include <kern/thread.h>
33 #if defined(__arm64__)
34 #include <pexpert/arm64/board_config.h>
35 #include <arm/pmap/pmap_pt_geometry.h>
36 #endif /* defined(__arm64__) */
37 #include <vm/vm_map.h>
38
39 extern void read_random(void* buffer, u_int numBytes);
40
41 extern ledger_template_t task_ledger_template;
42
43 extern boolean_t arm_force_fast_fault(ppnum_t, vm_prot_t, int, void*);
44 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
45
46 kern_return_t test_pmap_enter_disconnect(unsigned int num_loops);
47 kern_return_t test_pmap_compress_remove(unsigned int num_loops);
48 kern_return_t test_pmap_exec_remove(unsigned int num_loops);
49 kern_return_t test_pmap_nesting(unsigned int num_loops);
50 kern_return_t test_pmap_iommu_disconnect(void);
51 kern_return_t test_pmap_extended(void);
52 void test_pmap_call_overhead(unsigned int num_loops);
53 uint64_t test_pmap_page_protect_overhead(unsigned int num_loops, unsigned int num_aliases);
54
55 #define PMAP_TEST_VA (0xDEAD << PAGE_SHIFT)
56
57 typedef struct {
58 pmap_t pmap;
59 vm_map_address_t va;
60 processor_t proc;
61 ppnum_t pn;
62 volatile boolean_t stop;
63 } pmap_test_thread_args;
64
65 static pmap_t
pmap_create_wrapper(unsigned int flags)66 pmap_create_wrapper(unsigned int flags)
67 {
68 pmap_t new_pmap = NULL;
69 ledger_t ledger;
70 assert(task_ledger_template != NULL);
71 if ((ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES)) == NULL) {
72 return NULL;
73 }
74 new_pmap = pmap_create_options(ledger, 0, flags);
75 ledger_dereference(ledger);
76 return new_pmap;
77 }
78
79 static void
pmap_disconnect_thread(void * arg,wait_result_t __unused wres)80 pmap_disconnect_thread(void *arg, wait_result_t __unused wres)
81 {
82 pmap_test_thread_args *args = arg;
83 do {
84 pmap_disconnect(args->pn);
85 } while (!args->stop);
86 thread_wakeup((event_t)args);
87 }
88
89 kern_return_t
test_pmap_enter_disconnect(unsigned int num_loops)90 test_pmap_enter_disconnect(unsigned int num_loops)
91 {
92 kern_return_t kr = KERN_SUCCESS;
93 thread_t disconnect_thread;
94 pmap_t new_pmap = pmap_create_wrapper(0);
95 if (new_pmap == NULL) {
96 return KERN_FAILURE;
97 }
98 vm_page_t m = vm_page_grab();
99 if (m == VM_PAGE_NULL) {
100 pmap_destroy(new_pmap);
101 return KERN_FAILURE;
102 }
103 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
104 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .pn = phys_page};
105 kern_return_t res = kernel_thread_start(pmap_disconnect_thread, &args, &disconnect_thread);
106 if (res) {
107 pmap_destroy(new_pmap);
108 vm_page_lock_queues();
109 vm_page_free(m);
110 vm_page_unlock_queues();
111 return res;
112 }
113 thread_deallocate(disconnect_thread);
114
115 while (num_loops-- != 0) {
116 kr = pmap_enter(new_pmap, PMAP_TEST_VA, phys_page,
117 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE);
118 assert(kr == KERN_SUCCESS);
119 }
120
121 assert_wait((event_t)&args, THREAD_UNINT);
122 args.stop = TRUE;
123 thread_block(THREAD_CONTINUE_NULL);
124
125 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
126 vm_page_lock_queues();
127 vm_page_free(m);
128 vm_page_unlock_queues();
129 pmap_destroy(new_pmap);
130 return KERN_SUCCESS;
131 }
132
133 static void
pmap_remove_thread(void * arg,wait_result_t __unused wres)134 pmap_remove_thread(void *arg, wait_result_t __unused wres)
135 {
136 pmap_test_thread_args *args = arg;
137 do {
138 kern_return_t kr = pmap_enter_options(args->pmap, args->va, args->pn,
139 VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_OPTIONS_INTERNAL, NULL);
140 assert(kr == KERN_SUCCESS);
141 pmap_remove(args->pmap, args->va, args->va + PAGE_SIZE);
142 } while (!args->stop);
143 thread_wakeup((event_t)args);
144 }
145
146 /**
147 * Test that a mapping to a physical page can be concurrently removed while
148 * the page is being compressed, without triggering accounting panics.
149 *
150 * @param num_loops The number of test loops to run
151 *
152 * @return KERN_SUCCESS if the test runs to completion, otherwise an
153 * appropriate error code.
154 */
155 kern_return_t
test_pmap_compress_remove(unsigned int num_loops)156 test_pmap_compress_remove(unsigned int num_loops)
157 {
158 thread_t remove_thread;
159 pmap_t new_pmap = pmap_create_wrapper(0);
160 if (new_pmap == NULL) {
161 return KERN_FAILURE;
162 }
163 vm_page_t m = vm_page_grab();
164 if (m == VM_PAGE_NULL) {
165 pmap_destroy(new_pmap);
166 return KERN_FAILURE;
167 }
168 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
169 pmap_test_thread_args args = {.pmap = new_pmap, .stop = FALSE, .va = PMAP_TEST_VA, .pn = phys_page};
170 kern_return_t res = kernel_thread_start_priority(pmap_remove_thread,
171 &args, current_thread()->base_pri, &remove_thread);
172 if (res) {
173 pmap_destroy(new_pmap);
174 vm_page_lock_queues();
175 vm_page_free(m);
176 vm_page_unlock_queues();
177 return res;
178 }
179 thread_deallocate(remove_thread);
180
181 while (num_loops-- != 0) {
182 pmap_disconnect_options(phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
183 }
184
185 assert_wait((event_t)&args, THREAD_UNINT);
186 args.stop = TRUE;
187 thread_block(THREAD_CONTINUE_NULL);
188
189 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + PAGE_SIZE);
190 pmap_destroy(new_pmap);
191 vm_page_lock_queues();
192 vm_page_free(m);
193 vm_page_unlock_queues();
194 return KERN_SUCCESS;
195 }
196
197
198 kern_return_t
test_pmap_exec_remove(unsigned int num_loops __unused)199 test_pmap_exec_remove(unsigned int num_loops __unused)
200 {
201 return KERN_NOT_SUPPORTED;
202 }
203
204
205 #if defined(__arm64__)
206
207 static const vm_map_address_t nesting_start = SHARED_REGION_BASE;
208 static const vm_map_address_t nesting_size = 16 * ARM_16K_TT_L2_SIZE;
209
210 static void
pmap_nest_thread(void * arg,wait_result_t __unused wres)211 pmap_nest_thread(void *arg, wait_result_t __unused wres)
212 {
213 const pmap_test_thread_args *args = arg;
214 pmap_t main_pmap = pmap_create_wrapper(0);
215 kern_return_t kr;
216
217 thread_bind(args->proc);
218 thread_block(THREAD_CONTINUE_NULL);
219
220 /**
221 * Exercise nesting and unnesting while bound to the specified CPU (if non-NULL).
222 * The unnesting size here should match the unnesting size used in the first
223 * unnesting step of the main thread, in order to avoid concurrently unnesting
224 * beyond that region and violating the checks against over-unnesting performed
225 * in the main thread.
226 */
227 if (main_pmap != NULL) {
228 kr = pmap_nest(main_pmap, args->pmap, nesting_start, nesting_size);
229 assert(kr == KERN_SUCCESS);
230
231 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
232 assert(kr == KERN_SUCCESS);
233 }
234
235 thread_bind(PROCESSOR_NULL);
236 thread_block(THREAD_CONTINUE_NULL);
237
238 assert_wait((event_t)(uintptr_t)&(args->stop), THREAD_UNINT);
239 if (!args->stop) {
240 thread_block(THREAD_CONTINUE_NULL);
241 } else {
242 clear_wait(current_thread(), THREAD_AWAKENED);
243 }
244
245 /* Unnest all remaining mappings so that we can safely destroy our pmap. */
246 if (main_pmap != NULL) {
247 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
248 assert(kr == KERN_SUCCESS);
249 pmap_destroy(main_pmap);
250 }
251
252 thread_wakeup((event_t)arg);
253 }
254
255 /**
256 * Test that pmap_nest() and pmap_unnest() work correctly when executed concurrently from
257 * multiple threads. Spawn some worker threads at elevated priority and bound to the
258 * same CPU in order to provoke preemption of the nest/unnest operation.
259 *
260 * @param num_loops The number of nest/unnest loops to perform. This should be kept to
261 * a small number because each cycle is expensive and may consume a global shared
262 * region ID.
263 *
264 * @return KERN_SUCCESS if all tests succeed, an appropriate error code otherwise.
265 */
266 kern_return_t
test_pmap_nesting(unsigned int num_loops)267 test_pmap_nesting(unsigned int num_loops)
268 {
269 kern_return_t kr = KERN_SUCCESS;
270
271 vm_page_t m1 = VM_PAGE_NULL, m2 = VM_PAGE_NULL;
272
273 m1 = vm_page_grab();
274 m2 = vm_page_grab();
275 if ((m1 == VM_PAGE_NULL) || (m2 == VM_PAGE_NULL)) {
276 kr = KERN_FAILURE;
277 goto test_nesting_cleanup;
278 }
279 const ppnum_t pp1 = VM_PAGE_GET_PHYS_PAGE(m1);
280 const ppnum_t pp2 = VM_PAGE_GET_PHYS_PAGE(m2);
281 for (unsigned int i = 0; (i < num_loops) && (kr == KERN_SUCCESS); i++) {
282 pmap_t nested_pmap = pmap_create_wrapper(0);
283 pmap_t main_pmap = pmap_create_wrapper(0);
284 if ((nested_pmap == NULL) || (main_pmap == NULL)) {
285 pmap_destroy(main_pmap);
286 pmap_destroy(nested_pmap);
287 kr = KERN_FAILURE;
288 break;
289 }
290 pmap_set_nested(nested_pmap);
291 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
292 uint8_t rand;
293 read_random(&rand, sizeof(rand));
294 uint8_t rand_mod = rand % 3;
295 if (rand_mod == 0) {
296 continue;
297 }
298 kr = pmap_enter(nested_pmap, va, (rand_mod == 1) ? pp1 : pp2, VM_PROT_READ,
299 VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE);
300 assert(kr == KERN_SUCCESS);
301 }
302 kr = pmap_nest(main_pmap, nested_pmap, nesting_start, nesting_size);
303 assert(kr == KERN_SUCCESS);
304
305 /* Validate the initial nest operation produced global mappings within the nested pmap. */
306 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
307 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
308 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
309 if (nested_pte != main_pte) {
310 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx",
311 __func__, nested_pte, main_pte, (unsigned long long)va);
312 }
313 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
314 panic("%s: nested_pte (%p) is not global for va 0x%llx",
315 __func__, nested_pte, (unsigned long long)va);
316 }
317 }
318
319 /* Now kick off various worker threads to concurrently nest and unnest. */
320 const processor_t nest_proc = current_processor();
321 thread_bind(nest_proc);
322 thread_block(THREAD_CONTINUE_NULL);
323
324 /**
325 * Avoid clogging the CPUs with high-priority kernel threads on older devices.
326 * Testing has shown this may provoke a userspace watchdog timeout.
327 */
328 #define TEST_NEST_THREADS 4
329 #if TEST_NEST_THREADS >= MAX_CPUS
330 #undef TEST_NEST_THREADS
331 #define TEST_NEST_THREADS MAX_CPUS - 1
332 #endif
333 thread_t nest_threads[TEST_NEST_THREADS];
334 kern_return_t thread_krs[TEST_NEST_THREADS];
335 pmap_test_thread_args args[TEST_NEST_THREADS];
336 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
337 args[j].pmap = nested_pmap;
338 args[j].stop = FALSE;
339 /**
340 * Spawn the worker threads at various priorities at the high end of the kernel range,
341 * and bind every other thread to the same CPU as this thread to provoke preemption,
342 * while also allowing some threads to run concurrently on other CPUs.
343 */
344 args[j].proc = ((j % 2) ? PROCESSOR_NULL : nest_proc);
345 thread_krs[j] = kernel_thread_start_priority(pmap_nest_thread, &args[j], MAXPRI_KERNEL - (j % 4), &nest_threads[j]);
346 if (thread_krs[j] == KERN_SUCCESS) {
347 thread_set_thread_name(nest_threads[j], "pmap_nest_thread");
348 }
349 }
350
351 /* Unnest the bulk of the nested region and validate that it produced the expected PTE contents. */
352 kr = pmap_unnest(main_pmap, nesting_start, nesting_size - ARM_16K_TT_L2_SIZE);
353 assert(kr == KERN_SUCCESS);
354
355 for (vm_map_address_t va = nesting_start; va < (nesting_start + nesting_size - ARM_16K_TT_L2_SIZE); va += PAGE_SIZE) {
356 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
357 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
358
359 if (main_pte != NULL) {
360 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
361 __func__, main_pte, (unsigned long long)va);
362 }
363 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
364 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
365 __func__, nested_pte, (unsigned long long)va);
366 }
367 }
368
369 /* Validate that the prior unnest did not unnest too much. */
370 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
371 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
372 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
373 if (nested_pte != main_pte) {
374 panic("%s: nested_pte (%p) is not identical to main_pte (%p) for va 0x%llx following adjacent unnest",
375 __func__, nested_pte, main_pte, (unsigned long long)va);
376 }
377 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && (*nested_pte & ARM_PTE_NG)) {
378 panic("%s: nested_pte (%p) is not global for va 0x%llx following adjacent unnest",
379 __func__, nested_pte, (unsigned long long)va);
380 }
381 }
382
383 /* Now unnest the remainder. */
384 kr = pmap_unnest(main_pmap, nesting_start + nesting_size - ARM_16K_TT_L2_SIZE, ARM_16K_TT_L2_SIZE);
385 assert(kr == KERN_SUCCESS);
386
387 thread_bind(PROCESSOR_NULL);
388 thread_block(THREAD_CONTINUE_NULL);
389
390 for (vm_map_address_t va = nesting_start + nesting_size - ARM_16K_TT_L2_SIZE; va < (nesting_start + nesting_size); va += PAGE_SIZE) {
391 pt_entry_t *nested_pte = pmap_pte(nested_pmap, va);
392 pt_entry_t *main_pte = pmap_pte(main_pmap, va);
393
394 if (main_pte != NULL) {
395 panic("%s: main_pte (%p) is not NULL for unnested VA 0x%llx",
396 __func__, main_pte, (unsigned long long)va);
397 }
398 if ((nested_pte != NULL) && (*nested_pte != ARM_PTE_EMPTY) && !(*nested_pte & ARM_PTE_NG)) {
399 panic("%s: nested_pte (%p) is global for va 0x%llx following unnest",
400 __func__, nested_pte, (unsigned long long)va);
401 }
402 }
403
404 for (unsigned int j = 0; j < (sizeof(nest_threads) / sizeof(nest_threads[0])); j++) {
405 if (thread_krs[j] == KERN_SUCCESS) {
406 assert_wait((event_t)&args[j], THREAD_UNINT);
407 args[j].stop = TRUE;
408 thread_wakeup((event_t)(uintptr_t)&(args[j].stop));
409 thread_block(THREAD_CONTINUE_NULL);
410 } else {
411 kr = thread_krs[j];
412 }
413 }
414 pmap_remove(nested_pmap, nesting_start, nesting_start + nesting_size);
415 pmap_destroy(main_pmap);
416 pmap_destroy(nested_pmap);
417 }
418
419 test_nesting_cleanup:
420 vm_page_lock_queues();
421 if (m1 != VM_PAGE_NULL) {
422 vm_page_free(m1);
423 }
424 if (m2 != VM_PAGE_NULL) {
425 vm_page_free(m2);
426 }
427 vm_page_unlock_queues();
428
429 return kr;
430 }
431
432 #else /* defined(__arm64__) */
433
434 kern_return_t
test_pmap_nesting(unsigned int num_loops __unused)435 test_pmap_nesting(unsigned int num_loops __unused)
436 {
437 return KERN_NOT_SUPPORTED;
438 }
439
440 #endif /* defined(__arm64__) */
441
442 kern_return_t
test_pmap_iommu_disconnect(void)443 test_pmap_iommu_disconnect(void)
444 {
445 return KERN_SUCCESS;
446 }
447
448
449 kern_return_t
test_pmap_extended(void)450 test_pmap_extended(void)
451 {
452 return KERN_SUCCESS;
453 }
454
455 void
test_pmap_call_overhead(unsigned int num_loops __unused)456 test_pmap_call_overhead(unsigned int num_loops __unused)
457 {
458 #if defined(__arm64__)
459 pmap_t pmap = current_thread()->map->pmap;
460 for (unsigned int i = 0; i < num_loops; ++i) {
461 pmap_nop(pmap);
462 }
463 #endif
464 }
465
466 uint64_t
test_pmap_page_protect_overhead(unsigned int num_loops __unused,unsigned int num_aliases __unused)467 test_pmap_page_protect_overhead(unsigned int num_loops __unused, unsigned int num_aliases __unused)
468 {
469 uint64_t duration = 0;
470 #if defined(__arm64__)
471 pmap_t new_pmap = pmap_create_wrapper(0);
472 vm_page_t m = vm_page_grab();
473 kern_return_t kr = KERN_SUCCESS;
474
475 vm_page_lock_queues();
476 if (m != VM_PAGE_NULL) {
477 vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
478 }
479 vm_page_unlock_queues();
480
481 if ((new_pmap == NULL) || (m == VM_PAGE_NULL)) {
482 goto ppo_cleanup;
483 }
484
485 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
486
487 for (unsigned int loop = 0; loop < num_loops; ++loop) {
488 for (unsigned int alias = 0; alias < num_aliases; ++alias) {
489 kr = pmap_enter(new_pmap, PMAP_TEST_VA + (PAGE_SIZE * alias), phys_page,
490 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE);
491 assert(kr == KERN_SUCCESS);
492 }
493
494 uint64_t start_time = mach_absolute_time();
495
496 pmap_page_protect_options(phys_page, VM_PROT_READ, 0, NULL);
497
498 duration += (mach_absolute_time() - start_time);
499
500 pmap_remove(new_pmap, PMAP_TEST_VA, PMAP_TEST_VA + (num_aliases * PAGE_SIZE));
501 }
502
503 ppo_cleanup:
504 vm_page_lock_queues();
505 if (m != VM_PAGE_NULL) {
506 vm_page_free(m);
507 }
508 vm_page_unlock_queues();
509 if (new_pmap != NULL) {
510 pmap_destroy(new_pmap);
511 }
512 #endif
513 return duration;
514 }
515