1 /*
2 * Copyright (c) 2011-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System Copyright (c) 1991,1990,1989,1988,1987 Carnegie
33 * Mellon University All Rights Reserved.
34 *
35 * Permission to use, copy, modify and distribute this software and its
36 * documentation is hereby granted, provided that both the copyright notice
37 * and this permission notice appear in all copies of the software,
38 * derivative works or modified versions, and any portions thereof, and that
39 * both notices appear in supporting documentation.
40 *
41 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION.
42 * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES
43 * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
44 *
45 * Carnegie Mellon requests users of this software to return to
46 *
47 * Software Distribution Coordinator or [email protected]
48 * School of Computer Science Carnegie Mellon University Pittsburgh PA
49 * 15213-3890
50 *
51 * any improvements or extensions that they make and grant Carnegie Mellon the
52 * rights to redistribute these changes.
53 */
54
55 #include <mach_ldebug.h>
56
57 #define LOCK_PRIVATE 1
58
59 #include <vm/pmap.h>
60 #include <vm/vm_map.h>
61 #include <kern/kalloc.h>
62 #include <kern/cpu_number.h>
63 #include <kern/locks.h>
64 #include <kern/misc_protos.h>
65 #include <kern/thread.h>
66 #include <kern/processor.h>
67 #include <kern/sched_prim.h>
68 #include <kern/debug.h>
69 #include <string.h>
70 #include <tests/xnupost.h>
71
72 #if MACH_KDB
73 #include <ddb/db_command.h>
74 #include <ddb/db_output.h>
75 #include <ddb/db_sym.h>
76 #include <ddb/db_print.h>
77 #endif /* MACH_KDB */
78
79 #include <san/kasan.h>
80 #include <sys/kdebug.h>
81 #include <sys/munge.h>
82 #include <machine/cpu_capabilities.h>
83 #include <arm/cpu_data_internal.h>
84 #include <arm/pmap.h>
85
86 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
87 #include <arm64/amcc_rorgn.h>
88 #endif // defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
89
90 kern_return_t arm64_lock_test(void);
91 kern_return_t arm64_munger_test(void);
92 kern_return_t arm64_pan_test(void);
93 kern_return_t arm64_late_pan_test(void);
94 #if defined(HAS_APPLE_PAC)
95 #include <ptrauth.h>
96 kern_return_t arm64_ropjop_test(void);
97 #endif
98 #if defined(KERNEL_INTEGRITY_CTRR)
99 kern_return_t ctrr_test(void);
100 kern_return_t ctrr_test_cpu(void);
101 #endif
102
103 // exception handler ignores this fault address during PAN test
104 #if __ARM_PAN_AVAILABLE__
105 const uint64_t pan_ro_value = 0xFEEDB0B0DEADBEEF;
106 vm_offset_t pan_test_addr = 0;
107 vm_offset_t pan_ro_addr = 0;
108 volatile int pan_exception_level = 0;
109 volatile char pan_fault_value = 0;
110 #endif
111
112
113 #include <libkern/OSAtomic.h>
114 #define LOCK_TEST_ITERATIONS 50
115 static hw_lock_data_t lt_hw_lock;
116 static lck_spin_t lt_lck_spin_t;
117 static lck_mtx_t lt_mtx;
118 static lck_rw_t lt_rwlock;
119 static volatile uint32_t lt_counter = 0;
120 static volatile int lt_spinvolatile;
121 static volatile uint32_t lt_max_holders = 0;
122 static volatile uint32_t lt_upgrade_holders = 0;
123 static volatile uint32_t lt_max_upgrade_holders = 0;
124 static volatile uint32_t lt_num_holders = 0;
125 static volatile uint32_t lt_done_threads;
126 static volatile uint32_t lt_target_done_threads;
127 static volatile uint32_t lt_cpu_bind_id = 0;
128
129 static void
lt_note_another_blocking_lock_holder()130 lt_note_another_blocking_lock_holder()
131 {
132 hw_lock_lock(<_hw_lock, LCK_GRP_NULL);
133 lt_num_holders++;
134 lt_max_holders = (lt_max_holders < lt_num_holders) ? lt_num_holders : lt_max_holders;
135 hw_lock_unlock(<_hw_lock);
136 }
137
138 static void
lt_note_blocking_lock_release()139 lt_note_blocking_lock_release()
140 {
141 hw_lock_lock(<_hw_lock, LCK_GRP_NULL);
142 lt_num_holders--;
143 hw_lock_unlock(<_hw_lock);
144 }
145
146 static void
lt_spin_a_little_bit()147 lt_spin_a_little_bit()
148 {
149 uint32_t i;
150
151 for (i = 0; i < 10000; i++) {
152 lt_spinvolatile++;
153 }
154 }
155
156 static void
lt_sleep_a_little_bit()157 lt_sleep_a_little_bit()
158 {
159 delay(100);
160 }
161
162 static void
lt_grab_mutex()163 lt_grab_mutex()
164 {
165 lck_mtx_lock(<_mtx);
166 lt_note_another_blocking_lock_holder();
167 lt_sleep_a_little_bit();
168 lt_counter++;
169 lt_note_blocking_lock_release();
170 lck_mtx_unlock(<_mtx);
171 }
172
173 static void
lt_grab_mutex_with_try()174 lt_grab_mutex_with_try()
175 {
176 while (0 == lck_mtx_try_lock(<_mtx)) {
177 ;
178 }
179 lt_note_another_blocking_lock_holder();
180 lt_sleep_a_little_bit();
181 lt_counter++;
182 lt_note_blocking_lock_release();
183 lck_mtx_unlock(<_mtx);
184 }
185
186 static void
lt_grab_rw_exclusive()187 lt_grab_rw_exclusive()
188 {
189 lck_rw_lock_exclusive(<_rwlock);
190 lt_note_another_blocking_lock_holder();
191 lt_sleep_a_little_bit();
192 lt_counter++;
193 lt_note_blocking_lock_release();
194 lck_rw_done(<_rwlock);
195 }
196
197 static void
lt_grab_rw_exclusive_with_try()198 lt_grab_rw_exclusive_with_try()
199 {
200 while (0 == lck_rw_try_lock_exclusive(<_rwlock)) {
201 lt_sleep_a_little_bit();
202 }
203
204 lt_note_another_blocking_lock_holder();
205 lt_sleep_a_little_bit();
206 lt_counter++;
207 lt_note_blocking_lock_release();
208 lck_rw_done(<_rwlock);
209 }
210
211 /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840)
212 * static void
213 * lt_grab_rw_shared()
214 * {
215 * lck_rw_lock_shared(<_rwlock);
216 * lt_counter++;
217 *
218 * lt_note_another_blocking_lock_holder();
219 * lt_sleep_a_little_bit();
220 * lt_note_blocking_lock_release();
221 *
222 * lck_rw_done(<_rwlock);
223 * }
224 */
225
226 /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840)
227 * static void
228 * lt_grab_rw_shared_with_try()
229 * {
230 * while(0 == lck_rw_try_lock_shared(<_rwlock));
231 * lt_counter++;
232 *
233 * lt_note_another_blocking_lock_holder();
234 * lt_sleep_a_little_bit();
235 * lt_note_blocking_lock_release();
236 *
237 * lck_rw_done(<_rwlock);
238 * }
239 */
240
241 static void
lt_upgrade_downgrade_rw()242 lt_upgrade_downgrade_rw()
243 {
244 boolean_t upgraded, success;
245
246 success = lck_rw_try_lock_shared(<_rwlock);
247 if (!success) {
248 lck_rw_lock_shared(<_rwlock);
249 }
250
251 lt_note_another_blocking_lock_holder();
252 lt_sleep_a_little_bit();
253 lt_note_blocking_lock_release();
254
255 upgraded = lck_rw_lock_shared_to_exclusive(<_rwlock);
256 if (!upgraded) {
257 success = lck_rw_try_lock_exclusive(<_rwlock);
258
259 if (!success) {
260 lck_rw_lock_exclusive(<_rwlock);
261 }
262 }
263
264 lt_upgrade_holders++;
265 if (lt_upgrade_holders > lt_max_upgrade_holders) {
266 lt_max_upgrade_holders = lt_upgrade_holders;
267 }
268
269 lt_counter++;
270 lt_sleep_a_little_bit();
271
272 lt_upgrade_holders--;
273
274 lck_rw_lock_exclusive_to_shared(<_rwlock);
275
276 lt_spin_a_little_bit();
277 lck_rw_done(<_rwlock);
278 }
279
280 #if __AMP__
281 const int limit = 1000000;
282 static int lt_stress_local_counters[MAX_CPUS];
283
284 lck_ticket_t lt_ticket_lock;
285 lck_grp_t lt_ticket_grp;
286
287 static void
lt_stress_ticket_lock()288 lt_stress_ticket_lock()
289 {
290 int local_counter = 0;
291
292 uint cpuid = cpu_number();
293
294 kprintf("%s>cpu %d starting\n", __FUNCTION__, cpuid);
295
296 lck_ticket_lock(<_ticket_lock, <_ticket_grp);
297 lt_counter++;
298 local_counter++;
299 lck_ticket_unlock(<_ticket_lock);
300
301 while (lt_counter < lt_target_done_threads) {
302 ;
303 }
304
305 kprintf("%s>cpu %d started\n", __FUNCTION__, cpuid);
306
307 while (lt_counter < limit) {
308 lck_ticket_lock(<_ticket_lock, <_ticket_grp);
309 if (lt_counter < limit) {
310 lt_counter++;
311 local_counter++;
312 }
313 lck_ticket_unlock(<_ticket_lock);
314 }
315
316 lt_stress_local_counters[cpuid] = local_counter;
317
318 kprintf("%s>final counter %d cpu %d incremented the counter %d times\n", __FUNCTION__, lt_counter, cpuid, local_counter);
319 }
320 #endif
321
322 static void
lt_grab_hw_lock()323 lt_grab_hw_lock()
324 {
325 hw_lock_lock(<_hw_lock, LCK_GRP_NULL);
326 lt_counter++;
327 lt_spin_a_little_bit();
328 hw_lock_unlock(<_hw_lock);
329 }
330
331 static void
lt_grab_hw_lock_with_try()332 lt_grab_hw_lock_with_try()
333 {
334 while (0 == hw_lock_try(<_hw_lock, LCK_GRP_NULL)) {
335 ;
336 }
337 lt_counter++;
338 lt_spin_a_little_bit();
339 hw_lock_unlock(<_hw_lock);
340 }
341
342 static void
lt_grab_hw_lock_with_to()343 lt_grab_hw_lock_with_to()
344 {
345 (void)hw_lock_to(<_hw_lock, &hw_lock_spin_policy, LCK_GRP_NULL);
346 lt_counter++;
347 lt_spin_a_little_bit();
348 hw_lock_unlock(<_hw_lock);
349 }
350
351 static void
lt_grab_spin_lock()352 lt_grab_spin_lock()
353 {
354 lck_spin_lock(<_lck_spin_t);
355 lt_counter++;
356 lt_spin_a_little_bit();
357 lck_spin_unlock(<_lck_spin_t);
358 }
359
360 static void
lt_grab_spin_lock_with_try()361 lt_grab_spin_lock_with_try()
362 {
363 while (0 == lck_spin_try_lock(<_lck_spin_t)) {
364 ;
365 }
366 lt_counter++;
367 lt_spin_a_little_bit();
368 lck_spin_unlock(<_lck_spin_t);
369 }
370
371 static volatile boolean_t lt_thread_lock_grabbed;
372 static volatile boolean_t lt_thread_lock_success;
373
374 static void
lt_reset()375 lt_reset()
376 {
377 lt_counter = 0;
378 lt_max_holders = 0;
379 lt_num_holders = 0;
380 lt_max_upgrade_holders = 0;
381 lt_upgrade_holders = 0;
382 lt_done_threads = 0;
383 lt_target_done_threads = 0;
384 lt_cpu_bind_id = 0;
385
386 OSMemoryBarrier();
387 }
388
389 static void
lt_trylock_hw_lock_with_to()390 lt_trylock_hw_lock_with_to()
391 {
392 OSMemoryBarrier();
393 while (!lt_thread_lock_grabbed) {
394 lt_sleep_a_little_bit();
395 OSMemoryBarrier();
396 }
397 lt_thread_lock_success = hw_lock_to(<_hw_lock,
398 &hw_lock_test_give_up_policy, LCK_GRP_NULL);
399 OSMemoryBarrier();
400 mp_enable_preemption();
401 }
402
403 static void
lt_trylock_spin_try_lock()404 lt_trylock_spin_try_lock()
405 {
406 OSMemoryBarrier();
407 while (!lt_thread_lock_grabbed) {
408 lt_sleep_a_little_bit();
409 OSMemoryBarrier();
410 }
411 lt_thread_lock_success = lck_spin_try_lock(<_lck_spin_t);
412 OSMemoryBarrier();
413 }
414
415 static void
lt_trylock_thread(void * arg,wait_result_t wres __unused)416 lt_trylock_thread(void *arg, wait_result_t wres __unused)
417 {
418 void (*func)(void) = (void (*)(void))arg;
419
420 func();
421
422 OSIncrementAtomic((volatile SInt32*) <_done_threads);
423 }
424
425 static void
lt_start_trylock_thread(thread_continue_t func)426 lt_start_trylock_thread(thread_continue_t func)
427 {
428 thread_t thread;
429 kern_return_t kr;
430
431 kr = kernel_thread_start(lt_trylock_thread, func, &thread);
432 assert(kr == KERN_SUCCESS);
433
434 thread_deallocate(thread);
435 }
436
437 static void
lt_wait_for_lock_test_threads()438 lt_wait_for_lock_test_threads()
439 {
440 OSMemoryBarrier();
441 /* Spin to reduce dependencies */
442 while (lt_done_threads < lt_target_done_threads) {
443 lt_sleep_a_little_bit();
444 OSMemoryBarrier();
445 }
446 OSMemoryBarrier();
447 }
448
449 static kern_return_t
lt_test_trylocks()450 lt_test_trylocks()
451 {
452 boolean_t success;
453 extern unsigned int real_ncpus;
454
455 /*
456 * First mtx try lock succeeds, second fails.
457 */
458 success = lck_mtx_try_lock(<_mtx);
459 T_ASSERT_NOTNULL(success, "First mtx try lock");
460 success = lck_mtx_try_lock(<_mtx);
461 T_ASSERT_NULL(success, "Second mtx try lock for a locked mtx");
462 lck_mtx_unlock(<_mtx);
463
464 /*
465 * After regular grab, can't try lock.
466 */
467 lck_mtx_lock(<_mtx);
468 success = lck_mtx_try_lock(<_mtx);
469 T_ASSERT_NULL(success, "try lock should fail after regular lck_mtx_lock");
470 lck_mtx_unlock(<_mtx);
471
472 /*
473 * Two shared try locks on a previously unheld rwlock suceed, and a
474 * subsequent exclusive attempt fails.
475 */
476 success = lck_rw_try_lock_shared(<_rwlock);
477 T_ASSERT_NOTNULL(success, "Two shared try locks on a previously unheld rwlock should succeed");
478 success = lck_rw_try_lock_shared(<_rwlock);
479 T_ASSERT_NOTNULL(success, "Two shared try locks on a previously unheld rwlock should succeed");
480 success = lck_rw_try_lock_exclusive(<_rwlock);
481 T_ASSERT_NULL(success, "exclusive lock attempt on previously held lock should fail");
482 lck_rw_done(<_rwlock);
483 lck_rw_done(<_rwlock);
484
485 /*
486 * After regular shared grab, can trylock
487 * for shared but not for exclusive.
488 */
489 lck_rw_lock_shared(<_rwlock);
490 success = lck_rw_try_lock_shared(<_rwlock);
491 T_ASSERT_NOTNULL(success, "After regular shared grab another shared try lock should succeed.");
492 success = lck_rw_try_lock_exclusive(<_rwlock);
493 T_ASSERT_NULL(success, "After regular shared grab an exclusive lock attempt should fail.");
494 lck_rw_done(<_rwlock);
495 lck_rw_done(<_rwlock);
496
497 /*
498 * An exclusive try lock succeeds, subsequent shared and exclusive
499 * attempts fail.
500 */
501 success = lck_rw_try_lock_exclusive(<_rwlock);
502 T_ASSERT_NOTNULL(success, "An exclusive try lock should succeed");
503 success = lck_rw_try_lock_shared(<_rwlock);
504 T_ASSERT_NULL(success, "try lock in shared mode attempt after an exclusive grab should fail");
505 success = lck_rw_try_lock_exclusive(<_rwlock);
506 T_ASSERT_NULL(success, "try lock in exclusive mode attempt after an exclusive grab should fail");
507 lck_rw_done(<_rwlock);
508
509 /*
510 * After regular exclusive grab, neither kind of trylock succeeds.
511 */
512 lck_rw_lock_exclusive(<_rwlock);
513 success = lck_rw_try_lock_shared(<_rwlock);
514 T_ASSERT_NULL(success, "After regular exclusive grab, shared trylock should not succeed");
515 success = lck_rw_try_lock_exclusive(<_rwlock);
516 T_ASSERT_NULL(success, "After regular exclusive grab, exclusive trylock should not succeed");
517 lck_rw_done(<_rwlock);
518
519 /*
520 * First spin lock attempts succeed, second attempts fail.
521 */
522 success = hw_lock_try(<_hw_lock, LCK_GRP_NULL);
523 T_ASSERT_NOTNULL(success, "First spin lock attempts should succeed");
524 success = hw_lock_try(<_hw_lock, LCK_GRP_NULL);
525 T_ASSERT_NULL(success, "Second attempt to spin lock should fail");
526 hw_lock_unlock(<_hw_lock);
527
528 hw_lock_lock(<_hw_lock, LCK_GRP_NULL);
529 success = hw_lock_try(<_hw_lock, LCK_GRP_NULL);
530 T_ASSERT_NULL(success, "After taking spin lock, trylock attempt should fail");
531 hw_lock_unlock(<_hw_lock);
532
533 lt_reset();
534 lt_thread_lock_grabbed = false;
535 lt_thread_lock_success = true;
536 lt_target_done_threads = 1;
537 OSMemoryBarrier();
538 lt_start_trylock_thread(lt_trylock_hw_lock_with_to);
539 success = hw_lock_to(<_hw_lock, &hw_lock_test_give_up_policy, LCK_GRP_NULL);
540 T_ASSERT_NOTNULL(success, "First spin lock with timeout should succeed");
541 if (real_ncpus == 1) {
542 mp_enable_preemption(); /* if we re-enable preemption, the other thread can timeout and exit */
543 }
544 OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed);
545 lt_wait_for_lock_test_threads();
546 T_ASSERT_NULL(lt_thread_lock_success, "Second spin lock with timeout should fail and timeout");
547 if (real_ncpus == 1) {
548 mp_disable_preemption(); /* don't double-enable when we unlock */
549 }
550 hw_lock_unlock(<_hw_lock);
551
552 lt_reset();
553 lt_thread_lock_grabbed = false;
554 lt_thread_lock_success = true;
555 lt_target_done_threads = 1;
556 OSMemoryBarrier();
557 lt_start_trylock_thread(lt_trylock_hw_lock_with_to);
558 hw_lock_lock(<_hw_lock, LCK_GRP_NULL);
559 if (real_ncpus == 1) {
560 mp_enable_preemption(); /* if we re-enable preemption, the other thread can timeout and exit */
561 }
562 OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed);
563 lt_wait_for_lock_test_threads();
564 T_ASSERT_NULL(lt_thread_lock_success, "after taking a spin lock, lock attempt with timeout should fail");
565 if (real_ncpus == 1) {
566 mp_disable_preemption(); /* don't double-enable when we unlock */
567 }
568 hw_lock_unlock(<_hw_lock);
569
570 success = lck_spin_try_lock(<_lck_spin_t);
571 T_ASSERT_NOTNULL(success, "spin trylock of previously unheld lock should succeed");
572 success = lck_spin_try_lock(<_lck_spin_t);
573 T_ASSERT_NULL(success, "spin trylock attempt of previously held lock (with trylock) should fail");
574 lck_spin_unlock(<_lck_spin_t);
575
576 lt_reset();
577 lt_thread_lock_grabbed = false;
578 lt_thread_lock_success = true;
579 lt_target_done_threads = 1;
580 lt_start_trylock_thread(lt_trylock_spin_try_lock);
581 lck_spin_lock(<_lck_spin_t);
582 if (real_ncpus == 1) {
583 mp_enable_preemption(); /* if we re-enable preemption, the other thread can timeout and exit */
584 }
585 OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed);
586 lt_wait_for_lock_test_threads();
587 T_ASSERT_NULL(lt_thread_lock_success, "spin trylock attempt of previously held lock should fail");
588 if (real_ncpus == 1) {
589 mp_disable_preemption(); /* don't double-enable when we unlock */
590 }
591 lck_spin_unlock(<_lck_spin_t);
592
593 return KERN_SUCCESS;
594 }
595
596 static void
lt_thread(void * arg,wait_result_t wres __unused)597 lt_thread(void *arg, wait_result_t wres __unused)
598 {
599 void (*func)(void) = (void (*)(void))arg;
600 uint32_t i;
601
602 for (i = 0; i < LOCK_TEST_ITERATIONS; i++) {
603 func();
604 }
605
606 OSIncrementAtomic((volatile SInt32*) <_done_threads);
607 }
608
609 static void
lt_start_lock_thread(thread_continue_t func)610 lt_start_lock_thread(thread_continue_t func)
611 {
612 thread_t thread;
613 kern_return_t kr;
614
615 kr = kernel_thread_start(lt_thread, func, &thread);
616 assert(kr == KERN_SUCCESS);
617
618 thread_deallocate(thread);
619 }
620
621 #if __AMP__
622 static void
lt_bound_thread(void * arg,wait_result_t wres __unused)623 lt_bound_thread(void *arg, wait_result_t wres __unused)
624 {
625 void (*func)(void) = (void (*)(void))arg;
626
627 int cpuid = OSIncrementAtomic((volatile SInt32 *)<_cpu_bind_id);
628
629 processor_t processor = processor_list;
630 while ((processor != NULL) && (processor->cpu_id != cpuid)) {
631 processor = processor->processor_list;
632 }
633
634 if (processor != NULL) {
635 thread_bind(processor);
636 }
637
638 thread_block(THREAD_CONTINUE_NULL);
639
640 func();
641
642 OSIncrementAtomic((volatile SInt32*) <_done_threads);
643 }
644
645 static void
lt_e_thread(void * arg,wait_result_t wres __unused)646 lt_e_thread(void *arg, wait_result_t wres __unused)
647 {
648 void (*func)(void) = (void (*)(void))arg;
649
650 thread_t thread = current_thread();
651
652 thread_bind_cluster_type(thread, 'e', false);
653
654 func();
655
656 OSIncrementAtomic((volatile SInt32*) <_done_threads);
657 }
658
659 static void
lt_p_thread(void * arg,wait_result_t wres __unused)660 lt_p_thread(void *arg, wait_result_t wres __unused)
661 {
662 void (*func)(void) = (void (*)(void))arg;
663
664 thread_t thread = current_thread();
665
666 thread_bind_cluster_type(thread, 'p', false);
667
668 func();
669
670 OSIncrementAtomic((volatile SInt32*) <_done_threads);
671 }
672
673 static void
lt_start_lock_thread_e(thread_continue_t func)674 lt_start_lock_thread_e(thread_continue_t func)
675 {
676 thread_t thread;
677 kern_return_t kr;
678
679 kr = kernel_thread_start(lt_e_thread, func, &thread);
680 assert(kr == KERN_SUCCESS);
681
682 thread_deallocate(thread);
683 }
684
685 static void
lt_start_lock_thread_p(thread_continue_t func)686 lt_start_lock_thread_p(thread_continue_t func)
687 {
688 thread_t thread;
689 kern_return_t kr;
690
691 kr = kernel_thread_start(lt_p_thread, func, &thread);
692 assert(kr == KERN_SUCCESS);
693
694 thread_deallocate(thread);
695 }
696
697 static void
lt_start_lock_thread_bound(thread_continue_t func)698 lt_start_lock_thread_bound(thread_continue_t func)
699 {
700 thread_t thread;
701 kern_return_t kr;
702
703 kr = kernel_thread_start(lt_bound_thread, func, &thread);
704 assert(kr == KERN_SUCCESS);
705
706 thread_deallocate(thread);
707 }
708 #endif
709
710 static kern_return_t
lt_test_locks()711 lt_test_locks()
712 {
713 #if SCHED_HYGIENE_DEBUG
714 /*
715 * When testing, the preemption disable threshold may be hit (for
716 * example when testing a lock timeout). To avoid this, the preemption
717 * disable measurement is temporarily disabled during lock testing.
718 */
719 int old_mode = sched_preemption_disable_debug_mode;
720 if (old_mode == SCHED_HYGIENE_MODE_PANIC) {
721 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
722 }
723 #endif /* SCHED_HYGIENE_DEBUG */
724
725 kern_return_t kr = KERN_SUCCESS;
726 lck_grp_attr_t *lga = lck_grp_attr_alloc_init();
727 lck_grp_t *lg = lck_grp_alloc_init("lock test", lga);
728
729 lck_mtx_init(<_mtx, lg, LCK_ATTR_NULL);
730 lck_rw_init(<_rwlock, lg, LCK_ATTR_NULL);
731 lck_spin_init(<_lck_spin_t, lg, LCK_ATTR_NULL);
732 hw_lock_init(<_hw_lock);
733
734 T_LOG("Testing locks.");
735
736 /* Try locks (custom) */
737 lt_reset();
738
739 T_LOG("Running try lock test.");
740 kr = lt_test_trylocks();
741 T_EXPECT_NULL(kr, "try lock test failed.");
742
743 /* Uncontended mutex */
744 T_LOG("Running uncontended mutex test.");
745 lt_reset();
746 lt_target_done_threads = 1;
747 lt_start_lock_thread(lt_grab_mutex);
748 lt_wait_for_lock_test_threads();
749 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
750 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
751
752 /* Contended mutex:try locks*/
753 T_LOG("Running contended mutex test.");
754 lt_reset();
755 lt_target_done_threads = 3;
756 lt_start_lock_thread(lt_grab_mutex);
757 lt_start_lock_thread(lt_grab_mutex);
758 lt_start_lock_thread(lt_grab_mutex);
759 lt_wait_for_lock_test_threads();
760 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
761 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
762
763 /* Contended mutex: try locks*/
764 T_LOG("Running contended mutex trylock test.");
765 lt_reset();
766 lt_target_done_threads = 3;
767 lt_start_lock_thread(lt_grab_mutex_with_try);
768 lt_start_lock_thread(lt_grab_mutex_with_try);
769 lt_start_lock_thread(lt_grab_mutex_with_try);
770 lt_wait_for_lock_test_threads();
771 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
772 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
773
774 /* Uncontended exclusive rwlock */
775 T_LOG("Running uncontended exclusive rwlock test.");
776 lt_reset();
777 lt_target_done_threads = 1;
778 lt_start_lock_thread(lt_grab_rw_exclusive);
779 lt_wait_for_lock_test_threads();
780 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
781 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
782
783 /* Uncontended shared rwlock */
784
785 /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840)
786 * T_LOG("Running uncontended shared rwlock test.");
787 * lt_reset();
788 * lt_target_done_threads = 1;
789 * lt_start_lock_thread(lt_grab_rw_shared);
790 * lt_wait_for_lock_test_threads();
791 * T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
792 * T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
793 */
794
795 /* Contended exclusive rwlock */
796 T_LOG("Running contended exclusive rwlock test.");
797 lt_reset();
798 lt_target_done_threads = 3;
799 lt_start_lock_thread(lt_grab_rw_exclusive);
800 lt_start_lock_thread(lt_grab_rw_exclusive);
801 lt_start_lock_thread(lt_grab_rw_exclusive);
802 lt_wait_for_lock_test_threads();
803 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
804 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
805
806 /* One shared, two exclusive */
807 /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840)
808 * T_LOG("Running test with one shared and two exclusive rw lock threads.");
809 * lt_reset();
810 * lt_target_done_threads = 3;
811 * lt_start_lock_thread(lt_grab_rw_shared);
812 * lt_start_lock_thread(lt_grab_rw_exclusive);
813 * lt_start_lock_thread(lt_grab_rw_exclusive);
814 * lt_wait_for_lock_test_threads();
815 * T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
816 * T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
817 */
818
819 /* Four shared */
820 /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840)
821 * T_LOG("Running test with four shared holders.");
822 * lt_reset();
823 * lt_target_done_threads = 4;
824 * lt_start_lock_thread(lt_grab_rw_shared);
825 * lt_start_lock_thread(lt_grab_rw_shared);
826 * lt_start_lock_thread(lt_grab_rw_shared);
827 * lt_start_lock_thread(lt_grab_rw_shared);
828 * lt_wait_for_lock_test_threads();
829 * T_EXPECT_LE_UINT(lt_max_holders, 4, NULL);
830 */
831
832 /* Three doing upgrades and downgrades */
833 T_LOG("Running test with threads upgrading and downgrading.");
834 lt_reset();
835 lt_target_done_threads = 3;
836 lt_start_lock_thread(lt_upgrade_downgrade_rw);
837 lt_start_lock_thread(lt_upgrade_downgrade_rw);
838 lt_start_lock_thread(lt_upgrade_downgrade_rw);
839 lt_wait_for_lock_test_threads();
840 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
841 T_EXPECT_LE_UINT(lt_max_holders, 3, NULL);
842 T_EXPECT_EQ_UINT(lt_max_upgrade_holders, 1, NULL);
843
844 /* Uncontended - exclusive trylocks */
845 T_LOG("Running test with single thread doing exclusive rwlock trylocks.");
846 lt_reset();
847 lt_target_done_threads = 1;
848 lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
849 lt_wait_for_lock_test_threads();
850 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
851 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
852
853 /* Uncontended - shared trylocks */
854 /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840)
855 * T_LOG("Running test with single thread doing shared rwlock trylocks.");
856 * lt_reset();
857 * lt_target_done_threads = 1;
858 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
859 * lt_wait_for_lock_test_threads();
860 * T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
861 * T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
862 */
863
864 /* Three doing exclusive trylocks */
865 T_LOG("Running test with threads doing exclusive rwlock trylocks.");
866 lt_reset();
867 lt_target_done_threads = 3;
868 lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
869 lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
870 lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
871 lt_wait_for_lock_test_threads();
872 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
873 T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL);
874
875 /* Three doing shared trylocks */
876 /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840)
877 * T_LOG("Running test with threads doing shared rwlock trylocks.");
878 * lt_reset();
879 * lt_target_done_threads = 3;
880 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
881 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
882 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
883 * lt_wait_for_lock_test_threads();
884 * T_EXPECT_LE_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
885 * T_EXPECT_LE_UINT(lt_max_holders, 3, NULL);
886 */
887
888 /* Three doing various trylocks */
889 /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840)
890 * T_LOG("Running test with threads doing mixed rwlock trylocks.");
891 * lt_reset();
892 * lt_target_done_threads = 4;
893 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
894 * lt_start_lock_thread(lt_grab_rw_shared_with_try);
895 * lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
896 * lt_start_lock_thread(lt_grab_rw_exclusive_with_try);
897 * lt_wait_for_lock_test_threads();
898 * T_EXPECT_LE_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
899 * T_EXPECT_LE_UINT(lt_max_holders, 2, NULL);
900 */
901
902 /* HW locks */
903 T_LOG("Running test with hw_lock_lock()");
904 lt_reset();
905 lt_target_done_threads = 3;
906 lt_start_lock_thread(lt_grab_hw_lock);
907 lt_start_lock_thread(lt_grab_hw_lock);
908 lt_start_lock_thread(lt_grab_hw_lock);
909 lt_wait_for_lock_test_threads();
910 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
911
912 #if __AMP__
913 /* Ticket locks stress test */
914 T_LOG("Running Ticket locks stress test with lck_ticket_lock()");
915 extern unsigned int real_ncpus;
916 lck_grp_init(<_ticket_grp, "ticket lock stress", LCK_GRP_ATTR_NULL);
917 lck_ticket_init(<_ticket_lock, <_ticket_grp);
918 lt_reset();
919 lt_target_done_threads = real_ncpus;
920 for (processor_t processor = processor_list; processor != NULL; processor = processor->processor_list) {
921 lt_start_lock_thread_bound(lt_stress_ticket_lock);
922 }
923 lt_wait_for_lock_test_threads();
924 bool starvation = false;
925 uint total_local_count = 0;
926 for (processor_t processor = processor_list; processor != NULL; processor = processor->processor_list) {
927 starvation = starvation || (lt_stress_local_counters[processor->cpu_id] < 10);
928 total_local_count += lt_stress_local_counters[processor->cpu_id];
929 }
930 if (total_local_count != lt_counter) {
931 T_FAIL("Lock failure\n");
932 } else if (starvation) {
933 T_FAIL("Lock starvation found\n");
934 } else {
935 T_PASS("Ticket locks stress test with lck_ticket_lock()");
936 }
937
938 /* AMP ticket locks stress test */
939 T_LOG("Running AMP Ticket locks stress test bound to clusters with lck_ticket_lock()");
940 lt_reset();
941 lt_target_done_threads = real_ncpus;
942 for (processor_t processor = processor_list; processor != NULL; processor = processor->processor_list) {
943 processor_set_t pset = processor->processor_set;
944 if (pset->pset_cluster_type == PSET_AMP_P) {
945 lt_start_lock_thread_p(lt_stress_ticket_lock);
946 } else if (pset->pset_cluster_type == PSET_AMP_E) {
947 lt_start_lock_thread_e(lt_stress_ticket_lock);
948 } else {
949 lt_start_lock_thread(lt_stress_ticket_lock);
950 }
951 }
952 lt_wait_for_lock_test_threads();
953 #endif
954
955 /* HW locks: trylocks */
956 T_LOG("Running test with hw_lock_try()");
957 lt_reset();
958 lt_target_done_threads = 3;
959 lt_start_lock_thread(lt_grab_hw_lock_with_try);
960 lt_start_lock_thread(lt_grab_hw_lock_with_try);
961 lt_start_lock_thread(lt_grab_hw_lock_with_try);
962 lt_wait_for_lock_test_threads();
963 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
964
965 /* HW locks: with timeout */
966 T_LOG("Running test with hw_lock_to()");
967 lt_reset();
968 lt_target_done_threads = 3;
969 lt_start_lock_thread(lt_grab_hw_lock_with_to);
970 lt_start_lock_thread(lt_grab_hw_lock_with_to);
971 lt_start_lock_thread(lt_grab_hw_lock_with_to);
972 lt_wait_for_lock_test_threads();
973 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
974
975 /* Spin locks */
976 T_LOG("Running test with lck_spin_lock()");
977 lt_reset();
978 lt_target_done_threads = 3;
979 lt_start_lock_thread(lt_grab_spin_lock);
980 lt_start_lock_thread(lt_grab_spin_lock);
981 lt_start_lock_thread(lt_grab_spin_lock);
982 lt_wait_for_lock_test_threads();
983 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
984
985 /* Spin locks: trylocks */
986 T_LOG("Running test with lck_spin_try_lock()");
987 lt_reset();
988 lt_target_done_threads = 3;
989 lt_start_lock_thread(lt_grab_spin_lock_with_try);
990 lt_start_lock_thread(lt_grab_spin_lock_with_try);
991 lt_start_lock_thread(lt_grab_spin_lock_with_try);
992 lt_wait_for_lock_test_threads();
993 T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL);
994
995 #if SCHED_HYGIENE_DEBUG
996 sched_preemption_disable_debug_mode = old_mode;
997 #endif /* SCHED_HYGIENE_DEBUG */
998
999 return KERN_SUCCESS;
1000 }
1001
1002 #define MT_MAX_ARGS 8
1003 #define MT_INITIAL_VALUE 0xfeedbeef
1004 #define MT_W_VAL (0x00000000feedbeefULL) /* Drop in zeros */
1005 #define MT_S_VAL (0xfffffffffeedbeefULL) /* High bit is 1, so sign-extends as negative */
1006 #define MT_L_VAL (((uint64_t)MT_INITIAL_VALUE) | (((uint64_t)MT_INITIAL_VALUE) << 32)) /* Two back-to-back */
1007
1008 typedef void (*sy_munge_t)(void*);
1009
1010 #define MT_FUNC(x) #x, x
1011 struct munger_test {
1012 const char *mt_name;
1013 sy_munge_t mt_func;
1014 uint32_t mt_in_words;
1015 uint32_t mt_nout;
1016 uint64_t mt_expected[MT_MAX_ARGS];
1017 } munger_tests[] = {
1018 {MT_FUNC(munge_w), 1, 1, {MT_W_VAL}},
1019 {MT_FUNC(munge_ww), 2, 2, {MT_W_VAL, MT_W_VAL}},
1020 {MT_FUNC(munge_www), 3, 3, {MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1021 {MT_FUNC(munge_wwww), 4, 4, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1022 {MT_FUNC(munge_wwwww), 5, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1023 {MT_FUNC(munge_wwwwww), 6, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1024 {MT_FUNC(munge_wwwwwww), 7, 7, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1025 {MT_FUNC(munge_wwwwwwww), 8, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1026 {MT_FUNC(munge_wl), 3, 2, {MT_W_VAL, MT_L_VAL}},
1027 {MT_FUNC(munge_wwl), 4, 3, {MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1028 {MT_FUNC(munge_wwlll), 8, 5, {MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}},
1029 {MT_FUNC(munge_wlw), 4, 3, {MT_W_VAL, MT_L_VAL, MT_W_VAL}},
1030 {MT_FUNC(munge_wlwwwll), 10, 7, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}},
1031 {MT_FUNC(munge_wlwwwllw), 11, 8, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL}},
1032 {MT_FUNC(munge_wlwwlwlw), 11, 8, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}},
1033 {MT_FUNC(munge_wll), 5, 3, {MT_W_VAL, MT_L_VAL, MT_L_VAL}},
1034 {MT_FUNC(munge_wlll), 7, 4, {MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}},
1035 {MT_FUNC(munge_wllwwll), 11, 7, {MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}},
1036 {MT_FUNC(munge_wwwlw), 6, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}},
1037 {MT_FUNC(munge_wwwlww), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL}},
1038 {MT_FUNC(munge_wwwlwww), 8, 7, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1039 {MT_FUNC(munge_wwwl), 5, 4, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1040 {MT_FUNC(munge_wwwwlw), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}},
1041 {MT_FUNC(munge_wwwwllww), 10, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL}},
1042 {MT_FUNC(munge_wwwwl), 6, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1043 {MT_FUNC(munge_wwwwwl), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1044 {MT_FUNC(munge_wwwwwlww), 9, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL}},
1045 {MT_FUNC(munge_wwwwwllw), 10, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL}},
1046 {MT_FUNC(munge_wwwwwlll), 11, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}},
1047 {MT_FUNC(munge_wwwwwwl), 8, 7, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1048 {MT_FUNC(munge_wwwwwwlw), 9, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}},
1049 {MT_FUNC(munge_wwwwwwll), 10, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}},
1050 {MT_FUNC(munge_wsw), 3, 3, {MT_W_VAL, MT_S_VAL, MT_W_VAL}},
1051 {MT_FUNC(munge_wws), 3, 3, {MT_W_VAL, MT_W_VAL, MT_S_VAL}},
1052 {MT_FUNC(munge_wwwsw), 5, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_S_VAL, MT_W_VAL}},
1053 {MT_FUNC(munge_llllll), 12, 6, {MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}},
1054 {MT_FUNC(munge_llll), 8, 4, {MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}},
1055 {MT_FUNC(munge_l), 2, 1, {MT_L_VAL}},
1056 {MT_FUNC(munge_lw), 3, 2, {MT_L_VAL, MT_W_VAL}},
1057 {MT_FUNC(munge_lwww), 5, 4, {MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1058 {MT_FUNC(munge_lwwwwwww), 9, 8, {MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}},
1059 {MT_FUNC(munge_wlwwwl), 8, 6, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}},
1060 {MT_FUNC(munge_wwlwwwl), 9, 7, {MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}
1061 };
1062
1063 #define MT_TEST_COUNT (sizeof(munger_tests) / sizeof(struct munger_test))
1064
1065 static void
mt_reset(uint32_t in_words,size_t total_size,uint32_t * data)1066 mt_reset(uint32_t in_words, size_t total_size, uint32_t *data)
1067 {
1068 uint32_t i;
1069
1070 for (i = 0; i < in_words; i++) {
1071 data[i] = MT_INITIAL_VALUE;
1072 }
1073
1074 if (in_words * sizeof(uint32_t) < total_size) {
1075 bzero(&data[in_words], total_size - in_words * sizeof(uint32_t));
1076 }
1077 }
1078
1079 static void
mt_test_mungers()1080 mt_test_mungers()
1081 {
1082 uint64_t data[MT_MAX_ARGS];
1083 uint32_t i, j;
1084
1085 for (i = 0; i < MT_TEST_COUNT; i++) {
1086 struct munger_test *test = &munger_tests[i];
1087 int pass = 1;
1088
1089 T_LOG("Testing %s", test->mt_name);
1090
1091 mt_reset(test->mt_in_words, sizeof(data), (uint32_t*)data);
1092 test->mt_func(data);
1093
1094 for (j = 0; j < test->mt_nout; j++) {
1095 if (data[j] != test->mt_expected[j]) {
1096 T_FAIL("Index %d: expected %llx, got %llx.", j, test->mt_expected[j], data[j]);
1097 pass = 0;
1098 }
1099 }
1100 if (pass) {
1101 T_PASS(test->mt_name);
1102 }
1103 }
1104 }
1105
1106 #if defined(HAS_APPLE_PAC)
1107
1108
1109 kern_return_t
arm64_ropjop_test()1110 arm64_ropjop_test()
1111 {
1112 T_LOG("Testing ROP/JOP");
1113
1114 /* how is ROP/JOP configured */
1115 boolean_t config_rop_enabled = TRUE;
1116 boolean_t config_jop_enabled = TRUE;
1117
1118
1119 if (config_jop_enabled) {
1120 /* jop key */
1121 uint64_t apiakey_hi = __builtin_arm_rsr64("APIAKEYHI_EL1");
1122 uint64_t apiakey_lo = __builtin_arm_rsr64("APIAKEYLO_EL1");
1123
1124 T_EXPECT(apiakey_hi != 0 && apiakey_lo != 0, NULL);
1125 }
1126
1127 if (config_rop_enabled) {
1128 /* rop key */
1129 uint64_t apibkey_hi = __builtin_arm_rsr64("APIBKEYHI_EL1");
1130 uint64_t apibkey_lo = __builtin_arm_rsr64("APIBKEYLO_EL1");
1131
1132 T_EXPECT(apibkey_hi != 0 && apibkey_lo != 0, NULL);
1133
1134 /* sign a KVA (the address of this function) */
1135 uint64_t kva_signed = (uint64_t) ptrauth_sign_unauthenticated((void *)&config_rop_enabled, ptrauth_key_asib, 0);
1136
1137 /* assert it was signed (changed) */
1138 T_EXPECT(kva_signed != (uint64_t)&config_rop_enabled, NULL);
1139
1140 /* authenticate the newly signed KVA */
1141 uint64_t kva_authed = (uint64_t) ml_auth_ptr_unchecked((void *)kva_signed, ptrauth_key_asib, 0);
1142
1143 /* assert the authed KVA is the original KVA */
1144 T_EXPECT(kva_authed == (uint64_t)&config_rop_enabled, NULL);
1145
1146 /* corrupt a signed ptr, auth it, ensure auth failed */
1147 uint64_t kva_corrupted = kva_signed ^ 1;
1148
1149 /* authenticate the corrupted pointer */
1150 kva_authed = (uint64_t) ml_auth_ptr_unchecked((void *)kva_corrupted, ptrauth_key_asib, 0);
1151
1152 /* when AuthIB fails, bits 63:62 will be set to 2'b10 */
1153 uint64_t auth_fail_mask = 3ULL << 61;
1154 uint64_t authib_fail = 2ULL << 61;
1155
1156 /* assert the failed authIB of corrupted pointer is tagged */
1157 T_EXPECT((kva_authed & auth_fail_mask) == authib_fail, NULL);
1158 }
1159
1160 return KERN_SUCCESS;
1161 }
1162 #endif /* defined(HAS_APPLE_PAC) */
1163
1164 #if __ARM_PAN_AVAILABLE__
1165
1166 struct pan_test_thread_args {
1167 volatile bool join;
1168 };
1169
1170 static void
arm64_pan_test_thread(void * arg,wait_result_t __unused wres)1171 arm64_pan_test_thread(void *arg, wait_result_t __unused wres)
1172 {
1173 T_ASSERT(__builtin_arm_rsr("pan") != 0, NULL);
1174
1175 struct pan_test_thread_args *args = arg;
1176
1177 for (processor_t p = processor_list; p != NULL; p = p->processor_list) {
1178 thread_bind(p);
1179 thread_block(THREAD_CONTINUE_NULL);
1180 kprintf("Running PAN test on cpu %d\n", p->cpu_id);
1181 arm64_pan_test();
1182 }
1183
1184 /* unbind thread from specific cpu */
1185 thread_bind(PROCESSOR_NULL);
1186 thread_block(THREAD_CONTINUE_NULL);
1187
1188 while (!args->join) {
1189 ;
1190 }
1191
1192 thread_wakeup(args);
1193 }
1194
1195 kern_return_t
arm64_late_pan_test()1196 arm64_late_pan_test()
1197 {
1198 thread_t thread;
1199 kern_return_t kr;
1200
1201 struct pan_test_thread_args args;
1202 args.join = false;
1203
1204 kr = kernel_thread_start(arm64_pan_test_thread, &args, &thread);
1205 assert(kr == KERN_SUCCESS);
1206
1207 thread_deallocate(thread);
1208
1209 assert_wait(&args, THREAD_UNINT);
1210 args.join = true;
1211 thread_block(THREAD_CONTINUE_NULL);
1212 return KERN_SUCCESS;
1213 }
1214
1215 // Disable KASAN checking for PAN tests as the fixed commpage address doesn't have a shadow mapping
1216
1217 static NOKASAN bool
arm64_pan_test_pan_enabled_fault_handler(arm_saved_state_t * state)1218 arm64_pan_test_pan_enabled_fault_handler(arm_saved_state_t * state)
1219 {
1220 bool retval = false;
1221 uint32_t esr = get_saved_state_esr(state);
1222 esr_exception_class_t class = ESR_EC(esr);
1223 fault_status_t fsc = ISS_IA_FSC(ESR_ISS(esr));
1224 uint32_t cpsr = get_saved_state_cpsr(state);
1225 uint64_t far = get_saved_state_far(state);
1226
1227 if ((class == ESR_EC_DABORT_EL1) && (fsc == FSC_PERMISSION_FAULT_L3) &&
1228 (cpsr & PSR64_PAN) &&
1229 ((esr & ISS_DA_WNR) ? mmu_kvtop_wpreflight(far) : mmu_kvtop(far))) {
1230 ++pan_exception_level;
1231 // read the user-accessible value to make sure
1232 // pan is enabled and produces a 2nd fault from
1233 // the exception handler
1234 if (pan_exception_level == 1) {
1235 ml_expect_fault_begin(arm64_pan_test_pan_enabled_fault_handler, far);
1236 pan_fault_value = *(volatile char *)far;
1237 ml_expect_fault_end();
1238 __builtin_arm_wsr("pan", 1); // turn PAN back on after the nested exception cleared it for this context
1239 }
1240 // this fault address is used for PAN test
1241 // disable PAN and rerun
1242 mask_saved_state_cpsr(state, 0, PSR64_PAN);
1243
1244 retval = true;
1245 }
1246
1247 return retval;
1248 }
1249
1250 static NOKASAN bool
arm64_pan_test_pan_disabled_fault_handler(arm_saved_state_t * state)1251 arm64_pan_test_pan_disabled_fault_handler(arm_saved_state_t * state)
1252 {
1253 bool retval = false;
1254 uint32_t esr = get_saved_state_esr(state);
1255 esr_exception_class_t class = ESR_EC(esr);
1256 fault_status_t fsc = ISS_IA_FSC(ESR_ISS(esr));
1257 uint32_t cpsr = get_saved_state_cpsr(state);
1258
1259 if ((class == ESR_EC_DABORT_EL1) && (fsc == FSC_PERMISSION_FAULT_L3) &&
1260 !(cpsr & PSR64_PAN)) {
1261 ++pan_exception_level;
1262 // On an exception taken from a PAN-disabled context, verify
1263 // that PAN is re-enabled for the exception handler and that
1264 // accessing the test address produces a PAN fault.
1265 ml_expect_fault_begin(arm64_pan_test_pan_enabled_fault_handler, pan_test_addr);
1266 pan_fault_value = *(volatile char *)pan_test_addr;
1267 ml_expect_fault_end();
1268 __builtin_arm_wsr("pan", 1); // turn PAN back on after the nested exception cleared it for this context
1269 add_saved_state_pc(state, 4);
1270
1271 retval = true;
1272 }
1273
1274 return retval;
1275 }
1276
1277 NOKASAN kern_return_t
arm64_pan_test()1278 arm64_pan_test()
1279 {
1280 bool values_match = false;
1281 vm_offset_t priv_addr = 0;
1282
1283 T_LOG("Testing PAN.");
1284
1285
1286 T_ASSERT((__builtin_arm_rsr("SCTLR_EL1") & SCTLR_PAN_UNCHANGED) == 0, "SCTLR_EL1.SPAN must be cleared");
1287
1288 T_ASSERT(__builtin_arm_rsr("pan") != 0, NULL);
1289
1290 pan_exception_level = 0;
1291 pan_fault_value = 0xDE;
1292
1293 // Create an empty pmap, so we can map a user-accessible page
1294 pmap_t pmap = pmap_create_options(NULL, 0, PMAP_CREATE_64BIT);
1295 T_ASSERT(pmap != NULL, NULL);
1296
1297 // Get a physical page to back the mapping
1298 vm_page_t vm_page = vm_page_grab();
1299 T_ASSERT(vm_page != VM_PAGE_NULL, NULL);
1300 ppnum_t pn = VM_PAGE_GET_PHYS_PAGE(vm_page);
1301 pmap_paddr_t pa = ptoa(pn);
1302
1303 // Write to the underlying physical page through the physical aperture
1304 // so we can test against a known value
1305 priv_addr = phystokv((pmap_paddr_t)pa);
1306 *(volatile char *)priv_addr = 0xAB;
1307
1308 // Map the page in the user address space at some, non-zero address
1309 pan_test_addr = PAGE_SIZE;
1310 pmap_enter(pmap, pan_test_addr, pn, VM_PROT_READ, VM_PROT_READ, 0, true, PMAP_MAPPING_TYPE_INFER);
1311
1312 // Context-switch with PAN disabled is prohibited; prevent test logging from
1313 // triggering a voluntary context switch.
1314 mp_disable_preemption();
1315
1316 // Insert the user's pmap root table pointer in TTBR0
1317 pmap_t old_pmap = vm_map_pmap(current_thread()->map);
1318 pmap_switch(pmap);
1319
1320 // Below should trigger a PAN exception as pan_test_addr is accessible
1321 // in user mode
1322 // The exception handler, upon recognizing the fault address is pan_test_addr,
1323 // will disable PAN and rerun this instruction successfully
1324 ml_expect_fault_begin(arm64_pan_test_pan_enabled_fault_handler, pan_test_addr);
1325 values_match = (*(volatile char *)pan_test_addr == *(volatile char *)priv_addr);
1326 ml_expect_fault_end();
1327 T_ASSERT(values_match, NULL);
1328
1329 T_ASSERT(pan_exception_level == 2, NULL);
1330
1331 T_ASSERT(__builtin_arm_rsr("pan") == 0, NULL);
1332
1333 T_ASSERT(pan_fault_value == *(char *)priv_addr, NULL);
1334
1335 pan_exception_level = 0;
1336 pan_fault_value = 0xAD;
1337 pan_ro_addr = (vm_offset_t) &pan_ro_value;
1338
1339 // Force a permission fault while PAN is disabled to make sure PAN is
1340 // re-enabled during the exception handler.
1341 ml_expect_fault_begin(arm64_pan_test_pan_disabled_fault_handler, pan_ro_addr);
1342 *((volatile uint64_t*)pan_ro_addr) = 0xFEEDFACECAFECAFE;
1343 ml_expect_fault_end();
1344
1345 T_ASSERT(pan_exception_level == 2, NULL);
1346
1347 T_ASSERT(__builtin_arm_rsr("pan") == 0, NULL);
1348
1349 T_ASSERT(pan_fault_value == *(char *)priv_addr, NULL);
1350
1351 pmap_switch(old_pmap);
1352
1353 pan_ro_addr = 0;
1354
1355 __builtin_arm_wsr("pan", 1);
1356
1357 mp_enable_preemption();
1358
1359 pmap_remove(pmap, pan_test_addr, pan_test_addr + PAGE_SIZE);
1360 pan_test_addr = 0;
1361
1362 vm_page_lock_queues();
1363 vm_page_free(vm_page);
1364 vm_page_unlock_queues();
1365 pmap_destroy(pmap);
1366
1367 return KERN_SUCCESS;
1368 }
1369 #endif /* __ARM_PAN_AVAILABLE__ */
1370
1371
1372 kern_return_t
arm64_lock_test()1373 arm64_lock_test()
1374 {
1375 return lt_test_locks();
1376 }
1377
1378 kern_return_t
arm64_munger_test()1379 arm64_munger_test()
1380 {
1381 mt_test_mungers();
1382 return 0;
1383 }
1384
1385 #if defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST)
1386 SECURITY_READ_ONLY_LATE(uint64_t) ctrr_ro_test;
1387 uint64_t ctrr_nx_test = 0xd65f03c0; /* RET */
1388 volatile uint64_t ctrr_exception_esr;
1389 vm_offset_t ctrr_test_va;
1390 vm_offset_t ctrr_test_page;
1391
1392 kern_return_t
ctrr_test(void)1393 ctrr_test(void)
1394 {
1395 processor_t p;
1396 boolean_t ctrr_disable = FALSE;
1397
1398 PE_parse_boot_argn("-unsafe_kernel_text", &ctrr_disable, sizeof(ctrr_disable));
1399
1400 #if CONFIG_CSR_FROM_DT
1401 if (csr_unsafe_kernel_text) {
1402 ctrr_disable = TRUE;
1403 }
1404 #endif /* CONFIG_CSR_FROM_DT */
1405
1406 if (ctrr_disable) {
1407 T_LOG("Skipping CTRR test when -unsafe_kernel_text boot-arg present");
1408 return KERN_SUCCESS;
1409 }
1410
1411 T_LOG("Running CTRR test.");
1412
1413 for (p = processor_list; p != NULL; p = p->processor_list) {
1414 thread_bind(p);
1415 thread_block(THREAD_CONTINUE_NULL);
1416 T_LOG("Running CTRR test on cpu %d\n", p->cpu_id);
1417 ctrr_test_cpu();
1418 }
1419
1420 /* unbind thread from specific cpu */
1421 thread_bind(PROCESSOR_NULL);
1422 thread_block(THREAD_CONTINUE_NULL);
1423
1424 return KERN_SUCCESS;
1425 }
1426
1427 static bool
ctrr_test_ro_fault_handler(arm_saved_state_t * state)1428 ctrr_test_ro_fault_handler(arm_saved_state_t * state)
1429 {
1430 bool retval = false;
1431 uint32_t esr = get_saved_state_esr(state);
1432 esr_exception_class_t class = ESR_EC(esr);
1433 fault_status_t fsc = ISS_DA_FSC(ESR_ISS(esr));
1434
1435 if ((class == ESR_EC_DABORT_EL1) && (fsc == FSC_PERMISSION_FAULT_L3)) {
1436 ctrr_exception_esr = esr;
1437 add_saved_state_pc(state, 4);
1438 retval = true;
1439 }
1440
1441 return retval;
1442 }
1443
1444 static bool
ctrr_test_nx_fault_handler(arm_saved_state_t * state)1445 ctrr_test_nx_fault_handler(arm_saved_state_t * state)
1446 {
1447 bool retval = false;
1448 uint32_t esr = get_saved_state_esr(state);
1449 esr_exception_class_t class = ESR_EC(esr);
1450 fault_status_t fsc = ISS_IA_FSC(ESR_ISS(esr));
1451
1452 if ((class == ESR_EC_IABORT_EL1) && (fsc == FSC_PERMISSION_FAULT_L3)) {
1453 ctrr_exception_esr = esr;
1454 /* return to the instruction immediately after the call to NX page */
1455 set_saved_state_pc(state, get_saved_state_lr(state));
1456 retval = true;
1457 }
1458
1459 return retval;
1460 }
1461
1462 // Disable KASAN checking for CTRR tests as the test VA doesn't have a shadow mapping
1463
1464 /* test CTRR on a cpu, caller to bind thread to desired cpu */
1465 /* ctrr_test_page was reserved during bootstrap process */
1466 NOKASAN kern_return_t
ctrr_test_cpu(void)1467 ctrr_test_cpu(void)
1468 {
1469 ppnum_t ro_pn, nx_pn;
1470 uint64_t *ctrr_ro_test_ptr;
1471 void (*ctrr_nx_test_ptr)(void);
1472 kern_return_t kr;
1473 uint64_t prot = 0;
1474 extern vm_offset_t virtual_space_start;
1475
1476 /* ctrr read only region = [rorgn_begin_va, rorgn_end_va) */
1477
1478 vm_offset_t rorgn_begin_va = phystokv(__builtin_arm_rsr64("S3_4_C15_C2_3"));
1479 vm_offset_t rorgn_end_va = phystokv(__builtin_arm_rsr64("S3_4_C15_C2_4")) + 1;
1480 vm_offset_t ro_test_va = (vm_offset_t)&ctrr_ro_test;
1481 vm_offset_t nx_test_va = (vm_offset_t)&ctrr_nx_test;
1482
1483 T_EXPECT(rorgn_begin_va <= ro_test_va && ro_test_va < rorgn_end_va, "Expect ro_test_va to be inside the CTRR region");
1484 T_EXPECT((nx_test_va < rorgn_begin_va) ^ (nx_test_va >= rorgn_end_va), "Expect nx_test_va to be outside the CTRR region");
1485
1486 ro_pn = pmap_find_phys(kernel_pmap, ro_test_va);
1487 nx_pn = pmap_find_phys(kernel_pmap, nx_test_va);
1488 T_EXPECT(ro_pn && nx_pn, "Expect ro page number and nx page number to be non zero");
1489
1490 T_LOG("test virtual page: %p, ctrr_ro_test: %p, ctrr_nx_test: %p, ro_pn: %x, nx_pn: %x ",
1491 (void *)ctrr_test_page, &ctrr_ro_test, &ctrr_nx_test, ro_pn, nx_pn);
1492
1493 prot = pmap_get_arm64_prot(kernel_pmap, ctrr_test_page);
1494 T_EXPECT(~prot & ARM_TTE_VALID, "Expect ctrr_test_page to be unmapped");
1495
1496 T_LOG("Read only region test mapping virtual page %p to CTRR RO page number %d", ctrr_test_page, ro_pn);
1497 kr = pmap_enter(kernel_pmap, ctrr_test_page, ro_pn,
1498 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
1499 T_EXPECT(kr == KERN_SUCCESS, "Expect pmap_enter of RW mapping to succeed");
1500
1501 // assert entire mmu prot path (Hierarchical protection model) is NOT RO
1502 // fetch effective block level protections from table/block entries
1503 prot = pmap_get_arm64_prot(kernel_pmap, ctrr_test_page);
1504 T_EXPECT(ARM_PTE_EXTRACT_AP(prot) == AP_RWNA && (prot & ARM_PTE_PNX), "Mapping is EL1 RWNX");
1505
1506 ctrr_test_va = ctrr_test_page + (ro_test_va & PAGE_MASK);
1507 ctrr_ro_test_ptr = (void *)ctrr_test_va;
1508
1509 T_LOG("Read only region test writing to %p to provoke data abort", ctrr_ro_test_ptr);
1510
1511 // should cause data abort
1512 ml_expect_fault_begin(ctrr_test_ro_fault_handler, ctrr_test_va);
1513 *ctrr_ro_test_ptr = 1;
1514 ml_expect_fault_end();
1515
1516 // ensure write permission fault at expected level
1517 // data abort handler will set ctrr_exception_esr when ctrr_test_va takes a permission fault
1518
1519 T_EXPECT(ESR_EC(ctrr_exception_esr) == ESR_EC_DABORT_EL1, "Data Abort from EL1 expected");
1520 T_EXPECT(ISS_DA_FSC(ESR_ISS(ctrr_exception_esr)) == FSC_PERMISSION_FAULT_L3, "Permission Fault Expected");
1521 T_EXPECT(ESR_ISS(ctrr_exception_esr) & ISS_DA_WNR, "Write Fault Expected");
1522
1523 ctrr_test_va = 0;
1524 ctrr_exception_esr = 0;
1525 pmap_remove(kernel_pmap, ctrr_test_page, ctrr_test_page + PAGE_SIZE);
1526
1527 T_LOG("No execute test mapping virtual page %p to CTRR PXN page number %d", ctrr_test_page, nx_pn);
1528
1529 kr = pmap_enter(kernel_pmap, ctrr_test_page, nx_pn,
1530 VM_PROT_READ | VM_PROT_EXECUTE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE, PMAP_MAPPING_TYPE_INFER);
1531 T_EXPECT(kr == KERN_SUCCESS, "Expect pmap_enter of RX mapping to succeed");
1532
1533 // assert entire mmu prot path (Hierarchical protection model) is NOT XN
1534 prot = pmap_get_arm64_prot(kernel_pmap, ctrr_test_page);
1535 T_EXPECT(ARM_PTE_EXTRACT_AP(prot) == AP_RONA && (~prot & ARM_PTE_PNX), "Mapping is EL1 ROX");
1536
1537 ctrr_test_va = ctrr_test_page + (nx_test_va & PAGE_MASK);
1538 #if __has_feature(ptrauth_calls)
1539 ctrr_nx_test_ptr = ptrauth_sign_unauthenticated((void *)ctrr_test_va, ptrauth_key_function_pointer, 0);
1540 #else
1541 ctrr_nx_test_ptr = (void *)ctrr_test_va;
1542 #endif
1543
1544 T_LOG("No execute test calling ctrr_nx_test_ptr(): %p to provoke instruction abort", ctrr_nx_test_ptr);
1545
1546 // should cause prefetch abort
1547 ml_expect_fault_begin(ctrr_test_nx_fault_handler, ctrr_test_va);
1548 ctrr_nx_test_ptr();
1549 ml_expect_fault_end();
1550
1551 // TODO: ensure execute permission fault at expected level
1552 T_EXPECT(ESR_EC(ctrr_exception_esr) == ESR_EC_IABORT_EL1, "Instruction abort from EL1 Expected");
1553 T_EXPECT(ISS_DA_FSC(ESR_ISS(ctrr_exception_esr)) == FSC_PERMISSION_FAULT_L3, "Permission Fault Expected");
1554
1555 ctrr_test_va = 0;
1556 ctrr_exception_esr = 0;
1557
1558 pmap_remove(kernel_pmap, ctrr_test_page, ctrr_test_page + PAGE_SIZE);
1559
1560 T_LOG("Expect no faults when reading CTRR region to verify correct programming of CTRR limits");
1561 for (vm_offset_t addr = rorgn_begin_va; addr < rorgn_end_va; addr += 8) {
1562 volatile uint64_t x = *(uint64_t *)addr;
1563 (void) x; /* read for side effect only */
1564 }
1565
1566 return KERN_SUCCESS;
1567 }
1568 #endif /* defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST) */
1569
1570
1571