1 #include <darwintest.h>
2 #include <darwintest_perf.h>
3
4 T_GLOBAL_META(T_META_NAMESPACE("xnu.vm"));
5
6 #include <machine/cpu_capabilities.h>
7 #include <sys/mman.h>
8 #include <errno.h>
9 #include <fcntl.h>
10 #include <stdint.h>
11 #include <libkern/OSCacheControl.h>
12 #include <unistd.h>
13 #include <signal.h>
14 #include <stdlib.h>
15 #include <sys/sysctl.h>
16
17 #include <mach/vm_param.h>
18 #include <pthread.h>
19
20 #include <os/thread_self_restrict.h>
21
22 #include <mach/mach.h>
23 #include <mach/mach_error.h>
24 #include <mach/mach_init.h>
25 #include <mach/mach_port.h>
26 #include <mach/mach_vm.h>
27 #include <mach/vm_map.h>
28 #include <mach/task.h>
29
30 T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true));
31
32 #if defined(__arm64__)
33 /* PAGE_SIZE on ARM64 is an expression derived from a non-const global variable */
34 #define PAD_SIZE PAGE_MAX_SIZE
35 #else
36 #define PAD_SIZE PAGE_MIN_SIZE
37 #endif
38
39 /* Enumerations */
40 typedef enum _access_type {
41 ACCESS_READ,
42 ACCESS_WRITE,
43 } access_type_t;
44
45 typedef enum _fault_strategy {
46 FAULT_STRAT_NONE,
47 FAULT_STRAT_RW_TPRO,
48 } fault_strategy_t;
49
50 /* Structures */
51 typedef struct {
52 uint64_t fault_count;
53 fault_strategy_t fault_strategy;
54 bool fault_expected;
55 } fault_state_t;
56
57 /* Globals */
58 static bool key_created = false;
59 static pthread_key_t fault_state_key;
60
61 /*
62 * The pager will only map entries with TPRO if we need to perform fixups.
63 * Otherwise it really is const. Ensure we forge a struct that will require
64 * dynamic rebasing.
65 */
66 typedef struct {
67 void *reloc;
68 uint32_t magic;
69 char bytes[PAD_SIZE - 12];
70 } const_page_t;
71
72 typedef struct {
73 const_page_t one;
74 const_page_t two;
75 char ro[PAD_SIZE];
76 } const_state_t;
77
78 #define MAGIC(state) (void *)&state->magic
79
80 /*
81 * Force known data into our __DATA_CONST segment. The pager will be responsible
82 * for handling the mapping of this.
83 */
84 __attribute__((section("__DATA_CONST,__pager")))
85 __attribute__((aligned(PAD_SIZE)))
86 static const_state_t pager_state = {
87 .one.reloc = &pager_state,
88 .two.reloc = &pager_state,
89 .one.magic = 0x41414141,
90 .two.magic = 0x41414141,
91 .ro = "CCCC"
92 };
93
94 /* Allocate a fault_state_t, and associate it with the current thread. */
95 static fault_state_t *
fault_state_create(void)96 fault_state_create(void)
97 {
98 fault_state_t * fault_state = malloc(sizeof(fault_state_t));
99
100 if (fault_state) {
101 fault_state->fault_count = 0;
102 fault_state->fault_strategy = FAULT_STRAT_NONE;
103 fault_state->fault_expected = false;
104
105 if (pthread_setspecific(fault_state_key, fault_state)) {
106 free(fault_state);
107 fault_state = NULL;
108 }
109 }
110
111 return fault_state;
112 }
113
114 /* Disassociate the given fault state from the current thread, and destroy it. */
115 static void
fault_state_destroy(void * fault_state)116 fault_state_destroy(void * fault_state)
117 {
118 if (fault_state == NULL) {
119 T_ASSERT_FAIL("Attempted to fault_state_destroy NULL");
120 }
121
122 free(fault_state);
123 }
124
125 /*
126 * A signal handler that attempts to resolve anticipated faults through use of
127 * the os_thread_self_restrict_rwx functions.
128 */
129 static void
access_failed_handler(int signum)130 access_failed_handler(int signum)
131 {
132 fault_state_t * fault_state;
133
134 /* This handler should ONLY handle SIGBUS. */
135 if (signum != SIGBUS) {
136 T_ASSERT_FAIL("Unexpected signal sent to handler");
137 }
138
139 if (!(fault_state = pthread_getspecific(fault_state_key))) {
140 T_ASSERT_FAIL("Failed to retrieve fault state");
141 }
142
143 if (!(fault_state->fault_expected)) {
144 T_ASSERT_FAIL("Unexpected fault taken");
145 }
146
147 /* We should not see a second fault. */
148 fault_state->fault_expected = false;
149
150 switch (fault_state->fault_strategy) {
151 case FAULT_STRAT_NONE:
152 T_ASSERT_FAIL("No fault strategy");
153
154 /* Just in case we try to do something different. */
155 break;
156 case FAULT_STRAT_RW_TPRO:
157 os_thread_self_restrict_tpro_to_rw();
158 break;
159 }
160
161 fault_state->fault_count++;
162 }
163
164 /*
165 * Attempt the specified access; if the access faults, this will return true;
166 * otherwise, it will return false.
167 */
168 static bool
does_access_fault(access_type_t access_type,void * addr,uint32_t value)169 does_access_fault(access_type_t access_type, void * addr, uint32_t value)
170 {
171 uint64_t old_fault_count;
172 uint64_t new_fault_count;
173
174 fault_state_t * fault_state;
175
176 struct sigaction old_action; /* Save area for any existing action. */
177 struct sigaction new_action; /* The action we wish to install for SIGBUS. */
178
179 bool retval = false;
180
181 new_action.sa_handler = access_failed_handler; /* A handler for write failures. */
182 new_action.sa_mask = 0; /* Don't modify the mask. */
183 new_action.sa_flags = 0; /* Flags? Who needs those? */
184
185 if (addr == NULL) {
186 T_ASSERT_FAIL("Access attempted against NULL");
187 }
188
189 if (!(fault_state = pthread_getspecific(fault_state_key))) {
190 T_ASSERT_FAIL("Failed to retrieve fault state");
191 }
192
193 old_fault_count = fault_state->fault_count;
194
195 /* Install a handler so that we can catch SIGBUS. */
196 sigaction(SIGBUS, &new_action, &old_action);
197
198 /* Perform the requested operation. */
199 switch (access_type) {
200 case ACCESS_READ:
201 fault_state->fault_strategy = FAULT_STRAT_RW_TPRO;
202 fault_state->fault_expected = true;
203
204 __sync_synchronize();
205
206 #if defined(__arm64__)
207 uint8_t a = *((volatile uint8_t *)addr);
208 #endif
209 __sync_synchronize();
210
211 fault_state->fault_expected = false;
212 fault_state->fault_strategy = FAULT_STRAT_NONE;
213
214 break;
215
216 case ACCESS_WRITE:
217 fault_state->fault_strategy = FAULT_STRAT_RW_TPRO;
218 fault_state->fault_expected = true;
219
220 __sync_synchronize();
221
222 *((volatile uint32_t *)addr) = value;
223
224 __sync_synchronize();
225
226 fault_state->fault_expected = false;
227 fault_state->fault_strategy = FAULT_STRAT_NONE;
228
229 break;
230 }
231
232 /* Restore the old SIGBUS handler. */
233 sigaction(SIGBUS, &old_action, NULL);
234
235 new_fault_count = fault_state->fault_count;
236
237 if (new_fault_count > old_fault_count) {
238 /* Indicate that we took a fault. */
239 retval = true;
240 }
241
242 return retval;
243 }
244
245 static bool
does_read_fault(void * addr)246 does_read_fault(void * addr)
247 {
248 return does_access_fault(ACCESS_READ, addr, 0);
249 }
250
251 static bool
does_write_fault(void * addr,uint32_t value)252 does_write_fault(void * addr, uint32_t value)
253 {
254 return does_access_fault(ACCESS_WRITE, addr, value);
255 }
256
257 static bool
has_pager_support(void)258 has_pager_support(void)
259 {
260 uint32_t enabled = false;
261 size_t output_size = sizeof(enabled);
262
263 (void)sysctlbyname("vm.pmap_tpro_pagers",
264 &enabled, &output_size, NULL, 0);
265 return enabled;
266 }
267
268 static void
cleanup(void)269 cleanup(void)
270 {
271 fault_state_t * fault_state;
272
273 if (!(fault_state = pthread_getspecific(fault_state_key))) {
274 T_ASSERT_FAIL("Failed to retrieve fault state");
275
276 T_ASSERT_POSIX_ZERO(pthread_setspecific(fault_state_key, NULL), "Remove fault_state");
277 fault_state_destroy(fault_state);
278 }
279
280 if (key_created) {
281 T_ASSERT_POSIX_ZERO(pthread_key_delete(fault_state_key), "Delete fault state key");
282 }
283
284 return;
285 }
286
287 static void
288 thread_self_restrict_test(void (^test)(void))
289 {
290 int err = 0;
291
292 T_SETUPBEGIN;
293 T_ATEND(cleanup);
294
295 /* Set up the necessary state for the test. */
296 err = pthread_key_create(&fault_state_key, fault_state_destroy);
297 T_ASSERT_POSIX_ZERO(err, 0, "Create pthread key");
298 key_created = true;
299
300 T_ASSERT_NOTNULL(fault_state_create(), "Create fault state");
301 T_SETUPEND;
302
303 test();
304 }
305
306 static void
fork_child_test(const_page_t * state)307 fork_child_test(const_page_t *state)
308 {
309 pid_t pid;
310 int statloc;
311
312 pid = fork();
313 if (pid == 0) {
314 T_EXPECT_EQ(state->magic, 0x45454545, "Expected magic on fork");
315
316 os_thread_self_restrict_tpro_to_rw();
317 T_EXPECT_EQ(os_thread_self_restrict_tpro_is_writable(), true, "TPRO region configured as read-write in child");
318 T_EXPECT_EQ(does_write_fault((void *)&state->bytes, 0x47474747), 0, "write to pager backed memory in child (no fault)");
319 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x46464646), 0, "write to pager backed memory in child (no fault)");
320 exit(0);
321 }
322
323 if (pid < 0) {
324 T_ASSERT_POSIX_SUCCESS(pid, "fork");
325 }
326
327 waitpid(pid, &statloc, 0);
328 }
329
330 static void
pager_test(const_page_t * state)331 pager_test(const_page_t *state)
332 {
333 kern_return_t kr;
334 uint32_t pre;
335 vm_prot_t curprot, maxprot;
336 mach_vm_address_t addr = 0;
337 const_page_t *copy_state = NULL;
338 mach_port_t cow_port = MACH_PORT_NULL;
339 memory_object_size_t me_size = PAGE_SIZE;
340
341 /*
342 * Validate our initial status quo. TPRO permissions should be RO,
343 * so we should be able to read from our pager backed mapping but
344 * should fault when trying to write to it.
345 */
346 T_EXPECT_EQ(os_thread_self_restrict_tpro_is_writable(), false, "TPRO region starts read-only");
347 T_EXPECT_EQ(does_read_fault(MAGIC(state)), 0, "read from pager backed memory");
348 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x43434343), 1, "write to pager backed memory (detect fault)");
349
350 /*
351 * Toggle permissions to RW and attempt a write. We should succeed.
352 */
353 os_thread_self_restrict_tpro_to_rw();
354 T_EXPECT_EQ(os_thread_self_restrict_tpro_is_writable(), true, "TPRO region configured as read-write");
355 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x44444444), 0, "write to pager backed memory (no fault)");
356
357 /*
358 * Toggle permissions to RO and attempt a write. We should detect
359 * the fault
360 */
361 os_thread_self_restrict_tpro_to_ro();
362 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x45454545), 1, "write to pager backed memory (detect fault)");
363
364 /*
365 * Fork a child process and ensure that writes into the pager backed
366 * regions are not observed by the parent. They should now be COW.
367 */
368 pre = state->magic;
369 fork_child_test(state);
370 T_EXPECT_EQ(pre, state->magic, "write from child should not be observed");
371
372 /*
373 * Ensure that if we remap the target region in a shared manner that we
374 * inherit TPRO. Remapping should be successful but we still rely on
375 * TPRO permissions to toggle r--/rw-
376 */
377 kr = mach_vm_remap(mach_task_self(),
378 &addr,
379 PAGE_SIZE,
380 0, /* mask */
381 VM_FLAGS_ANYWHERE,
382 mach_task_self(),
383 (mach_vm_address_t)state,
384 FALSE, /* copy */
385 &curprot,
386 &maxprot,
387 VM_INHERIT_DEFAULT);
388 T_EXPECT_POSIX_SUCCESS(kr, "mach_vm_remap(SHARED)");
389 copy_state = (const_page_t *)addr;
390
391 os_thread_self_restrict_tpro_to_ro();
392 T_EXPECT_EQ(os_thread_self_restrict_tpro_is_writable(), false, "TPRO configured as read-only");
393 T_EXPECT_EQ(curprot, VM_PROT_READ, "TPRO region should be VM_PROT_READ");
394 T_EXPECT_EQ(does_write_fault(MAGIC(copy_state), 0x46464646), 1, "write to remapped region (detect fault)");
395 os_thread_self_restrict_tpro_to_rw();
396 T_EXPECT_EQ(does_write_fault(MAGIC(copy_state), 0x46464646), 0, "write to remapped region (no fault)");
397 T_EXPECT_EQ(0x46464646, state->magic, "write into copied region should be observed");
398
399 /*
400 * Ensure that if we remap the region that we do not observe writes to
401 * the new copy in __DATA_CONST itself.
402 */
403 kr = mach_vm_remap(mach_task_self(),
404 (mach_vm_address_t *)©_state,
405 PAGE_SIZE,
406 0, /* mask */
407 VM_FLAGS_ANYWHERE,
408 mach_task_self(),
409 (mach_vm_address_t)state,
410 TRUE, /* copy */
411 &curprot,
412 &maxprot,
413 VM_INHERIT_DEFAULT);
414 T_EXPECT_POSIX_SUCCESS(kr, "mach_vm_remap(COPY)");
415
416 /*
417 * Toggle TPRO RW and write to the new copied region
418 */
419 pre = state->magic;
420 os_thread_self_restrict_tpro_to_rw();
421 T_EXPECT_EQ(os_thread_self_restrict_tpro_is_writable(), true, "TPRO region configured as read-write");
422 T_EXPECT_EQ(does_write_fault(MAGIC(copy_state), 0x46464646), 0, "write to pager backed memory (no fault)");
423 T_EXPECT_EQ(pre, state->magic, "write into copied region should not be observed");
424
425 /*
426 * Make a memory entry for our target region and attempt to map it in
427 * in a shared fashion. We should succeed but it should transparently
428 * copy the target VM object as extracting TPRO VM entries will fail.
429 * Writes to the new region should therefore not be observed.
430 */
431 me_size = PAGE_SIZE;
432 kr = mach_make_memory_entry_64(mach_task_self(),
433 &me_size,
434 (mach_vm_address_t)state,
435 MAP_MEM_VM_SHARE | VM_PROT_READ | VM_PROT_WRITE,
436 &cow_port,
437 MACH_PORT_NULL);
438 T_EXPECT_POSIX_SUCCESS(kr, "mach_make_memory_entry_64(MAP_MEM_VM_SHARE)");
439
440 pre = state->magic;
441 T_EXPECT_EQ(does_write_fault(MAGIC(copy_state), 0x48484849), 0, "write to mapped copy region (no fault)");
442 T_EXPECT_EQ(pre, state->magic, "write into copied region should not be observed");
443
444 copy_state = NULL;
445 kr = mach_vm_map(mach_task_self(),
446 (mach_vm_address_t *)©_state,
447 PAGE_SIZE,
448 0, /* mask */
449 VM_FLAGS_ANYWHERE,
450 cow_port,
451 0, /* offset */
452 TRUE, /* copy */
453 VM_PROT_READ | VM_PROT_WRITE,
454 VM_PROT_READ | VM_PROT_WRITE,
455 VM_INHERIT_DEFAULT);
456 T_EXPECT_POSIX_SUCCESS(kr, "mach_vm_map(cow_port)");
457
458 /*
459 * Pages of the copy will no longer be mapped in as TPRO. Both
460 * read/writes should work even with TPRO toggled RO.
461 */
462 pre = state->magic;
463 os_thread_self_restrict_tpro_to_ro();
464 T_EXPECT_EQ(does_write_fault(MAGIC(copy_state), 0x48484848), 0, "write to mapped copy region (no fault)");
465 T_EXPECT_EQ(pre, state->magic, "write into copied region should not be observed");
466
467 /*
468 * We've explored a number of ways to perform copies on the target
469 * objects in __DATA_CONST. Our first target page (&pager_state.one)
470 * should now be marked RO without TPRO permissions to handle any
471 * incoming write faults. Write to it directly again to ensure we
472 * fault back in with TPRO permissions.
473 */
474 os_thread_self_restrict_tpro_to_ro();
475 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x49494949), 1, "write to pager backed memory (detect fault)");
476 os_thread_self_restrict_tpro_to_rw();
477 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x4a4a4a4a), 0, "write to pager backed memory (no fault)");
478
479 /*
480 * Now we attempt to have the page paged out. On systems which support the
481 * compressor, we'll get paged out/compressed. On fault we should
482 * be pmapped back in with TPRO permissions.
483 */
484 mach_vm_behavior_set(mach_task_self(), (mach_vm_address_t)state, PAGE_SIZE, VM_BEHAVIOR_PAGEOUT);
485
486 /*
487 * Can verify in debugger at this point that page(s) have been
488 * paged out. If compressor pager is available the page should
489 * not be resident and compressor pager should be tied to the
490 * top level VM object.
491 */
492 os_thread_self_restrict_tpro_to_ro();
493 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x49494949), 1, "write to pager backed memory after pageout (detect fault)");
494 os_thread_self_restrict_tpro_to_rw();
495 T_EXPECT_EQ(does_write_fault(MAGIC(state), 0x4a4a4a4a), 0, "write to pager backed memory after pageout (no fault)");
496
497 /*
498 * Try and reprotect the region. We should fail
499 */
500 kr = vm_protect(mach_task_self(), (mach_vm_address_t)state, PAGE_SIZE, FALSE, VM_PROT_DEFAULT);
501 T_EXPECT_POSIX_ERROR(kr, KERN_PROTECTION_FAILURE, "vm_protect(RW) should fail");
502
503 os_thread_self_restrict_tpro_to_ro();
504 }
505
506 T_DECL(thread_self_restrict_pagers,
507 "Verify that the TPRO pager interfaces work correctly")
508 {
509 #if __arm64__
510 /* Check to see that we support the necessary hardware features. */
511 if (!os_thread_self_restrict_tpro_is_supported() || !has_pager_support()) {
512 T_SKIP("no hardware TPRO support enabled on this system");
513 }
514
515 thread_self_restrict_test(^{
516 pager_test(&pager_state.one);
517
518 /*
519 * Ensure that touching the second pager supported page exhibits
520 * identical behaviour in order to validate the transitions between
521 * VM entry & copy object chains.
522 */
523 pager_test(&pager_state.two);
524
525 /*
526 * Try and write to a normal __DATA_CONST page that isn't backed by
527 * the dyld pager. The kernel will have mapped this directly but
528 * should still maintain TPRO protection.
529 */
530 os_thread_self_restrict_tpro_to_ro();
531 T_EXPECT_EQ(does_write_fault(&pager_state.ro[0], 0x41414141), 1, "write to __DATA_CONST should succeed (no fault)");
532 os_thread_self_restrict_tpro_to_rw();
533 T_EXPECT_EQ(does_write_fault(&pager_state.ro[0], 0x41414141), 0, "write to __DATA_CONST should fail (detect fault)");
534 });
535 #else
536 T_SKIP("thread_self_restrict_pagers not supported on this system");
537 #endif /* __arm64__ */
538 }
539