1 /*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if CONFIG_EXCLAVES
30
31 #include <stdint.h>
32 #include <mach/kern_return.h>
33 #include <kern/assert.h>
34 #include <kern/misc_protos.h>
35 #include <kern/locks.h>
36 #include <kern/thread.h>
37 #include <vm/pmap.h>
38 #include <mach/exclaves_l4.h>
39
40 #include "exclaves_debug.h"
41 #include "exclaves_xnuproxy.h"
42 #include "exclaves_resource.h"
43 #include "exclaves_upcalls.h"
44 #include "exclaves_internal.h"
45
46 #include "kern/exclaves.tightbeam.h"
47
48 #include <xnuproxy/messages.h>
49
50 /* -------------------------------------------------------------------------- */
51 #pragma mark IPC bootstrap
52
53 /* Lock protecting the use of the bootstrap scheduling context */
54 static LCK_MTX_DECLARE(exclaves_xnuproxy_lock, &exclaves_lck_grp);
55
56 /*
57 * Bootstrap context. Used for context allocate/free. Initialized in
58 * exclaves_xnuproxy_init().
59 */
60 static exclaves_ctx_t exclaves_bootstrap_ctx = {};
61
62 /*
63 * Switch the current thread to use the bootstrap context. Stash the old context
64 * into the supplied arguments.
65 * Returns with exclaves_xnuproxy_lock held.
66 */
67 static void
exclaves_bootstrap_context_acquire(exclaves_ctx_t * save_ctx)68 exclaves_bootstrap_context_acquire(exclaves_ctx_t *save_ctx)
69 {
70 assert3p(exclaves_bootstrap_ctx.ipcb, !=, NULL);
71 assert3u(save_ctx->scid, !=, exclaves_bootstrap_ctx.scid);
72
73 lck_mtx_lock(&exclaves_xnuproxy_lock);
74
75 thread_t thread = current_thread();
76
77 *save_ctx = thread->th_exclaves_ipc_ctx;
78
79 thread->th_exclaves_ipc_ctx = exclaves_bootstrap_ctx;
80
81 LCK_MTX_ASSERT(&exclaves_xnuproxy_lock, LCK_MTX_ASSERT_OWNED);
82 }
83
84 /*
85 * Restore the scheduling context of the current thread.
86 * Returns with exclaves_xnuproxy_lock released.
87 */
88 static void
exclaves_bootstrap_context_release(const exclaves_ctx_t * restore_ctx)89 exclaves_bootstrap_context_release(const exclaves_ctx_t *restore_ctx)
90 {
91 assert3u(restore_ctx->scid, !=, exclaves_bootstrap_ctx.scid);
92
93 LCK_MTX_ASSERT(&exclaves_xnuproxy_lock, LCK_MTX_ASSERT_OWNED);
94
95 thread_t thread = current_thread();
96 assert3p(thread->th_exclaves_ipc_ctx.ipcb, ==, exclaves_bootstrap_ctx.ipcb);
97
98 /* Reset */
99 thread->th_exclaves_ipc_ctx = *restore_ctx;
100
101 lck_mtx_unlock(&exclaves_xnuproxy_lock);
102 }
103
104 /* -------------------------------------------------------------------------- */
105 #pragma mark IPC buffer count
106
107 /*
108 * Number of allocated ipcb buffers. Estimates the number of active exclave
109 * threads.
110 */
111 static _Atomic size_t exclaves_ipcb_cnt;
112
113 size_t
exclaves_ipc_buffer_count(void)114 exclaves_ipc_buffer_count(void)
115 {
116 return os_atomic_load(&exclaves_ipcb_cnt, relaxed);
117 }
118
119 static void
exclaves_ipc_buffer_count_inc(void)120 exclaves_ipc_buffer_count_inc(void)
121 {
122 os_atomic_inc(&exclaves_ipcb_cnt, relaxed);
123 }
124
125 static void
exclaves_ipc_buffer_count_dec(void)126 exclaves_ipc_buffer_count_dec(void)
127 {
128 __assert_only size_t orig_ipcb_cnt =
129 os_atomic_dec_orig(&exclaves_ipcb_cnt, relaxed);
130 assert3u(orig_ipcb_cnt, >=, 1);
131 }
132
133 /* -------------------------------------------------------------------------- */
134 #pragma mark IPC buffer cache
135
136 /*
137 * A (simple, for now...) cache of IPC buffers for communicating with XNU-Proxy.
138 * The cache itself is realtime safe and relies on a spin lock for
139 * synchronization. However, if there's no cached buffer available, the calling
140 * code will fallback to doing a full IPC buffer allocation with xnu-proxy. This
141 * involves taking a mutex and is not realtime safe.
142 */
143
144 /*
145 * Determines the maximum size of the buffer cache. Can be overriden via an EDT
146 * entry or boot-arg.
147 */
148 TUNABLE_DEV_WRITEABLE(unsigned int, exclaves_ipc_buffer_cache_max,
149 "exclaves_ipcb_cache", 16);
150
151 /* Current count of entries in the buffer cache. */
152 static unsigned int exclaves_ipc_buffer_cache_count = 0;
153
154 /* Intrusive linked list within the unused IPC buffer */
155 typedef struct exclaves_ipc_buffer_cache_item {
156 struct exclaves_ipc_buffer_cache_item *next;
157 Exclaves_L4_Word_t scid;
158 }__attribute__((__packed__)) exclaves_ipc_buffer_cache_item_t;
159
160 static_assert(Exclaves_L4_IpcBuffer_Size >=
161 sizeof(exclaves_ipc_buffer_cache_item_t),
162 "Invalid Exclaves_L4_IpcBuffer_Size");
163
164 static LCK_SPIN_DECLARE(exclaves_ipc_buffer_cache_lock, &exclaves_lck_grp);
165 static exclaves_ipc_buffer_cache_item_t *exclaves_ipc_buffer_cache;
166
167 static bool
exclaves_ipc_buffer_cache_alloc(exclaves_ctx_t * ctx)168 exclaves_ipc_buffer_cache_alloc(exclaves_ctx_t *ctx)
169 {
170 lck_spin_lock(&exclaves_ipc_buffer_cache_lock);
171
172 if (exclaves_ipc_buffer_cache_count == 0) {
173 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
174 return false;
175 }
176
177 assert3p(exclaves_ipc_buffer_cache, !=, NULL);
178
179 exclaves_ipc_buffer_cache_item_t *cached_buffer = exclaves_ipc_buffer_cache;
180 exclaves_ipc_buffer_cache = cached_buffer->next;
181
182 exclaves_ipc_buffer_cache_count--;
183
184 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
185
186 ctx->ipcb = (void *)cached_buffer;
187 ctx->scid = cached_buffer->scid;
188 ctx->usecnt = 0;
189
190 /*
191 * Zero out this usage of the buffer to avoid any confusion in
192 * xnu-proxy.
193 */
194 cached_buffer->next = NULL;
195 cached_buffer->scid = 0;
196
197 return true;
198 }
199
200 static bool
exclaves_ipc_buffer_cache_free(exclaves_ctx_t * ctx)201 exclaves_ipc_buffer_cache_free(exclaves_ctx_t *ctx)
202 {
203 assert3u(ctx->scid, !=, exclaves_bootstrap_ctx.scid);
204
205 /* Zero out the IPC buffer to avoid having old IPC data lying around. */
206 bzero(ctx->ipcb, Exclaves_L4_IpcBuffer_Size);
207
208 lck_spin_lock(&exclaves_ipc_buffer_cache_lock);
209
210 #if 0 /* Removed with the fix for rdar://126257712 */
211 /* Don't free into the cache if the cache has hit its limit. */
212 if (exclaves_ipc_buffer_cache_count == exclaves_ipc_buffer_cache_max) {
213 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
214 return false;
215 }
216 #endif
217
218 exclaves_ipc_buffer_cache_item_t *cached_buffer = NULL;
219
220 cached_buffer = (void *)ctx->ipcb;
221 cached_buffer->scid = ctx->scid;
222
223 ctx->ipcb = NULL;
224 ctx->scid = 0;
225 ctx->usecnt = 0;
226
227 cached_buffer->next = exclaves_ipc_buffer_cache;
228 exclaves_ipc_buffer_cache = cached_buffer;
229
230 exclaves_ipc_buffer_cache_count++;
231
232 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
233
234 return true;
235 }
236
237 static kern_return_t
exclaves_ipc_buffer_cache_init(void)238 exclaves_ipc_buffer_cache_init(void)
239 {
240 if (exclaves_ipc_buffer_cache_max == 0) {
241 return KERN_SUCCESS;
242 }
243
244 kern_return_t kr = KERN_FAILURE;
245
246 assert3p(exclaves_ipc_buffer_cache, ==, NULL);
247
248 exclaves_ctx_t *ctx = kalloc_type(exclaves_ctx_t,
249 exclaves_ipc_buffer_cache_max, Z_WAITOK | Z_ZERO | Z_NOFAIL);
250
251 /*
252 * Pre-warm the cache by allocating up to cache_max and then releasing
253 * the allocated contexts back into the cache.
254 */
255 for (unsigned int i = 0; i < exclaves_ipc_buffer_cache_max; i++) {
256 kr = exclaves_xnuproxy_ctx_alloc(&ctx[i]);
257 if (kr != KERN_SUCCESS) {
258 kfree_type(exclaves_ctx_t,
259 exclaves_ipc_buffer_cache_max, ctx);
260 return kr;
261 }
262 }
263
264 /*
265 * Release the newly allocated contexts so they ends up in the cache. We
266 * know this will succeed because the only failure modes of
267 * exclaves_xnuproxy_ctx_free are if the downcall fails. The downcall
268 * won't be used here as we *know* that the buffer cache is active.
269 */
270 for (unsigned int i = 0; i < exclaves_ipc_buffer_cache_max; i++) {
271 kr = exclaves_xnuproxy_ctx_free(&ctx[i]);
272 assert3u(kr, ==, KERN_SUCCESS);
273 }
274
275 kfree_type(exclaves_ctx_t, exclaves_ipc_buffer_cache_max, ctx);
276
277 return KERN_SUCCESS;
278 }
279
280
281 /* -------------------------------------------------------------------------- */
282 #pragma mark xnu-proxy calls
283
284 static xnuproxy_cmd_s xnuproxy_cmd_client = {0};
285
286 kern_return_t
exclaves_xnuproxy_ctx_alloc(exclaves_ctx_t * ctx)287 exclaves_xnuproxy_ctx_alloc(exclaves_ctx_t *ctx)
288 {
289 assert3p(ctx, !=, NULL);
290
291 /* Try to allocate it from the cache. */
292 if (exclaves_ipc_buffer_cache_alloc(ctx)) {
293 assert(ctx->usecnt == 0);
294 return KERN_SUCCESS;
295 }
296
297 /*
298 * Fallback to a full allocation with xnuproxy. This must be done in the
299 * context of the bootstrap scheduling context.
300 */
301 exclaves_ctx_t stash_ctx = {};
302 __block exclaves_ctx_t local_ctx = {};
303
304 exclaves_bootstrap_context_acquire(&stash_ctx);
305
306 /* This may spawn a new exclaves thread. */
307 thread_exclaves_state_flags_t state = current_thread()->th_exclaves_state;
308 current_thread()->th_exclaves_state |= TH_EXCLAVES_SPAWN_EXPECTED;
309
310 tb_error_t ret = xnuproxy_cmd_ipccontextallocate(&xnuproxy_cmd_client,
311 ^(xnuproxy_ipccontext_s c) {
312 local_ctx.ipcb = (Exclaves_L4_IpcBuffer_t *)phystokv(c.buffer);
313 local_ctx.scid = c.scid;
314 });
315
316 /* Restore the old state (which itself may have set the SPAWN flag). */
317 current_thread()->th_exclaves_state = state;
318
319 exclaves_bootstrap_context_release(&stash_ctx);
320
321 if (ret != TB_ERROR_SUCCESS) {
322 exclaves_debug_printf(show_errors,
323 "allocate context: failure %u\n", ret);
324 return KERN_FAILURE;
325 }
326
327 /* Update count. */
328 exclaves_ipc_buffer_count_inc();
329
330 *ctx = local_ctx;
331
332 assert(ctx->usecnt == 0);
333 return KERN_SUCCESS;
334 }
335
336 kern_return_t
exclaves_xnuproxy_ctx_free(exclaves_ctx_t * ctx)337 exclaves_xnuproxy_ctx_free(exclaves_ctx_t *ctx)
338 {
339 assert3p(ctx, !=, NULL);
340
341 /* exclaves_bootstrap_ctx.scid should never be freed. */
342 if (ctx->scid == exclaves_bootstrap_ctx.scid) {
343 return KERN_SUCCESS;
344 }
345
346 assert(ctx->usecnt == 0);
347 /* Try to free it back to the cache. */
348 if (exclaves_ipc_buffer_cache_free(ctx)) {
349 return KERN_SUCCESS;
350 }
351
352 /*
353 * Fallback to a full free with xnuproxy. This must be done in the
354 * context of the bootstrap scheduling context.
355 */
356 exclaves_ctx_t stash_ctx = {};
357 __block exclaves_ctx_t local_ctx = *ctx;
358
359 exclaves_bootstrap_context_acquire(&stash_ctx);
360
361 xnuproxy_ipccontext_s c = {
362 .scid = local_ctx.scid,
363 };
364
365 tb_error_t ret = xnuproxy_cmd_ipccontextfree(&xnuproxy_cmd_client, &c);
366
367 exclaves_bootstrap_context_release(&stash_ctx);
368
369 if (ret != TB_ERROR_SUCCESS) {
370 exclaves_debug_printf(show_errors,
371 "free context: failure %u\n", ret);
372 return KERN_FAILURE;
373 }
374
375 ctx->ipcb = NULL;
376 ctx->scid = 0;
377 ctx->usecnt = 0;
378
379 /* Update count. */
380 exclaves_ipc_buffer_count_dec();
381
382 return KERN_SUCCESS;
383 }
384
385 static size_t
countof_char_v(const char_v_s * cv)386 countof_char_v(const char_v_s *cv)
387 {
388 assert3p(cv, !=, NULL);
389
390 __block size_t count = 0;
391
392 char__v_visit(cv,
393 ^( __unused size_t i, __unused const xnuproxy_char_s item) {
394 count++;
395 });
396
397 return count;
398 }
399
400 static void
copy_char_v(const char_v_s * src,char * dst)401 copy_char_v(const char_v_s *src, char *dst)
402 {
403 assert3p(src, !=, NULL);
404 assert3p(dst, !=, NULL);
405
406 char__v_visit(src,
407 ^(size_t i, const xnuproxy_char_s item) {
408 dst[i] = item;
409 });
410 }
411
412 /*
413 * Iterate over all the resources calling cb for each one.
414 */
415 kern_return_t
416 exclaves_xnuproxy_resource_info(void (^cb)(const char *name, const char *domain,
417 xnuproxy_resourcetype_s, uint64_t id, bool))
418 {
419 /* BEGIN IGNORE CODESTYLE */
420 tb_error_t ret = xnuproxy_cmd_resourceinfo(&xnuproxy_cmd_client,
421 ^(xnuproxy_resourceinfo_v_s ri) {
422 xnuproxy_resourceinfo__v_visit(&ri,
423 ^(__unused size_t i, const xnuproxy_resourceinfo_s *item) {
424 char name_copy[countof_char_v(&item->name)];
425 copy_char_v(&item->name, name_copy);
426
427 char domain_copy[countof_char_v(&item->domain)];
428 copy_char_v(&item->domain, domain_copy);
429
430 cb(name_copy, domain_copy,
431 (xnuproxy_resourcetype_s)item->type, item->id,
432 item->connected);
433 });
434 });
435 /* END IGNORE CODESTYLE */
436
437 if (ret != TB_ERROR_SUCCESS) {
438 exclaves_debug_printf(show_errors,
439 "resource info: failure %u\n", ret);
440 return KERN_FAILURE;
441 }
442
443 return KERN_SUCCESS;
444 }
445
446 kern_return_t
exclaves_xnuproxy_pmm_usage(void)447 exclaves_xnuproxy_pmm_usage(void)
448 {
449 tb_error_t ret = xnuproxy_cmd_pmmmemusage(&xnuproxy_cmd_client);
450 if (ret != TB_ERROR_SUCCESS) {
451 exclaves_debug_printf(show_errors,
452 "pmm usage: failure %u\n", ret);
453 return KERN_FAILURE;
454 }
455
456 return KERN_SUCCESS;
457 }
458
459 kern_return_t
exclaves_xnuproxy_panic_setup(uint64_t * phys,uint64_t * scid)460 exclaves_xnuproxy_panic_setup(uint64_t *phys, uint64_t *scid)
461 {
462 current_thread()->th_exclaves_state |= TH_EXCLAVES_SPAWN_EXPECTED;
463
464 tb_error_t ret = xnuproxy_cmd_panicsetup(&xnuproxy_cmd_client,
465 ^(xnuproxy_panicinfo_s panic_data) {
466 *phys = panic_data.buffer;
467 *scid = panic_data.scid;
468 });
469
470 current_thread()->th_exclaves_state &= ~TH_EXCLAVES_SPAWN_EXPECTED;
471
472 if (ret != TB_ERROR_SUCCESS) {
473 exclaves_debug_printf(show_errors,
474 "panic setup: failure %u\n", ret);
475 return KERN_FAILURE;
476 }
477
478 return KERN_SUCCESS;
479 }
480
481
482 /* -------------------------------------------------------------------------- */
483 #pragma mark legacy xnu-proxy calls
484
485
486 kern_return_t
exclaves_xnuproxy_audio_buffer_copyout(uint64_t id,uint64_t size1,uint64_t offset1,uint64_t size2,uint64_t offset2)487 exclaves_xnuproxy_audio_buffer_copyout(uint64_t id,
488 uint64_t size1, uint64_t offset1, uint64_t size2, uint64_t offset2)
489 {
490 __block xnuproxy_namedbufferstatus_s status =
491 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
492
493 /* BEGIN IGNORE CODESTYLE */
494 tb_error_t ret = xnuproxy_cmd_audiobuffercopyout(&xnuproxy_cmd_client,
495 id, size1, offset1, size2, offset2,
496 ^(xnuproxy_cmd_audiobuffercopyout__result_s result) {
497 if (xnuproxy_cmd_audiobuffercopyout__result_get_success(&result)) {
498 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
499 return;
500 }
501
502 xnuproxy_namedbufferstatus_s *status_p = NULL;
503 status_p = xnuproxy_cmd_audiobuffercopyout__result_get_failure(&result);
504 assert3p(status_p, !=, NULL);
505 status = *status_p;
506
507 exclaves_debug_printf(show_errors,
508 "audio buffer copyout: failure %u\n", status);
509 });
510
511 if (ret != TB_ERROR_SUCCESS) {
512 return KERN_FAILURE;
513 }
514
515 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
516 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
517 KERN_INVALID_ARGUMENT : KERN_FAILURE;
518 }
519 /* END IGNORE CODESTYLE */
520
521 return KERN_SUCCESS;
522 }
523
524 kern_return_t
exclaves_xnuproxy_audio_buffer_delete(uint64_t id)525 exclaves_xnuproxy_audio_buffer_delete(uint64_t id)
526 {
527 __block xnuproxy_namedbufferstatus_s status =
528 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
529
530 /* BEGIN IGNORE CODESTYLE */
531 tb_error_t ret = xnuproxy_cmd_audiobufferdelete(&xnuproxy_cmd_client, id,
532 ^(xnuproxy_cmd_audiobufferdelete__result_s result) {
533 if (xnuproxy_cmd_audiobufferdelete__result_get_success(&result)) {
534 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
535 return;
536 }
537
538 xnuproxy_namedbuffererror_s *error = NULL;
539 error = xnuproxy_cmd_audiobufferdelete__result_get_failure(&result);
540
541 assert3p(error, !=, NULL);
542 exclaves_debug_printf(show_errors,
543 "audio buffer delete: failure %u, %u\n",
544 error->status, error->substatus);
545 status = error->status;
546 });
547
548 if (ret != TB_ERROR_SUCCESS) {
549 return KERN_FAILURE;
550 }
551
552 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
553 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
554 KERN_INVALID_ARGUMENT : KERN_FAILURE;
555 }
556 /* END IGNORE CODESTYLE */
557
558 return KERN_SUCCESS;
559 }
560
561 kern_return_t
exclaves_xnuproxy_audio_buffer_map(uint64_t id,size_t size,bool * read_only)562 exclaves_xnuproxy_audio_buffer_map(uint64_t id, size_t size, bool *read_only)
563 {
564 __block xnuproxy_namedbufferstatus_s status =
565 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
566
567 /* BEGIN IGNORE CODESTYLE */
568 tb_error_t ret = xnuproxy_cmd_audiobuffermap(&xnuproxy_cmd_client, id,
569 size, ^(xnuproxy_cmd_audiobuffermap__result_s result) {
570 xnuproxy_mapinfo_s *map_info;
571 map_info = xnuproxy_cmd_audiobuffermap__result_get_success(&result);
572 if (map_info != NULL) {
573 *read_only = map_info->readonly;
574 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
575 return;
576 }
577
578 xnuproxy_namedbuffererror_s *error;
579 error = xnuproxy_cmd_audiobuffermap__result_get_failure(&result);
580 assert3p(error, !=, NULL);
581
582 exclaves_debug_printf(show_errors,
583 "audio buffer map: failure %u, %u\n",
584 error->status, error->substatus);
585 status = error->status;
586 });
587
588 if (ret != TB_ERROR_SUCCESS) {
589 return KERN_FAILURE;
590 }
591
592 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
593 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
594 KERN_INVALID_ARGUMENT : KERN_FAILURE;
595 }
596
597 /* END IGNORE CODESTYLE */
598
599 return KERN_SUCCESS;
600 }
601
602 kern_return_t
603 exclaves_xnuproxy_audio_buffer_layout(uint64_t id, uint32_t start,
604 uint32_t npages, kern_return_t (^cb)(uint64_t base, uint32_t npages))
605 {
606 __block xnuproxy_namedbufferstatus_s status = 0;
607 __block kern_return_t kret = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
608
609 /* BEGIN IGNORE CODESTYLE */
610 tb_error_t ret = xnuproxy_cmd_audiobufferlayout(&xnuproxy_cmd_client,
611 id, start, npages,
612 ^(xnuproxy_cmd_audiobufferlayout__result_s result) {
613 xnuproxy_namedbufferrange_v_s *ranges;
614 ranges = xnuproxy_cmd_audiobufferlayout__result_get_success(&result);
615 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
616 if (ranges != NULL) {
617 xnuproxy_namedbufferrange__v_visit(ranges,
618 ^(__unused size_t i, const xnuproxy_namedbufferrange_s *item) {
619 /*
620 * Only fail once. TB want to iterate over the
621 * entire array so we have to keep going but
622 * once the first failure is seen, don't bother
623 * calling cb.
624 */
625 if (kret == KERN_SUCCESS) {
626 kret = cb(item->address, item->numpages);
627 }
628 });
629
630 return;
631 }
632
633 xnuproxy_namedbuffererror_s *error;
634 error = xnuproxy_cmd_audiobufferlayout__result_get_failure(&result);
635 assert3p(error, !=, NULL);
636
637 exclaves_debug_printf(show_errors,
638 "audio buffer layout: failure %u, %u\n",
639 error->status, error->substatus);
640 status = error->status;
641 });
642
643 if (ret != TB_ERROR_SUCCESS) {
644 return KERN_FAILURE;
645 }
646
647 if (kret != KERN_SUCCESS) {
648 return kret;
649 }
650
651 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
652 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
653 KERN_INVALID_ARGUMENT : KERN_FAILURE;
654 }
655 /* END IGNORE CODESTYLE */
656
657 return KERN_SUCCESS;
658 }
659
660 kern_return_t
exclaves_xnuproxy_named_buffer_delete(uint64_t id)661 exclaves_xnuproxy_named_buffer_delete(uint64_t id)
662 {
663 __block xnuproxy_namedbufferstatus_s status =
664 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
665
666 /* BEGIN IGNORE CODESTYLE */
667 tb_error_t ret = xnuproxy_cmd_namedbufferdelete(&xnuproxy_cmd_client, id,
668 ^(xnuproxy_cmd_namedbufferdelete__result_s result) {
669 if (xnuproxy_cmd_namedbufferdelete__result_get_success(&result)) {
670 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
671 return;
672 }
673
674 xnuproxy_namedbuffererror_s *error = NULL;
675 error = xnuproxy_cmd_namedbufferdelete__result_get_failure(&result);
676
677 assert3p(error, !=, NULL);
678 exclaves_debug_printf(show_errors,
679 "named buffer delete: failure %u, %u\n",
680 error->status, error->substatus);
681 status = error->status;
682 });
683
684
685 if (ret != TB_ERROR_SUCCESS) {
686 return KERN_FAILURE;
687 }
688
689 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
690 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
691 KERN_INVALID_ARGUMENT : KERN_FAILURE;
692 }
693 /* END IGNORE CODESTYLE */
694
695 return KERN_SUCCESS;
696 }
697
698 kern_return_t
exclaves_xnuproxy_named_buffer_map(uint64_t id,size_t size,bool * read_only)699 exclaves_xnuproxy_named_buffer_map(uint64_t id, size_t size, bool *read_only)
700 {
701 __block xnuproxy_namedbufferstatus_s status =
702 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
703
704 /* BEGIN IGNORE CODESTYLE */
705 tb_error_t ret = xnuproxy_cmd_namedbuffermap(&xnuproxy_cmd_client, id,
706 size, ^(xnuproxy_cmd_namedbuffermap__result_s result) {
707 xnuproxy_mapinfo_s *map_info;
708 map_info = xnuproxy_cmd_namedbuffermap__result_get_success(&result);
709 if (map_info != NULL) {
710 *read_only = map_info->readonly;
711 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
712 return;
713 }
714
715 xnuproxy_namedbuffererror_s *error;
716 error = xnuproxy_cmd_namedbuffermap__result_get_failure(&result);
717 assert3p(error, !=, NULL);
718
719 exclaves_debug_printf(show_errors,
720 "named buffer map: failure %u, %u\n",
721 error->status, error->substatus);
722 status = error->status;
723 });
724
725 if (ret != TB_ERROR_SUCCESS) {
726 return KERN_FAILURE;
727 }
728
729 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
730 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
731 KERN_INVALID_ARGUMENT : KERN_FAILURE;
732 }
733 /* END IGNORE CODESTYLE */
734
735 return KERN_SUCCESS;
736 }
737
738 kern_return_t
739 exclaves_xnuproxy_named_buffer_layout(uint64_t id, uint32_t start,
740 uint32_t npages, kern_return_t (^cb)(uint64_t base, uint32_t npages))
741 {
742 __block xnuproxy_namedbufferstatus_s status =
743 XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
744 __block kern_return_t kret = KERN_SUCCESS;
745
746 /* BEGIN IGNORE CODESTYLE */
747 tb_error_t ret = xnuproxy_cmd_namedbufferlayout(&xnuproxy_cmd_client,
748 id, start, npages,
749 ^(xnuproxy_cmd_namedbufferlayout__result_s result) {
750 xnuproxy_namedbufferrange_v_s *ranges;
751 ranges = xnuproxy_cmd_namedbufferlayout__result_get_success(&result);
752 status = XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS;
753 if (ranges != NULL) {
754 xnuproxy_namedbufferrange__v_visit(ranges,
755 ^(__unused size_t i,
756 const xnuproxy_namedbufferrange_s *item) {
757 /*
758 * Only fail once. TB want to iterate over the
759 * entire array so we have to keep going but
760 * once the first failure is seen, don't bother
761 * calling cb.
762 */
763 if (kret == KERN_SUCCESS) {
764 kret = cb(item->address, item->numpages);
765 }
766 });
767
768 return;
769 }
770
771 xnuproxy_namedbuffererror_s *error;
772 error = xnuproxy_cmd_namedbufferlayout__result_get_failure(&result);
773 assert3p(error, !=, NULL);
774
775 exclaves_debug_printf(show_errors,
776 "named buffer layout: failure %u, %u\n",
777 error->status, error->substatus);
778 status = error->status;
779 });
780
781 if (ret != TB_ERROR_SUCCESS) {
782 return KERN_FAILURE;
783 }
784
785 if (kret != KERN_SUCCESS) {
786 return kret;
787 }
788
789 if (status != XNUPROXY_NAMEDBUFFERSTATUS_SUCCESS) {
790 return status == XNUPROXY_NAMEDBUFFERSTATUS_INVALIDARGUMENT ?
791 KERN_INVALID_ARGUMENT : KERN_FAILURE;
792 }
793 /* END IGNORE CODESTYLE */
794
795 return KERN_SUCCESS;
796 }
797
798 /* -------------------------------------------------------------------------- */
799 #pragma mark exclaves xnu-proxy downcall
800
801 #define exclaves_xnuproxy_endpoint_call_show_progress(operation, step, \
802 eid, scid, status) \
803 exclaves_debug_printf(show_progress, \
804 "exclaves: xnu proxy endpoint " #operation " " #step ":\t" \
805 "endpoint id %ld scid 0x%lx status %u\n", \
806 (eid), (scid), (status))
807 OS_NOINLINE
808 kern_return_t
exclaves_xnuproxy_endpoint_call(Exclaves_L4_Word_t endpoint_id)809 exclaves_xnuproxy_endpoint_call(Exclaves_L4_Word_t endpoint_id)
810 {
811 kern_return_t kr = KERN_SUCCESS;
812 thread_t thread = current_thread();
813 bool interrupted = false;
814
815 Exclaves_L4_Word_t scid = thread->th_exclaves_ipc_ctx.scid;
816 Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_ctx.ipcb;
817 xnuproxy_msg_status_t status =
818 XNUPROXY_MSG_STATUS_PROCESSING;
819
820 XNUPROXY_CR_ENDPOINT_ID(ipcb) = endpoint_id;
821 XNUPROXY_CR_STATUS(ipcb) = status;
822
823 exclaves_xnuproxy_endpoint_call_show_progress(call, entry,
824 endpoint_id, scid, status);
825
826 assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0);
827 thread->th_exclaves_state |= TH_EXCLAVES_RPC;
828 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC)
829 | DBG_FUNC_START, scid, endpoint_id);
830
831 while (1) {
832 kr = exclaves_scheduler_resume_scheduling_context(
833 &thread->th_exclaves_ipc_ctx, interrupted);
834 assert(kr == KERN_SUCCESS || kr == KERN_ABORTED);
835
836 /* A wait was interrupted. */
837 interrupted = kr == KERN_ABORTED;
838
839 status = (xnuproxy_msg_status_t)
840 XNUPROXY_CR_STATUS(ipcb);
841
842 switch (status) {
843 case XNUPROXY_MSG_STATUS_PROCESSING:
844 exclaves_xnuproxy_endpoint_call_show_progress(call, yielded,
845 endpoint_id, scid, status);
846 continue;
847
848 case XNUPROXY_MSG_STATUS_REPLY:
849 exclaves_xnuproxy_endpoint_call_show_progress(call, returned,
850 endpoint_id, scid, status);
851 kr = KERN_SUCCESS;
852 break;
853
854 case XNUPROXY_MSG_STATUS_UPCALL:
855 thread->th_exclaves_state |= TH_EXCLAVES_UPCALL;
856 endpoint_id = XNUPROXY_CR_ENDPOINT_ID(ipcb);
857 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL)
858 | DBG_FUNC_START, scid, endpoint_id);
859 exclaves_xnuproxy_endpoint_call_show_progress(upcall, entry,
860 endpoint_id, scid, status);
861 kr = exclaves_call_upcall_handler(endpoint_id);
862 XNUPROXY_CR_STATUS(ipcb) =
863 XNUPROXY_MSG_STATUS_PROCESSING;
864 /* TODO: More state returned than Success or OperationInvalid? */
865 XNUPROXY_CR_RETVAL(ipcb) =
866 (kr == KERN_SUCCESS) ? Exclaves_L4_Success :
867 Exclaves_L4_ErrorOperationInvalid;
868 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL)
869 | DBG_FUNC_END);
870 thread->th_exclaves_state &= ~TH_EXCLAVES_UPCALL;
871 exclaves_xnuproxy_endpoint_call_show_progress(upcall, returned,
872 endpoint_id, scid,
873 (unsigned int)XNUPROXY_CR_RETVAL(ipcb));
874 continue;
875
876 default:
877 // Should we have an assert(valid return) here?
878 exclaves_xnuproxy_endpoint_call_show_progress(call, failed,
879 endpoint_id, scid, status);
880 kr = KERN_FAILURE;
881 break;
882 }
883 break;
884 }
885
886 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC)
887 | DBG_FUNC_END);
888 thread->th_exclaves_state &= ~TH_EXCLAVES_RPC;
889
890 return kr;
891 }
892
893
894 /* -------------------------------------------------------------------------- */
895 #pragma mark exclaves xnu-proxy initialisation
896
897 kern_return_t
exclaves_xnuproxy_init(uint64_t bootinfo_pa)898 exclaves_xnuproxy_init(uint64_t bootinfo_pa)
899 {
900 assert3u(bootinfo_pa, !=, 0);
901
902 kern_return_t kr = KERN_FAILURE;
903
904 void *bootinfo_va = (void *)phystokv(bootinfo_pa);
905 assert3p(bootinfo_va, !=, NULL);
906 const size_t bootinfo_size =
907 xnuproxy_bootinfo__marshal_sizeof(&(xnuproxy_bootinfo_s){});
908
909 __block uint64_t endpoint = 0;
910
911 /* BEGIN IGNORE CODESTYLE */
912 tb_error_t ret = xnuproxy_bootinfo__unmarshal(bootinfo_va,
913 bootinfo_size, ^(xnuproxy_bootinfo_s bootinfo) {
914
915 /* Do the version check. */
916 if (bootinfo.version != XNUPROXY_VERSION_CURRENT) {
917 exclaves_debug_printf(show_errors,
918 "exclaves: mismatched xnuproxy message version, "
919 "xnuproxy: %u, xnu: %u\n", bootinfo.version,
920 XNUPROXY_VERSION_CURRENT);
921 return;
922 }
923
924 exclaves_debug_printf(show_progress,
925 "exclaves: xnuproxy message version: 0x%u\n",
926 XNUPROXY_VERSION_CURRENT);
927
928
929 if (!pmap_valid_address(bootinfo.buffer)) {
930 exclaves_debug_printf(show_errors,
931 "exclaves: invalid bootstrap IPC buffer address: "
932 "0x%llx\n", bootinfo.buffer);
933 return;
934 }
935
936 exclaves_bootstrap_ctx.scid = bootinfo.scid;
937 exclaves_bootstrap_ctx.ipcb =
938 (Exclaves_L4_IpcBuffer_t *)phystokv(bootinfo.buffer);
939 assert3p(exclaves_bootstrap_ctx.ipcb, !=, NULL);
940 exclaves_bootstrap_ctx.usecnt = 1;
941
942 endpoint = bootinfo.endpointid;
943 });
944
945 /* END IGNORE CODESTYLE */
946 if (ret != TB_ERROR_SUCCESS) {
947 exclaves_debug_printf(show_errors,
948 "failed to unmarshal bootinfo\n");
949 return KERN_FAILURE;
950 }
951
952 /*
953 * Check to see if we bailed out of the unmarshal block early which
954 * would indicate a failure (for example the version check may have
955 * failed).
956 */
957 if (endpoint == 0) {
958 return KERN_FAILURE;
959 }
960
961 if (exclaves_bootstrap_ctx.ipcb == NULL) {
962 return KERN_FAILURE;
963 }
964
965 /* BEGIN IGNORE CODESTYLE */
966 tb_endpoint_t ep = tb_endpoint_create_with_value(
967 TB_TRANSPORT_TYPE_XNU, endpoint, TB_ENDPOINT_OPTIONS_NONE);
968 /* END IGNORE CODESTYLE */
969 ret = xnuproxy_cmd__init(&xnuproxy_cmd_client, ep);
970 if (ret != TB_ERROR_SUCCESS) {
971 exclaves_debug_printf(show_errors,
972 "failed to create xnuproxy endpoint\n");
973 return KERN_FAILURE;
974 }
975 /* Downcalls to xnu-proxy now supported. */
976
977 kr = exclaves_ipc_buffer_cache_init();
978 if (kr != KERN_SUCCESS) {
979 return kr;
980 }
981
982 return KERN_SUCCESS;
983 }
984
985 #endif /* CONFIG_EXCLAVES */
986