1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30 #include "exclaves_upcalls.h"
31
32 #if CONFIG_EXCLAVES
33
34 #if __has_include(<Tightbeam/tightbeam.h>)
35
36 #include <mach/exclaves_l4.h>
37
38 #include <stdint.h>
39
40 #include <Tightbeam/tightbeam.h>
41 #include <Tightbeam/tightbeam_private.h>
42
43 #include <kern/kalloc.h>
44 #include <kern/locks.h>
45 #include <kern/task.h>
46
47 #include <xnuproxy/exclaves.h>
48 #include <xnuproxy/messages.h>
49
50 #include "kern/exclaves.tightbeam.h"
51
52 #include "exclaves_boot.h"
53 #include "exclaves_conclave.h"
54 #include "exclaves_debug.h"
55 #include "exclaves_driverkit.h"
56 #include "exclaves_memory.h"
57 #include "exclaves_stackshot.h"
58 #include "exclaves_storage.h"
59 #include "exclaves_test_stackshot.h"
60 #include "exclaves_xnuproxy.h"
61
62 #include <sys/errno.h>
63
64 #define EXCLAVES_ID_HELLO_EXCLAVE_EP \
65 (exclaves_service_lookup(EXCLAVES_DOMAIN_KERNEL, \
66 "com.apple.service.ExclavesCHelloServer"))
67
68 #define EXCLAVES_ID_TIGHTBEAM_UPCALL \
69 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM)
70
71 #define EXCLAVES_ID_TIGHTBEAM_PMM_UPCALL_V2 \
72 ((exclaves_id_t)XNUPROXY_PMM_UPCALL_TIGHTBEAM_V2)
73
74 #define EXCLAVES_ID_TIGHTBEAM_UPCALL_V2 \
75 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM_V2)
76
77 extern lck_mtx_t exclaves_boot_lock;
78
79 typedef struct exclaves_upcall_handler_registration {
80 exclaves_upcall_handler_t handler;
81 void *context;
82 } exclaves_upcall_handler_registration_t;
83
84 static exclaves_upcall_handler_registration_t
85 exclaves_upcall_handlers[NUM_XNUPROXY_UPCALLS];
86
87 #define EXCLAVES_OBSOLETE_UPCALL_TESTING 1 // TODO: delete (rdar://123929546)
88 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
89 #if DEVELOPMENT || DEBUG
90 static kern_return_t
91 exclaves_test_hello_upcall_handler(void *, exclaves_tag_t *, exclaves_badge_t);
92 #endif /* DEVELOPMENT || DEBUG */
93 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
94
95
96 /* -------------------------------------------------------------------------- */
97 #pragma mark Upcall Callouts
98
99 static tb_error_t
100 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t));
101
102 /*
103 * Tightbeam upcall callout table.
104 * Don't add inline functionality here, instead call directly into your
105 * sub-system.
106 */
107
108 static const xnuupcalls_xnuupcalls__server_s exclaves_tightbeam_upcalls = {
109 /* BEGIN IGNORE CODESTYLE */
110 /* Uncrustify doesn't deal well with Blocks. */
111 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
112 return exclaves_helloupcall(arg, completion);
113 },
114
115 .alloc = ^(const uint32_t npages, xnuupcalls_pagekind_s kind,
116 tb_error_t (^completion)(xnuupcalls_pagelist_s)) {
117 return exclaves_memory_upcall_legacy_alloc(npages, kind, completion);
118 },
119
120 .alloc_ext = ^(const uint32_t npages, xnuupcalls_pageallocflags_s flags,
121 tb_error_t (^completion)(xnuupcalls_pagelist_s)) {
122 return exclaves_memory_upcall_legacy_alloc_ext(npages, flags, completion);
123 },
124
125 .free = ^(const uint32_t pages[_Nonnull EXCLAVES_MEMORY_MAX_REQUEST],
126 const uint32_t npages, const xnuupcalls_pagekind_s kind,
127 tb_error_t (^completion)(void)) {
128 return exclaves_memory_upcall_legacy_free(pages, npages, kind, completion);
129 },
130
131 .free_ext = ^(const uint32_t pages[_Nonnull EXCLAVES_MEMORY_MAX_REQUEST],
132 const uint32_t npages, xnuupcalls_pagefreeflags_s flags, tb_error_t (^completion)(void)) {
133 return exclaves_memory_upcall_legacy_free_ext(pages, npages, flags, completion);
134 },
135
136 .root = ^(const uint8_t exclaveid[_Nonnull 32],
137 tb_error_t (^completion)(xnuupcalls_xnuupcalls_root__result_s)) {
138 return exclaves_storage_upcall_legacy_root(exclaveid, completion);
139 },
140
141 .open = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
142 const uint8_t name[_Nonnull 256],
143 tb_error_t (^completion)(xnuupcalls_xnuupcalls_open__result_s)) {
144 return exclaves_storage_upcall_legacy_open(fstag, rootid, name, completion);
145 },
146
147 .close = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
148 tb_error_t (^completion)(xnuupcalls_xnuupcalls_close__result_s)) {
149 return exclaves_storage_upcall_legacy_close(fstag, fileid, completion);
150 },
151
152 .create = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
153 const uint8_t name[_Nonnull 256],
154 tb_error_t (^completion)(xnuupcalls_xnuupcalls_create__result_s)) {
155 return exclaves_storage_upcall_legacy_create(fstag, rootid, name, completion);
156 },
157
158 .read = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
159 const struct xnuupcalls_iodesc_s *descriptor,
160 tb_error_t (^completion)(xnuupcalls_xnuupcalls_read__result_s)) {
161 return exclaves_storage_upcall_legacy_read(fstag, fileid, descriptor, completion);
162 },
163
164 .write = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
165 const struct xnuupcalls_iodesc_s *descriptor,
166 tb_error_t (^completion)(xnuupcalls_xnuupcalls_write__result_s)) {
167 return exclaves_storage_upcall_legacy_write(fstag, fileid, descriptor, completion);
168 },
169
170 .remove = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
171 const uint8_t name[_Nonnull 256],
172 tb_error_t (^completion)(xnuupcalls_xnuupcalls_remove__result_s)) {
173 return exclaves_storage_upcall_legacy_remove(fstag, rootid, name, completion);
174 },
175
176 .sync = ^(const enum xnuupcalls_fstag_s fstag,
177 const enum xnuupcalls_syncop_s op,
178 const uint64_t fileid,
179 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sync__result_s)) {
180 return exclaves_storage_upcall_legacy_sync(fstag, op, fileid, completion);
181 },
182
183 .readdir = ^(const enum xnuupcalls_fstag_s fstag,
184 const uint64_t fileid, const uint64_t buf,
185 const uint32_t length,
186 tb_error_t (^completion)(xnuupcalls_xnuupcalls_readdir__result_s)) {
187 return exclaves_storage_upcall_legacy_readdir(fstag, fileid, buf, length, completion);
188 },
189
190 .getsize = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
191 tb_error_t (^completion)(xnuupcalls_xnuupcalls_getsize__result_s)) {
192 return exclaves_storage_upcall_legacy_getsize(fstag, fileid, completion);
193 },
194
195 .sealstate = ^(const enum xnuupcalls_fstag_s fstag,
196 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sealstate__result_s)) {
197 return exclaves_storage_upcall_legacy_sealstate(fstag, completion);
198 },
199
200 .irq_register = ^(const uint64_t id, const int32_t index,
201 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_register__result_s)) {
202 return exclaves_driverkit_upcall_legacy_irq_register(id, index, completion);
203 },
204
205 .irq_remove = ^(const uint64_t id, const int32_t index,
206 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_remove__result_s)) {
207 return exclaves_driverkit_upcall_legacy_irq_remove(id, index, completion);
208 },
209
210 .irq_enable = ^(const uint64_t id, const int32_t index,
211 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_enable__result_s)) {
212 return exclaves_driverkit_upcall_legacy_irq_enable(id, index, completion);
213 },
214
215 .irq_disable = ^(const uint64_t id, const int32_t index,
216 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_disable__result_s)) {
217 return exclaves_driverkit_upcall_legacy_irq_disable(id, index, completion);
218 },
219
220 .timer_register = ^(const uint64_t id,
221 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_register__result_s)) {
222 return exclaves_driverkit_upcall_legacy_timer_register(id, completion);
223 },
224
225 .timer_remove = ^(const uint64_t id, const uint32_t timer_id,
226 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_remove__result_s)) {
227 return exclaves_driverkit_upcall_legacy_timer_remove(id, timer_id, completion);
228 },
229
230 .timer_enable = ^(const uint64_t id, const uint32_t timer_id,
231 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_enable__result_s)) {
232 return exclaves_driverkit_upcall_legacy_timer_enable(id, timer_id, completion);
233 },
234
235 .timer_disable = ^(const uint64_t id, const uint32_t timer_id,
236 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_disable__result_s)) {
237 return exclaves_driverkit_upcall_legacy_timer_disable(id, timer_id, completion);
238 },
239
240 .timer_set_timeout = ^(const uint64_t id, const uint32_t timer_id,
241 const struct xnuupcalls_drivertimerspecification_s *duration,
242 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_set_timeout__result_s)) {
243 return exclaves_driverkit_upcall_legacy_timer_set_timeout(id, timer_id, duration, completion);
244 },
245
246 .timer_cancel_timeout = ^(const uint64_t id, const uint32_t timer_id,
247 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_cancel_timeout__result_s)) {
248 return exclaves_driverkit_upcall_legacy_timer_cancel_timeout(id, timer_id, completion);
249 },
250
251 .lock_wl = ^(const uint64_t id,
252 tb_error_t (^completion)(xnuupcalls_xnuupcalls_lock_wl__result_s)) {
253 return exclaves_driverkit_upcall_legacy_lock_wl(id, completion);
254 },
255
256 .unlock_wl = ^(const uint64_t id,
257 tb_error_t (^completion)(xnuupcalls_xnuupcalls_unlock_wl__result_s)) {
258 return exclaves_driverkit_upcall_legacy_unlock_wl(id, completion);
259 },
260
261 .async_notification_signal = ^(const uint64_t id,
262 const uint32_t notificationID,
263 tb_error_t (^completion)(xnuupcalls_xnuupcalls_async_notification_signal__result_s)) {
264 return exclaves_driverkit_upcall_legacy_async_notification_signal(id,
265 notificationID, completion);
266 },
267
268 .mapper_activate = ^(const uint64_t id, const uint32_t mapperIndex,
269 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_activate__result_s)) {
270 return exclaves_driverkit_upcall_legacy_mapper_activate(id,
271 mapperIndex, completion);
272 },
273
274 .mapper_deactivate = ^(const uint64_t id, const uint32_t mapperIndex,
275 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_deactivate__result_s)) {
276 return exclaves_driverkit_upcall_legacy_mapper_deactivate(id,
277 mapperIndex, completion);
278 },
279
280 .notification_signal = ^(const uint64_t id, const uint32_t mask,
281 tb_error_t (^completion)(xnuupcalls_xnuupcalls_notification_signal__result_s)) {
282 return exclaves_driverkit_upcall_legacy_notification_signal(id, mask,
283 completion);
284 },
285
286 .ane_setpowerstate = ^(const uint64_t id, const uint32_t desiredState,
287 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_setpowerstate__result_s)) {
288 return exclaves_driverkit_upcall_legacy_ane_setpowerstate(id, desiredState,
289 completion);
290 },
291
292 .ane_worksubmit = ^(const uint64_t id, const uint64_t requestID,
293 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
294 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_worksubmit__result_s)) {
295 return exclaves_driverkit_upcall_legacy_ane_worksubmit(id, requestID,
296 taskDescriptorCount, submitTimestamp, completion);
297 },
298
299 .ane_workbegin = ^(const uint64_t id, const uint64_t requestID,
300 const uint64_t beginTimestamp,
301 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workbegin__result_s)) {
302 return exclaves_driverkit_upcall_legacy_ane_workbegin(id, requestID,
303 beginTimestamp, completion);
304 },
305
306 .ane_workend = ^(const uint64_t id, const uint64_t requestID,
307 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workend__result_s)) {
308 return exclaves_driverkit_upcall_legacy_ane_workend(id, requestID, completion);
309 },
310
311 .conclave_suspend = ^(const uint32_t flags,
312 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_suspend__result_s)) {
313 return exclaves_conclave_upcall_legacy_suspend(flags, completion);
314 },
315
316 .conclave_stop = ^(const uint32_t flags,
317 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_stop__result_s)) {
318 return exclaves_conclave_upcall_legacy_stop(flags, completion);
319 },
320 .conclave_crash_info = ^(const xnuupcalls_conclavesharedbuffer_s *shared_buf,
321 const uint32_t length,
322 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_crash_info__result_s)) {
323 return exclaves_conclave_upcall_legacy_crash_info(shared_buf, length, completion);
324 },
325 /* END IGNORE CODESTYLE */
326 };
327
328 static const xnuupcallsv2_memoryupcallsprivate__server_s exclaves_tightbeam_memory_upcalls_v2 = {
329 /* BEGIN IGNORE CODESTYLE */
330 /* Uncrustify doesn't deal well with Blocks. */
331 .alloc = ^(const uint32_t npages, xnuupcallsv2_pagekind_s kind,
332 tb_error_t (^completion)(xnuupcallsv2_pagelist_s)) {
333 return exclaves_memory_upcall_alloc(npages, kind, completion);
334 },
335
336 .alloc_ext = ^(const uint32_t npages, xnuupcallsv2_pageallocflagsv2_s flags, tb_error_t (^completion)(xnuupcallsv2_pagelist_s)) {
337 return exclaves_memory_upcall_alloc_ext(npages, flags, completion);
338 },
339
340 .free = ^(const xnuupcallsv2_pagelist_s pages, const xnuupcallsv2_pagekind_s kind,
341 tb_error_t (^completion)(void)) {
342 return exclaves_memory_upcall_free(pages, kind, completion);
343 },
344
345 .free_ext = ^(const xnuupcallsv2_pagelist_s pages, xnuupcallsv2_pagefreeflagsv2_s flags, tb_error_t (^completion)(void)) {
346 return exclaves_memory_upcall_free_ext(pages, flags, completion);
347 },
348 /* END IGNORE CODESTYLE */
349 };
350
351 #pragma clang diagnostic push
352 #pragma clang diagnostic ignored "-Wunused-parameter"
353 static const xnuupcallsv2_xnuupcalls__server_s exclaves_tightbeam_upcalls_v2 = {
354 /* BEGIN IGNORE CODESTYLE */
355 /* Uncrustify doesn't deal well with Blocks. */
356 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
357 return exclaves_helloupcall(arg, completion);
358 },
359
360 /* Unset memoryupcalls handlers */
361
362 .root = ^(const uint8_t exclaveid[_Nonnull 32],
363 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_root__result_s)) {
364 return exclaves_storage_upcall_root(exclaveid, completion);
365 },
366
367 .rootex = ^(const uint32_t fstag, const uint8_t exclaveid[_Nonnull 32],
368 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_rootex__result_s)) {
369 return exclaves_storage_upcall_rootex(fstag, exclaveid, completion);
370 },
371
372 .open = ^(const uint32_t fstag, const uint64_t rootid,
373 const uint8_t name[_Nonnull 256],
374 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_open__result_s)) {
375 return exclaves_storage_upcall_open(fstag, rootid, name, completion);
376 },
377
378 .close = ^(const uint32_t fstag, const uint64_t fileid,
379 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_close__result_s)) {
380 return exclaves_storage_upcall_close(fstag, fileid, completion);
381 },
382
383 .create = ^(const uint32_t fstag, const uint64_t rootid,
384 const uint8_t name[_Nonnull 256],
385 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_create__result_s)) {
386 return exclaves_storage_upcall_create(fstag, rootid, name, completion);
387 },
388
389 .read = ^(const uint32_t fstag, const uint64_t fileid,
390 const struct xnuupcallsv2_iodesc_s *descriptor,
391 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_read__result_s)) {
392 return exclaves_storage_upcall_read(fstag, fileid, descriptor, completion);
393 },
394
395 .write = ^(const uint32_t fstag, const uint64_t fileid,
396 const struct xnuupcallsv2_iodesc_s *descriptor,
397 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_write__result_s)) {
398 return exclaves_storage_upcall_write(fstag, fileid, descriptor, completion);
399 },
400
401 .remove = ^(const uint32_t fstag, const uint64_t rootid,
402 const uint8_t name[_Nonnull 256],
403 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_remove__result_s)) {
404 return exclaves_storage_upcall_remove(fstag, rootid, name, completion);
405 },
406
407 .sync = ^(const uint32_t fstag,
408 const xnuupcallsv2_syncop_s * _Nonnull op,
409 const uint64_t fileid,
410 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sync__result_s)) {
411 return exclaves_storage_upcall_sync(fstag, op, fileid, completion);
412 },
413
414 .readdir = ^(const uint32_t fstag,
415 const uint64_t fileid, const uint64_t buf,
416 const uint32_t length,
417 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_readdir__result_s)) {
418 return exclaves_storage_upcall_readdir(fstag, fileid, buf, length, completion);
419 },
420
421 .getsize = ^(const uint32_t fstag, const uint64_t fileid,
422 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_getsize__result_s)) {
423 return exclaves_storage_upcall_getsize(fstag, fileid, completion);
424 },
425
426 .sealstate = ^(const uint32_t fstag,
427 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sealstate__result_s)) {
428 return exclaves_storage_upcall_sealstate(fstag, completion);
429 },
430
431 .queryvolumegroup = ^(const uint8_t vguuid[_Nonnull 37],
432 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_queryvolumegroup__result_s)) {
433 return exclaves_storage_upcall_queryvolumegroup(vguuid, completion);
434 },
435
436 .irqregister = ^(const uint64_t id, const int32_t index,
437 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqregister__result_s)) {
438 return exclaves_driverkit_upcall_irq_register(id, index, completion);
439 },
440
441 .irqremove = ^(const uint64_t id, const int32_t index,
442 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqremove__result_s)) {
443 return exclaves_driverkit_upcall_irq_remove(id, index, completion);
444 },
445
446 .irqenable = ^(const uint64_t id, const int32_t index,
447 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqenable__result_s)) {
448 return exclaves_driverkit_upcall_irq_enable(id, index, completion);
449 },
450
451 .irqdisable = ^(const uint64_t id, const int32_t index,
452 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqdisable__result_s)) {
453 return exclaves_driverkit_upcall_irq_disable(id, index, completion);
454 },
455
456 .timerregister = ^(const uint64_t id,
457 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerregister__result_s)) {
458 return exclaves_driverkit_upcall_timer_register(id, completion);
459 },
460
461 .timerremove = ^(const uint64_t id, const uint32_t timerid,
462 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerremove__result_s)) {
463 return exclaves_driverkit_upcall_timer_remove(id, timerid, completion);
464 },
465
466 .timerenable = ^(const uint64_t id, const uint32_t timerid,
467 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerenable__result_s)) {
468 return exclaves_driverkit_upcall_timer_enable(id, timerid, completion);
469 },
470
471 .timerdisable = ^(const uint64_t id, const uint32_t timerid,
472 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerdisable__result_s)) {
473 return exclaves_driverkit_upcall_timer_disable(id, timerid, completion);
474 },
475
476 .timersettimeout = ^(const uint64_t id, const uint32_t timerid,
477 const struct xnuupcallsv2_drivertimerspecification_s *duration,
478 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timersettimeout__result_s)) {
479 return exclaves_driverkit_upcall_timer_set_timeout(id, timerid, duration, completion);
480 },
481
482 .timercanceltimeout = ^(const uint64_t id, const uint32_t timerid,
483 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timercanceltimeout__result_s)) {
484 return exclaves_driverkit_upcall_timer_cancel_timeout(id, timerid, completion);
485 },
486
487 .lockworkloop = ^(const uint64_t id,
488 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_lockworkloop__result_s)) {
489 return exclaves_driverkit_upcall_lock_workloop(id, completion);
490 },
491
492 .unlockworkloop = ^(const uint64_t id,
493 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_unlockworkloop__result_s)) {
494 return exclaves_driverkit_upcall_unlock_workloop(id, completion);
495 },
496
497 .asyncnotificationsignal = ^(const uint64_t id,
498 const uint32_t notificationID,
499 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_asyncnotificationsignal__result_s)) {
500 return exclaves_driverkit_upcall_async_notification_signal(id,
501 notificationID, completion);
502 },
503
504 .mapperactivate = ^(const uint64_t id, const uint32_t mapperIndex,
505 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperactivate__result_s)) {
506 return exclaves_driverkit_upcall_mapper_activate(id,
507 mapperIndex, completion);
508 },
509
510 .mapperdeactivate = ^(const uint64_t id, const uint32_t mapperIndex,
511 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperdeactivate__result_s)) {
512 return exclaves_driverkit_upcall_mapper_deactivate(id,
513 mapperIndex, completion);
514 },
515
516 .anesetpowerstate = ^(const uint64_t id, const uint32_t desiredState,
517 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_anesetpowerstate__result_s)) {
518 return exclaves_driverkit_upcall_ane_setpowerstate(id, desiredState,
519 completion);
520 },
521
522 .aneworksubmit = ^(const uint64_t id, const uint64_t requestID,
523 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
524 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworksubmit__result_s)) {
525 return exclaves_driverkit_upcall_ane_worksubmit(id, requestID,
526 taskDescriptorCount, submitTimestamp, completion);
527 },
528
529 .aneworkbegin = ^(const uint64_t id, const uint64_t requestID,
530 const uint64_t beginTimestamp,
531 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkbegin__result_s)) {
532 return exclaves_driverkit_upcall_ane_workbegin(id, requestID,
533 beginTimestamp, completion);
534 },
535
536 .aneworkend = ^(const uint64_t id, const uint64_t requestID,
537 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkend__result_s)) {
538 return exclaves_driverkit_upcall_ane_workend(id, requestID, completion);
539 },
540
541 .notificationsignal = ^(const uint64_t id, const uint32_t mask,
542 tb_error_t (^completion)(xnuupcallsv2_notificationupcallsprivate_notificationsignal__result_s)) {
543 return exclaves_driverkit_upcall_notification_signal(id, mask,
544 completion);
545 },
546
547 .suspend = ^(const uint32_t flags,
548 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_suspend__result_s)) {
549 return exclaves_conclave_upcall_suspend(flags, completion);
550 },
551
552 .stop = ^(const uint32_t flags,
553 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_stop__result_s)) {
554 return exclaves_conclave_upcall_stop(flags, completion);
555 },
556 .crashinfo = ^(const xnuupcallsv2_conclavesharedbuffer_s *shared_buf,
557 const uint32_t length,
558 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_crashinfo__result_s)) {
559 return exclaves_conclave_upcall_crash_info(shared_buf, length, completion);
560 },
561 /* END IGNORE CODESTYLE */
562 };
563 #pragma clang diagnostic pop
564
565 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)566 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
567 exclaves_upcall_handler_t upcall_handler)
568 {
569 assert3u(upcall_id, <, NUM_XNUPROXY_UPCALLS);
570 assert(upcall_handler != NULL);
571
572 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
573 assert(exclaves_upcall_handlers[upcall_id].handler == NULL);
574
575 exclaves_upcall_handlers[upcall_id] =
576 (exclaves_upcall_handler_registration_t){
577 .handler = upcall_handler,
578 .context = upcall_context,
579 };
580
581 return KERN_SUCCESS;
582 }
583
584 kern_return_t
exclaves_upcall_init(void)585 exclaves_upcall_init(void)
586 {
587 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
588
589 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
590 #if DEVELOPMENT || DEBUG
591 kern_return_t kr;
592 kr = exclaves_register_upcall_handler(
593 XNUPROXY_UPCALL_HELLOUPCALL, NULL,
594 exclaves_test_hello_upcall_handler);
595 assert3u(kr, ==, KERN_SUCCESS);
596 #endif /* DEVELOPMENT || DEBUG */
597 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
598
599 tb_endpoint_t tb_upcall_ep = tb_endpoint_create_with_value(
600 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL,
601 TB_ENDPOINT_OPTIONS_NONE);
602
603 tb_endpoint_t tb_pmm_upcall_v2_ep = tb_endpoint_create_with_value(
604 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_PMM_UPCALL_V2,
605 TB_ENDPOINT_OPTIONS_NONE);
606
607 tb_endpoint_t tb_upcall_v2_ep = tb_endpoint_create_with_value(
608 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL_V2,
609 TB_ENDPOINT_OPTIONS_NONE);
610
611 #pragma clang diagnostic push
612 #pragma clang diagnostic ignored "-Wcast-qual" /* FIXME: rdar://103647654 */
613 tb_error_t error = xnuupcalls_xnuupcalls__server_start(tb_upcall_ep,
614 (xnuupcalls_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls);
615 tb_error_t error2 = xnuupcallsv2_memoryupcallsprivate__server_start(
616 tb_pmm_upcall_v2_ep,
617 (xnuupcallsv2_memoryupcallsprivate__server_s*)&exclaves_tightbeam_memory_upcalls_v2);
618 tb_error_t error3 = xnuupcallsv2_xnuupcalls__server_start(tb_upcall_v2_ep,
619 (xnuupcallsv2_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls_v2);
620 #pragma clang diagnostic pop
621
622 return (error == TB_ERROR_SUCCESS
623 && error2 == TB_ERROR_SUCCESS
624 && error3 == TB_ERROR_SUCCESS)
625 ? KERN_SUCCESS : KERN_FAILURE;
626 }
627
628 /* Unslid pointers defining the range of code which triggers upcall handlers */
629 uintptr_t exclaves_upcall_range_start;
630 uintptr_t exclaves_upcall_range_end;
631
632 __startup_func
633 static void
initialize_exclaves_upcall_range(void)634 initialize_exclaves_upcall_range(void)
635 {
636 exclaves_upcall_range_start = VM_KERNEL_UNSLIDE(&exclaves_upcall_start_label);
637 assert3u(exclaves_upcall_range_start, !=, 0);
638 exclaves_upcall_range_end = VM_KERNEL_UNSLIDE(&exclaves_upcall_end_label);
639 assert3u(exclaves_upcall_range_end, !=, 0);
640 }
641 STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, initialize_exclaves_upcall_range);
642
643 bool
exclaves_upcall_in_range(uintptr_t addr,bool slid)644 exclaves_upcall_in_range(uintptr_t addr, bool slid)
645 {
646 return slid ?
647 exclaves_in_range(addr, (uintptr_t)&exclaves_upcall_start_label, (uintptr_t)&exclaves_upcall_end_label) :
648 exclaves_in_range(addr, exclaves_upcall_range_start, exclaves_upcall_range_end);
649 }
650
651 OS_NOINLINE
652 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)653 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
654 {
655 kern_return_t kr = KERN_INVALID_CAPABILITY;
656
657 __assert_only thread_t thread = current_thread();
658 assert3u(thread->th_exclaves_state & TH_EXCLAVES_UPCALL, !=, 0);
659
660 exclaves_tag_t tag = Exclaves_L4_GetMessageTag();
661
662 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
663 exclaves_badge_t badge = XNUPROXY_CR_UPCALL_BADGE(ipcb);
664
665 exclaves_upcall_handler_registration_t upcall_handler = {};
666
667 if (upcall_id < NUM_XNUPROXY_UPCALLS) {
668 upcall_handler = exclaves_upcall_handlers[upcall_id];
669 }
670 if (upcall_handler.handler) {
671 __asm__ volatile ( "EXCLAVES_UPCALL_START:\n\t");
672 kr = upcall_handler.handler(upcall_handler.context, &tag, badge);
673 __asm__ volatile ("EXCLAVES_UPCALL_END:\n\t");
674 Exclaves_L4_SetMessageTag(tag);
675 }
676
677 return kr;
678 }
679
680 /* -------------------------------------------------------------------------- */
681 #pragma mark Testing
682
683
684 static tb_error_t
685 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t))
686 {
687 #if DEVELOPMENT || DEBUG
688 exclaves_debug_printf(show_test_output,
689 "%s: Hello Tightbeam Upcall!\n", __func__);
690 tb_error_t ret = completion(~arg);
691 task_stop_conclave_upcall();
692 STACKSHOT_TESTPOINT(TP_UPCALL);
693 /* Emit kdebug event for kperf sampling testing */
694 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
695 return ret;
696 #else
697 (void)arg;
698 (void)completion;
699
700 return TB_ERROR_SUCCESS;
701 #endif /* DEVELOPMENT || DEBUG */
702 }
703
704 #if DEVELOPMENT || DEBUG
705
706 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
707 static kern_return_t
exclaves_test_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)708 exclaves_test_upcall_handler(void *context, exclaves_tag_t *tag,
709 exclaves_badge_t badge)
710 {
711 #pragma unused(context, badge)
712 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
713 assert(ipcb != NULL);
714
715 Exclaves_L4_Word_t mrs = Exclaves_L4_MessageTag_Mrs(*tag);
716 assert(mrs < Exclaves_L4_IpcBuffer_Mrs);
717 Exclaves_L4_Word_t crs = Exclaves_L4_MessageTag_Crs(*tag);
718 assert(crs == 0);
719 Exclaves_L4_Word_t label = Exclaves_L4_MessageTag_Label(*tag);
720
721 /* setup test reply message */
722 *tag = Exclaves_L4_MessageTag(mrs, 0, ~label, Exclaves_L4_False);
723 for (int i = 0; i < mrs; i++) {
724 Exclaves_L4_SetMessageMr(i, ~Exclaves_L4_GetMessageMr(i));
725 }
726
727 return KERN_SUCCESS;
728 }
729 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
730
731 static int
exclaves_hello_upcall_test(__unused int64_t in,int64_t * out)732 exclaves_hello_upcall_test(__unused int64_t in, int64_t *out)
733 {
734 tb_error_t tb_result;
735 exclaveschelloserver_tests_s client;
736 const unsigned long request = 0xdecafbadfeedfaceul;
737
738 if (exclaves_get_status() != EXCLAVES_STATUS_AVAILABLE) {
739 exclaves_debug_printf(show_test_output,
740 "%s: SKIPPED: Exclaves not available\n", __func__);
741 *out = -1;
742 return 0;
743 }
744
745 exclaves_debug_printf(show_test_output, "%s: STARTING\n", __func__);
746
747 tb_endpoint_t ep = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU,
748 EXCLAVES_ID_HELLO_EXCLAVE_EP, TB_ENDPOINT_OPTIONS_NONE);
749
750 tb_result = exclaveschelloserver_tests__init(&client, ep);
751 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
752
753 tb_result = exclaveschelloserver_tests_default_upcall(&client, request, ^(exclaveschelloserver_result_s result) {
754 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
755 assert3u(result.result, ==, 1);
756 assert3u(result.reply, ==, ((request >> 32) | (request << 32)));
757 });
758
759 exclaves_debug_printf(show_test_output, "%s: SUCCESS\n", __func__);
760 *out = 1;
761
762 return KERN_SUCCESS;
763 }
764 SYSCTL_TEST_REGISTER(exclaves_hello_upcall_test, exclaves_hello_upcall_test);
765
766 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
767 static kern_return_t
exclaves_test_hello_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)768 exclaves_test_hello_upcall_handler(void *context, exclaves_tag_t *tag,
769 exclaves_badge_t badge)
770 {
771 /* HelloUpcall test handler */
772 assert(context == NULL);
773 exclaves_debug_printf(show_test_output, "%s: Hello Upcall!\n", __func__);
774 task_stop_conclave_upcall();
775 STACKSHOT_TESTPOINT(TP_UPCALL);
776 /* Emit kdebug event for kperf sampling testing */
777 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
778 return exclaves_test_upcall_handler(context, tag, badge);
779 }
780 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
781
782 #endif /* DEVELOPMENT || DEBUG */
783
784 #endif /* __has_include(<Tightbeam/tightbeam.h>) */
785
786 #else /* CONFIG_EXCLAVES */
787
788 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)789 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
790 {
791 (void)upcall_id;
792 return KERN_NOT_SUPPORTED;
793 }
794
795 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)796 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
797 exclaves_upcall_handler_t upcall_handler)
798 {
799 (void)upcall_id;
800 (void)upcall_context;
801 (void)upcall_handler;
802
803 return KERN_NOT_SUPPORTED;
804 }
805
806 #endif /* CONFIG_EXCLAVES */
807