1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30 #include "exclaves_upcalls.h"
31
32 #if CONFIG_EXCLAVES
33
34 #if __has_include(<Tightbeam/tightbeam.h>)
35
36 #include <mach/exclaves_l4.h>
37
38 #include <stdint.h>
39
40 #include <Tightbeam/tightbeam.h>
41 #include <Tightbeam/tightbeam_private.h>
42
43 #include <kern/kalloc.h>
44 #include <kern/locks.h>
45 #include <kern/task.h>
46
47 #include <xnuproxy/exclaves.h>
48 #include <xnuproxy/messages.h>
49
50 #include "kern/exclaves.tightbeam.h"
51
52 #include "exclaves_boot.h"
53 #include "exclaves_conclave.h"
54 #include "exclaves_debug.h"
55 #include "exclaves_driverkit.h"
56 #include "exclaves_memory.h"
57 #include "exclaves_stackshot.h"
58 #include "exclaves_storage.h"
59 #include "exclaves_test_stackshot.h"
60 #include "exclaves_xnuproxy.h"
61 #include "exclaves_aoe.h"
62
63 #include <sys/errno.h>
64
65 #define EXCLAVES_ID_HELLO_EXCLAVE_EP \
66 (exclaves_service_lookup(EXCLAVES_DOMAIN_KERNEL, \
67 "com.apple.service.ExclavesCHelloServer"))
68
69 #define EXCLAVES_ID_TIGHTBEAM_UPCALL \
70 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM)
71
72 #define EXCLAVES_ID_TIGHTBEAM_PMM_UPCALL_V2 \
73 ((exclaves_id_t)XNUPROXY_PMM_UPCALL_TIGHTBEAM_V2)
74
75 #define EXCLAVES_ID_TIGHTBEAM_UPCALL_V2 \
76 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM_V2)
77
78 extern lck_mtx_t exclaves_boot_lock;
79
80 typedef struct exclaves_upcall_handler_registration {
81 exclaves_upcall_handler_t handler;
82 void *context;
83 } exclaves_upcall_handler_registration_t;
84
85 static exclaves_upcall_handler_registration_t
86 exclaves_upcall_handlers[NUM_XNUPROXY_UPCALLS];
87
88 #define EXCLAVES_OBSOLETE_UPCALL_TESTING 1 // TODO: delete (rdar://123929546)
89 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
90 #if DEVELOPMENT || DEBUG
91 static kern_return_t
92 exclaves_test_hello_upcall_handler(void *, exclaves_tag_t *, exclaves_badge_t);
93 #endif /* DEVELOPMENT || DEBUG */
94 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
95
96
97 /* -------------------------------------------------------------------------- */
98 #pragma mark Upcall Callouts
99
100 static tb_error_t
101 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t));
102
103 /*
104 * Tightbeam upcall callout table.
105 * Don't add inline functionality here, instead call directly into your
106 * sub-system.
107 */
108
109 static const xnuupcalls_xnuupcalls__server_s exclaves_tightbeam_upcalls = {
110 /* BEGIN IGNORE CODESTYLE */
111 /* Uncrustify doesn't deal well with Blocks. */
112 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
113 return exclaves_helloupcall(arg, completion);
114 },
115
116 .alloc = ^(const uint32_t npages, xnuupcalls_pagekind_s kind,
117 tb_error_t (^completion)(xnuupcalls_pagelist_s)) {
118 return exclaves_memory_upcall_legacy_alloc(npages, kind, completion);
119 },
120
121 .alloc_ext = ^(const uint32_t npages, xnuupcalls_pageallocflags_s flags,
122 tb_error_t (^completion)(xnuupcalls_pagelist_s)) {
123 return exclaves_memory_upcall_legacy_alloc_ext(npages, flags, completion);
124 },
125
126 .free = ^(const uint32_t pages[_Nonnull EXCLAVES_MEMORY_MAX_REQUEST],
127 const uint32_t npages, const xnuupcalls_pagekind_s kind,
128 tb_error_t (^completion)(void)) {
129 return exclaves_memory_upcall_legacy_free(pages, npages, kind, completion);
130 },
131
132 .free_ext = ^(const uint32_t pages[_Nonnull EXCLAVES_MEMORY_MAX_REQUEST],
133 const uint32_t npages, xnuupcalls_pagefreeflags_s flags, tb_error_t (^completion)(void)) {
134 return exclaves_memory_upcall_legacy_free_ext(pages, npages, flags, completion);
135 },
136
137 .root = ^(const uint8_t exclaveid[_Nonnull 32],
138 tb_error_t (^completion)(xnuupcalls_xnuupcalls_root__result_s)) {
139 return exclaves_storage_upcall_legacy_root(exclaveid, completion);
140 },
141
142 .open = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
143 const uint8_t name[_Nonnull 256],
144 tb_error_t (^completion)(xnuupcalls_xnuupcalls_open__result_s)) {
145 return exclaves_storage_upcall_legacy_open(fstag, rootid, name, completion);
146 },
147
148 .close = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
149 tb_error_t (^completion)(xnuupcalls_xnuupcalls_close__result_s)) {
150 return exclaves_storage_upcall_legacy_close(fstag, fileid, completion);
151 },
152
153 .create = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
154 const uint8_t name[_Nonnull 256],
155 tb_error_t (^completion)(xnuupcalls_xnuupcalls_create__result_s)) {
156 return exclaves_storage_upcall_legacy_create(fstag, rootid, name, completion);
157 },
158
159 .read = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
160 const struct xnuupcalls_iodesc_s *descriptor,
161 tb_error_t (^completion)(xnuupcalls_xnuupcalls_read__result_s)) {
162 return exclaves_storage_upcall_legacy_read(fstag, fileid, descriptor, completion);
163 },
164
165 .write = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
166 const struct xnuupcalls_iodesc_s *descriptor,
167 tb_error_t (^completion)(xnuupcalls_xnuupcalls_write__result_s)) {
168 return exclaves_storage_upcall_legacy_write(fstag, fileid, descriptor, completion);
169 },
170
171 .remove = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
172 const uint8_t name[_Nonnull 256],
173 tb_error_t (^completion)(xnuupcalls_xnuupcalls_remove__result_s)) {
174 return exclaves_storage_upcall_legacy_remove(fstag, rootid, name, completion);
175 },
176
177 .sync = ^(const enum xnuupcalls_fstag_s fstag,
178 const enum xnuupcalls_syncop_s op,
179 const uint64_t fileid,
180 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sync__result_s)) {
181 return exclaves_storage_upcall_legacy_sync(fstag, op, fileid, completion);
182 },
183
184 .readdir = ^(const enum xnuupcalls_fstag_s fstag,
185 const uint64_t fileid, const uint64_t buf,
186 const uint32_t length,
187 tb_error_t (^completion)(xnuupcalls_xnuupcalls_readdir__result_s)) {
188 return exclaves_storage_upcall_legacy_readdir(fstag, fileid, buf, length, completion);
189 },
190
191 .getsize = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
192 tb_error_t (^completion)(xnuupcalls_xnuupcalls_getsize__result_s)) {
193 return exclaves_storage_upcall_legacy_getsize(fstag, fileid, completion);
194 },
195
196 .sealstate = ^(const enum xnuupcalls_fstag_s fstag,
197 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sealstate__result_s)) {
198 return exclaves_storage_upcall_legacy_sealstate(fstag, completion);
199 },
200
201 .irq_register = ^(const uint64_t id, const int32_t index,
202 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_register__result_s)) {
203 return exclaves_driverkit_upcall_legacy_irq_register(id, index, completion);
204 },
205
206 .irq_remove = ^(const uint64_t id, const int32_t index,
207 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_remove__result_s)) {
208 return exclaves_driverkit_upcall_legacy_irq_remove(id, index, completion);
209 },
210
211 .irq_enable = ^(const uint64_t id, const int32_t index,
212 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_enable__result_s)) {
213 return exclaves_driverkit_upcall_legacy_irq_enable(id, index, completion);
214 },
215
216 .irq_disable = ^(const uint64_t id, const int32_t index,
217 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_disable__result_s)) {
218 return exclaves_driverkit_upcall_legacy_irq_disable(id, index, completion);
219 },
220
221 .timer_register = ^(const uint64_t id,
222 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_register__result_s)) {
223 return exclaves_driverkit_upcall_legacy_timer_register(id, completion);
224 },
225
226 .timer_remove = ^(const uint64_t id, const uint32_t timer_id,
227 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_remove__result_s)) {
228 return exclaves_driverkit_upcall_legacy_timer_remove(id, timer_id, completion);
229 },
230
231 .timer_enable = ^(const uint64_t id, const uint32_t timer_id,
232 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_enable__result_s)) {
233 return exclaves_driverkit_upcall_legacy_timer_enable(id, timer_id, completion);
234 },
235
236 .timer_disable = ^(const uint64_t id, const uint32_t timer_id,
237 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_disable__result_s)) {
238 return exclaves_driverkit_upcall_legacy_timer_disable(id, timer_id, completion);
239 },
240
241 .timer_set_timeout = ^(const uint64_t id, const uint32_t timer_id,
242 const struct xnuupcalls_drivertimerspecification_s *duration,
243 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_set_timeout__result_s)) {
244 return exclaves_driverkit_upcall_legacy_timer_set_timeout(id, timer_id, duration, completion);
245 },
246
247 .timer_cancel_timeout = ^(const uint64_t id, const uint32_t timer_id,
248 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_cancel_timeout__result_s)) {
249 return exclaves_driverkit_upcall_legacy_timer_cancel_timeout(id, timer_id, completion);
250 },
251
252 .lock_wl = ^(const uint64_t id,
253 tb_error_t (^completion)(xnuupcalls_xnuupcalls_lock_wl__result_s)) {
254 return exclaves_driverkit_upcall_legacy_lock_wl(id, completion);
255 },
256
257 .unlock_wl = ^(const uint64_t id,
258 tb_error_t (^completion)(xnuupcalls_xnuupcalls_unlock_wl__result_s)) {
259 return exclaves_driverkit_upcall_legacy_unlock_wl(id, completion);
260 },
261
262 .async_notification_signal = ^(const uint64_t id,
263 const uint32_t notificationID,
264 tb_error_t (^completion)(xnuupcalls_xnuupcalls_async_notification_signal__result_s)) {
265 return exclaves_driverkit_upcall_legacy_async_notification_signal(id,
266 notificationID, completion);
267 },
268
269 .mapper_activate = ^(const uint64_t id, const uint32_t mapperIndex,
270 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_activate__result_s)) {
271 return exclaves_driverkit_upcall_legacy_mapper_activate(id,
272 mapperIndex, completion);
273 },
274
275 .mapper_deactivate = ^(const uint64_t id, const uint32_t mapperIndex,
276 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_deactivate__result_s)) {
277 return exclaves_driverkit_upcall_legacy_mapper_deactivate(id,
278 mapperIndex, completion);
279 },
280
281 .notification_signal = ^(const uint64_t id, const uint32_t mask,
282 tb_error_t (^completion)(xnuupcalls_xnuupcalls_notification_signal__result_s)) {
283 return exclaves_driverkit_upcall_legacy_notification_signal(id, mask,
284 completion);
285 },
286
287 .ane_setpowerstate = ^(const uint64_t id, const uint32_t desiredState,
288 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_setpowerstate__result_s)) {
289 return exclaves_driverkit_upcall_legacy_ane_setpowerstate(id, desiredState,
290 completion);
291 },
292
293 .ane_worksubmit = ^(const uint64_t id, const uint64_t requestID,
294 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
295 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_worksubmit__result_s)) {
296 return exclaves_driverkit_upcall_legacy_ane_worksubmit(id, requestID,
297 taskDescriptorCount, submitTimestamp, completion);
298 },
299
300 .ane_workbegin = ^(const uint64_t id, const uint64_t requestID,
301 const uint64_t beginTimestamp,
302 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workbegin__result_s)) {
303 return exclaves_driverkit_upcall_legacy_ane_workbegin(id, requestID,
304 beginTimestamp, completion);
305 },
306
307 .ane_workend = ^(const uint64_t id, const uint64_t requestID,
308 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workend__result_s)) {
309 return exclaves_driverkit_upcall_legacy_ane_workend(id, requestID, completion);
310 },
311
312 .conclave_suspend = ^(const uint32_t flags,
313 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_suspend__result_s)) {
314 return exclaves_conclave_upcall_legacy_suspend(flags, completion);
315 },
316
317 .conclave_stop = ^(const uint32_t flags,
318 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_stop__result_s)) {
319 return exclaves_conclave_upcall_legacy_stop(flags, completion);
320 },
321 .conclave_crash_info = ^(const xnuupcalls_conclavesharedbuffer_s *shared_buf,
322 const uint32_t length,
323 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_crash_info__result_s)) {
324 return exclaves_conclave_upcall_legacy_crash_info(shared_buf, length, completion);
325 },
326 /* END IGNORE CODESTYLE */
327 };
328
329 static const xnuupcallsv2_memoryupcallsprivate__server_s exclaves_tightbeam_memory_upcalls_v2 = {
330 /* BEGIN IGNORE CODESTYLE */
331 /* Uncrustify doesn't deal well with Blocks. */
332 .alloc = ^(const uint32_t npages, xnuupcallsv2_pagekind_s kind,
333 tb_error_t (^completion)(xnuupcallsv2_pagelist_s)) {
334 return exclaves_memory_upcall_alloc(npages, kind, completion);
335 },
336
337 .alloc_ext = ^(const uint32_t npages, xnuupcallsv2_pageallocflagsv2_s flags, tb_error_t (^completion)(xnuupcallsv2_pagelist_s)) {
338 return exclaves_memory_upcall_alloc_ext(npages, flags, completion);
339 },
340
341 .free = ^(const xnuupcallsv2_pagelist_s pages, const xnuupcallsv2_pagekind_s kind,
342 tb_error_t (^completion)(void)) {
343 return exclaves_memory_upcall_free(pages, kind, completion);
344 },
345
346 .free_ext = ^(const xnuupcallsv2_pagelist_s pages, xnuupcallsv2_pagefreeflagsv2_s flags, tb_error_t (^completion)(void)) {
347 return exclaves_memory_upcall_free_ext(pages, flags, completion);
348 },
349 /* END IGNORE CODESTYLE */
350 };
351
352 #pragma clang diagnostic push
353 #pragma clang diagnostic ignored "-Wunused-parameter"
354 static const xnuupcallsv2_xnuupcalls__server_s exclaves_tightbeam_upcalls_v2 = {
355 /* BEGIN IGNORE CODESTYLE */
356 /* Uncrustify doesn't deal well with Blocks. */
357 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
358 return exclaves_helloupcall(arg, completion);
359 },
360
361 /* Unset memoryupcalls handlers */
362
363 .root = ^(const uint8_t exclaveid[_Nonnull 32],
364 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_root__result_s)) {
365 return exclaves_storage_upcall_root(exclaveid, completion);
366 },
367
368 .rootex = ^(const uint32_t fstag, const uint8_t exclaveid[_Nonnull 32],
369 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_rootex__result_s)) {
370 return exclaves_storage_upcall_rootex(fstag, exclaveid, completion);
371 },
372
373 .open = ^(const uint32_t fstag, const uint64_t rootid,
374 const uint8_t name[_Nonnull 256],
375 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_open__result_s)) {
376 return exclaves_storage_upcall_open(fstag, rootid, name, completion);
377 },
378
379 .close = ^(const uint32_t fstag, const uint64_t fileid,
380 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_close__result_s)) {
381 return exclaves_storage_upcall_close(fstag, fileid, completion);
382 },
383
384 .create = ^(const uint32_t fstag, const uint64_t rootid,
385 const uint8_t name[_Nonnull 256],
386 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_create__result_s)) {
387 return exclaves_storage_upcall_create(fstag, rootid, name, completion);
388 },
389
390 .read = ^(const uint32_t fstag, const uint64_t fileid,
391 const struct xnuupcallsv2_iodesc_s *descriptor,
392 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_read__result_s)) {
393 return exclaves_storage_upcall_read(fstag, fileid, descriptor, completion);
394 },
395
396 .write = ^(const uint32_t fstag, const uint64_t fileid,
397 const struct xnuupcallsv2_iodesc_s *descriptor,
398 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_write__result_s)) {
399 return exclaves_storage_upcall_write(fstag, fileid, descriptor, completion);
400 },
401
402 .remove = ^(const uint32_t fstag, const uint64_t rootid,
403 const uint8_t name[_Nonnull 256],
404 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_remove__result_s)) {
405 return exclaves_storage_upcall_remove(fstag, rootid, name, completion);
406 },
407
408 .sync = ^(const uint32_t fstag,
409 const xnuupcallsv2_syncop_s * _Nonnull op,
410 const uint64_t fileid,
411 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sync__result_s)) {
412 return exclaves_storage_upcall_sync(fstag, op, fileid, completion);
413 },
414
415 .readdir = ^(const uint32_t fstag,
416 const uint64_t fileid, const uint64_t buf,
417 const uint32_t length,
418 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_readdir__result_s)) {
419 return exclaves_storage_upcall_readdir(fstag, fileid, buf, length, completion);
420 },
421
422 .getsize = ^(const uint32_t fstag, const uint64_t fileid,
423 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_getsize__result_s)) {
424 return exclaves_storage_upcall_getsize(fstag, fileid, completion);
425 },
426
427 .sealstate = ^(const uint32_t fstag,
428 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sealstate__result_s)) {
429 return exclaves_storage_upcall_sealstate(fstag, completion);
430 },
431
432 .queryvolumegroup = ^(const uint8_t vguuid[_Nonnull 37],
433 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_queryvolumegroup__result_s)) {
434 return exclaves_storage_upcall_queryvolumegroup(vguuid, completion);
435 },
436
437 .irqregister = ^(const uint64_t id, const int32_t index,
438 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqregister__result_s)) {
439 return exclaves_driverkit_upcall_irq_register(id, index, completion);
440 },
441
442 .irqremove = ^(const uint64_t id, const int32_t index,
443 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqremove__result_s)) {
444 return exclaves_driverkit_upcall_irq_remove(id, index, completion);
445 },
446
447 .irqenable = ^(const uint64_t id, const int32_t index,
448 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqenable__result_s)) {
449 return exclaves_driverkit_upcall_irq_enable(id, index, completion);
450 },
451
452 .irqdisable = ^(const uint64_t id, const int32_t index,
453 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqdisable__result_s)) {
454 return exclaves_driverkit_upcall_irq_disable(id, index, completion);
455 },
456
457 .timerregister = ^(const uint64_t id,
458 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerregister__result_s)) {
459 return exclaves_driverkit_upcall_timer_register(id, completion);
460 },
461
462 .timerremove = ^(const uint64_t id, const uint32_t timerid,
463 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerremove__result_s)) {
464 return exclaves_driverkit_upcall_timer_remove(id, timerid, completion);
465 },
466
467 .timerenable = ^(const uint64_t id, const uint32_t timerid,
468 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerenable__result_s)) {
469 return exclaves_driverkit_upcall_timer_enable(id, timerid, completion);
470 },
471
472 .timerdisable = ^(const uint64_t id, const uint32_t timerid,
473 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerdisable__result_s)) {
474 return exclaves_driverkit_upcall_timer_disable(id, timerid, completion);
475 },
476
477 .timersettimeout = ^(const uint64_t id, const uint32_t timerid,
478 const struct xnuupcallsv2_drivertimerspecification_s *duration,
479 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timersettimeout__result_s)) {
480 return exclaves_driverkit_upcall_timer_set_timeout(id, timerid, duration, completion);
481 },
482
483 .timercanceltimeout = ^(const uint64_t id, const uint32_t timerid,
484 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timercanceltimeout__result_s)) {
485 return exclaves_driverkit_upcall_timer_cancel_timeout(id, timerid, completion);
486 },
487
488 .lockworkloop = ^(const uint64_t id,
489 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_lockworkloop__result_s)) {
490 return exclaves_driverkit_upcall_lock_workloop(id, completion);
491 },
492
493 .unlockworkloop = ^(const uint64_t id,
494 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_unlockworkloop__result_s)) {
495 return exclaves_driverkit_upcall_unlock_workloop(id, completion);
496 },
497
498 .asyncnotificationsignal = ^(const uint64_t id,
499 const uint32_t notificationID,
500 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_asyncnotificationsignal__result_s)) {
501 return exclaves_driverkit_upcall_async_notification_signal(id,
502 notificationID, completion);
503 },
504
505 .mapperactivate = ^(const uint64_t id, const uint32_t mapperIndex,
506 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperactivate__result_s)) {
507 return exclaves_driverkit_upcall_mapper_activate(id,
508 mapperIndex, completion);
509 },
510
511 .mapperdeactivate = ^(const uint64_t id, const uint32_t mapperIndex,
512 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperdeactivate__result_s)) {
513 return exclaves_driverkit_upcall_mapper_deactivate(id,
514 mapperIndex, completion);
515 },
516
517 .anesetpowerstate = ^(const uint64_t id, const uint32_t desiredState,
518 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_anesetpowerstate__result_s)) {
519 return exclaves_driverkit_upcall_ane_setpowerstate(id, desiredState,
520 completion);
521 },
522
523 .aneworksubmit = ^(const uint64_t id, const uint64_t requestID,
524 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
525 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworksubmit__result_s)) {
526 return exclaves_driverkit_upcall_ane_worksubmit(id, requestID,
527 taskDescriptorCount, submitTimestamp, completion);
528 },
529
530 .aneworkbegin = ^(const uint64_t id, const uint64_t requestID,
531 const uint64_t beginTimestamp,
532 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkbegin__result_s)) {
533 return exclaves_driverkit_upcall_ane_workbegin(id, requestID,
534 beginTimestamp, completion);
535 },
536
537 .aneworkend = ^(const uint64_t id, const uint64_t requestID,
538 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkend__result_s)) {
539 return exclaves_driverkit_upcall_ane_workend(id, requestID, completion);
540 },
541
542 .notificationsignal = ^(const uint64_t id, const uint32_t mask,
543 tb_error_t (^completion)(xnuupcallsv2_notificationupcallsprivate_notificationsignal__result_s)) {
544 return exclaves_driverkit_upcall_notification_signal(id, mask,
545 completion);
546 },
547
548 .suspend = ^(const uint32_t flags,
549 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_suspend__result_s)) {
550 return exclaves_conclave_upcall_suspend(flags, completion);
551 },
552
553 .stop = ^(const uint32_t flags,
554 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_stop__result_s)) {
555 return exclaves_conclave_upcall_stop(flags, completion);
556 },
557 .crashinfo = ^(const xnuupcallsv2_conclavesharedbuffer_s *shared_buf,
558 const uint32_t length,
559 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_crashinfo__result_s)) {
560 return exclaves_conclave_upcall_crash_info(shared_buf, length, completion);
561 },
562 .createpowerassertion = ^(
563 tb_error_t (^completion)(xnuupcallsv2_lpwupcallsprivate_createpowerassertion__result_s)) {
564 return exclaves_driverkit_upcall_lpw_createpowerassertion(completion);
565 },
566 .releasepowerassertion = ^(const xnuupcallsv2_assertionid_s id,
567 tb_error_t (^completion)(xnuupcallsv2_lpwupcallsprivate_releasepowerassertion__result_s)) {
568 return exclaves_driverkit_upcall_lpw_releasepowerassertion(id, completion);
569 },
570 .requestrunmode = ^(const xnuupcallsv2_lpwrunmode_s mode,
571 tb_error_t (^completion)(xnuupcallsv2_lpwupcallsprivate_requestrunmode__result_s)) {
572 return exclaves_driverkit_upcall_lpw_requestrunmode(mode, completion);
573 },
574 .workavailable = ^(const xnuupcallsv2_aoeworkinfo_s * workInfo,
575 tb_error_t (^completion)(void)) {
576 return exclaves_aoe_upcall_work_available(workInfo, completion);
577 },
578 /* END IGNORE CODESTYLE */
579 };
580 #pragma clang diagnostic pop
581
582 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)583 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
584 exclaves_upcall_handler_t upcall_handler)
585 {
586 assert3u(upcall_id, <, NUM_XNUPROXY_UPCALLS);
587 assert(upcall_handler != NULL);
588
589 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
590 assert(exclaves_upcall_handlers[upcall_id].handler == NULL);
591
592 exclaves_upcall_handlers[upcall_id] =
593 (exclaves_upcall_handler_registration_t){
594 .handler = upcall_handler,
595 .context = upcall_context,
596 };
597
598 return KERN_SUCCESS;
599 }
600
601 kern_return_t
exclaves_upcall_init(void)602 exclaves_upcall_init(void)
603 {
604 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
605
606 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
607 #if DEVELOPMENT || DEBUG
608 kern_return_t kr;
609 kr = exclaves_register_upcall_handler(
610 XNUPROXY_UPCALL_HELLOUPCALL, NULL,
611 exclaves_test_hello_upcall_handler);
612 assert3u(kr, ==, KERN_SUCCESS);
613 #endif /* DEVELOPMENT || DEBUG */
614 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
615
616 tb_endpoint_t tb_upcall_ep = tb_endpoint_create_with_value(
617 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL,
618 TB_ENDPOINT_OPTIONS_NONE);
619
620 tb_endpoint_t tb_pmm_upcall_v2_ep = tb_endpoint_create_with_value(
621 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_PMM_UPCALL_V2,
622 TB_ENDPOINT_OPTIONS_NONE);
623
624 tb_endpoint_t tb_upcall_v2_ep = tb_endpoint_create_with_value(
625 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL_V2,
626 TB_ENDPOINT_OPTIONS_NONE);
627
628 #pragma clang diagnostic push
629 #pragma clang diagnostic ignored "-Wcast-qual" /* FIXME: rdar://103647654 */
630 tb_error_t error = xnuupcalls_xnuupcalls__server_start(tb_upcall_ep,
631 (xnuupcalls_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls);
632 tb_error_t error2 = xnuupcallsv2_memoryupcallsprivate__server_start(
633 tb_pmm_upcall_v2_ep,
634 (xnuupcallsv2_memoryupcallsprivate__server_s*)&exclaves_tightbeam_memory_upcalls_v2);
635 tb_error_t error3 = xnuupcallsv2_xnuupcalls__server_start(tb_upcall_v2_ep,
636 (xnuupcallsv2_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls_v2);
637 #pragma clang diagnostic pop
638
639 return (error == TB_ERROR_SUCCESS
640 && error2 == TB_ERROR_SUCCESS
641 && error3 == TB_ERROR_SUCCESS)
642 ? KERN_SUCCESS : KERN_FAILURE;
643 }
644
645 /* Unslid pointers defining the range of code which triggers upcall handlers */
646 uintptr_t exclaves_upcall_range_start;
647 uintptr_t exclaves_upcall_range_end;
648
649 __startup_func
650 static void
initialize_exclaves_upcall_range(void)651 initialize_exclaves_upcall_range(void)
652 {
653 exclaves_upcall_range_start = VM_KERNEL_UNSLIDE(&exclaves_upcall_start_label);
654 assert3u(exclaves_upcall_range_start, !=, 0);
655 exclaves_upcall_range_end = VM_KERNEL_UNSLIDE(&exclaves_upcall_end_label);
656 assert3u(exclaves_upcall_range_end, !=, 0);
657 }
658 STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, initialize_exclaves_upcall_range);
659
660 bool
exclaves_upcall_in_range(uintptr_t addr,bool slid)661 exclaves_upcall_in_range(uintptr_t addr, bool slid)
662 {
663 return slid ?
664 exclaves_in_range(addr, (uintptr_t)&exclaves_upcall_start_label, (uintptr_t)&exclaves_upcall_end_label) :
665 exclaves_in_range(addr, exclaves_upcall_range_start, exclaves_upcall_range_end);
666 }
667
668 OS_NOINLINE
669 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)670 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
671 {
672 kern_return_t kr = KERN_INVALID_CAPABILITY;
673
674 __assert_only thread_t thread = current_thread();
675 assert3u(thread->th_exclaves_state & TH_EXCLAVES_UPCALL, !=, 0);
676
677 exclaves_tag_t tag = Exclaves_L4_GetMessageTag();
678
679 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
680 exclaves_badge_t badge = XNUPROXY_CR_UPCALL_BADGE(ipcb);
681
682 exclaves_upcall_handler_registration_t upcall_handler = {};
683
684 if (upcall_id < NUM_XNUPROXY_UPCALLS) {
685 upcall_handler = exclaves_upcall_handlers[upcall_id];
686 }
687 if (upcall_handler.handler) {
688 __asm__ volatile ( "EXCLAVES_UPCALL_START:\n\t");
689 kr = upcall_handler.handler(upcall_handler.context, &tag, badge);
690 __asm__ volatile ("EXCLAVES_UPCALL_END:\n\t");
691 Exclaves_L4_SetMessageTag(tag);
692 }
693
694 return kr;
695 }
696
697 /* -------------------------------------------------------------------------- */
698 #pragma mark Testing
699
700
701 static tb_error_t
702 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t))
703 {
704 #if DEVELOPMENT || DEBUG
705 exclaves_debug_printf(show_test_output,
706 "%s: Hello Tightbeam Upcall!\n", __func__);
707 tb_error_t ret = completion(~arg);
708 task_stop_conclave_upcall();
709 STACKSHOT_TESTPOINT(TP_UPCALL);
710 /* Emit kdebug event for kperf sampling testing */
711 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
712 return ret;
713 #else
714 (void)arg;
715 (void)completion;
716
717 return TB_ERROR_SUCCESS;
718 #endif /* DEVELOPMENT || DEBUG */
719 }
720
721 #if DEVELOPMENT || DEBUG
722
723 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
724 static kern_return_t
exclaves_test_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)725 exclaves_test_upcall_handler(void *context, exclaves_tag_t *tag,
726 exclaves_badge_t badge)
727 {
728 #pragma unused(context, badge)
729 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
730 assert(ipcb != NULL);
731
732 Exclaves_L4_Word_t mrs = Exclaves_L4_MessageTag_Mrs(*tag);
733 assert(mrs < Exclaves_L4_IpcBuffer_Mrs);
734 Exclaves_L4_Word_t crs = Exclaves_L4_MessageTag_Crs(*tag);
735 assert(crs == 0);
736 Exclaves_L4_Word_t label = Exclaves_L4_MessageTag_Label(*tag);
737
738 /* setup test reply message */
739 *tag = Exclaves_L4_MessageTag(mrs, 0, ~label, Exclaves_L4_False);
740 for (int i = 0; i < mrs; i++) {
741 Exclaves_L4_SetMessageMr(i, ~Exclaves_L4_GetMessageMr(i));
742 }
743
744 return KERN_SUCCESS;
745 }
746 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
747
748 static int
exclaves_hello_upcall_test(__unused int64_t in,int64_t * out)749 exclaves_hello_upcall_test(__unused int64_t in, int64_t *out)
750 {
751 tb_error_t tb_result;
752 exclaveschelloserver_tests_s client;
753 const unsigned long request = 0xdecafbadfeedfaceul;
754
755 if (exclaves_get_status() != EXCLAVES_STATUS_AVAILABLE) {
756 exclaves_debug_printf(show_test_output,
757 "%s: SKIPPED: Exclaves not available\n", __func__);
758 *out = -1;
759 return 0;
760 }
761
762 exclaves_debug_printf(show_test_output, "%s: STARTING\n", __func__);
763
764 tb_endpoint_t ep = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU,
765 EXCLAVES_ID_HELLO_EXCLAVE_EP, TB_ENDPOINT_OPTIONS_NONE);
766
767 tb_result = exclaveschelloserver_tests__init(&client, ep);
768 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
769
770 tb_result = exclaveschelloserver_tests_default_upcall(&client, request, ^(exclaveschelloserver_result_s result) {
771 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
772 assert3u(result.result, ==, 1);
773 assert3u(result.reply, ==, ((request >> 32) | (request << 32)));
774 });
775
776 exclaves_debug_printf(show_test_output, "%s: SUCCESS\n", __func__);
777 *out = 1;
778
779 return KERN_SUCCESS;
780 }
781 SYSCTL_TEST_REGISTER(exclaves_hello_upcall_test, exclaves_hello_upcall_test);
782
783 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
784 static kern_return_t
exclaves_test_hello_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)785 exclaves_test_hello_upcall_handler(void *context, exclaves_tag_t *tag,
786 exclaves_badge_t badge)
787 {
788 /* HelloUpcall test handler */
789 assert(context == NULL);
790 exclaves_debug_printf(show_test_output, "%s: Hello Upcall!\n", __func__);
791 task_stop_conclave_upcall();
792 STACKSHOT_TESTPOINT(TP_UPCALL);
793 /* Emit kdebug event for kperf sampling testing */
794 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
795 return exclaves_test_upcall_handler(context, tag, badge);
796 }
797 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
798
799 #endif /* DEVELOPMENT || DEBUG */
800
801 #endif /* __has_include(<Tightbeam/tightbeam.h>) */
802
803 #else /* CONFIG_EXCLAVES */
804
805 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)806 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
807 {
808 (void)upcall_id;
809 return KERN_NOT_SUPPORTED;
810 }
811
812 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)813 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
814 exclaves_upcall_handler_t upcall_handler)
815 {
816 (void)upcall_id;
817 (void)upcall_context;
818 (void)upcall_handler;
819
820 return KERN_NOT_SUPPORTED;
821 }
822
823 #endif /* CONFIG_EXCLAVES */
824