1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30 #include "exclaves_upcalls.h"
31
32 #if CONFIG_EXCLAVES
33
34 #if __has_include(<Tightbeam/tightbeam.h>)
35
36 #include <mach/exclaves_l4.h>
37
38 #include <stdint.h>
39
40 #include <Tightbeam/tightbeam.h>
41 #include <Tightbeam/tightbeam_private.h>
42
43 #include <kern/kalloc.h>
44 #include <kern/locks.h>
45 #include <kern/task.h>
46
47 #include <xnuproxy/exclaves.h>
48 #include <xnuproxy/messages.h>
49
50 #include "kern/exclaves.tightbeam.h"
51
52 #include "exclaves_boot.h"
53 #include "exclaves_conclave.h"
54 #include "exclaves_debug.h"
55 #include "exclaves_driverkit.h"
56 #include "exclaves_memory.h"
57 #include "exclaves_stackshot.h"
58 #include "exclaves_storage.h"
59 #include "exclaves_test_stackshot.h"
60 #include "exclaves_xnuproxy.h"
61
62 #include <sys/errno.h>
63
64 #define EXCLAVES_ID_HELLO_EXCLAVE_EP \
65 (exclaves_service_lookup(EXCLAVES_DOMAIN_KERNEL, \
66 "com.apple.service.ExclavesCHelloServer"))
67
68 #define EXCLAVES_ID_TIGHTBEAM_UPCALL \
69 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM)
70
71 #define EXCLAVES_ID_TIGHTBEAM_UPCALL_V2 \
72 ((exclaves_id_t)XNUPROXY_UPCALL_TIGHTBEAM_V2)
73
74 extern lck_mtx_t exclaves_boot_lock;
75
76 typedef struct exclaves_upcall_handler_registration {
77 exclaves_upcall_handler_t handler;
78 void *context;
79 } exclaves_upcall_handler_registration_t;
80
81 static exclaves_upcall_handler_registration_t
82 exclaves_upcall_handlers[NUM_XNUPROXY_UPCALLS];
83
84 #define EXCLAVES_OBSOLETE_UPCALL_TESTING 1 // TODO: delete (rdar://123929546)
85 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
86 #if DEVELOPMENT || DEBUG
87 static kern_return_t
88 exclaves_test_hello_upcall_handler(void *, exclaves_tag_t *, exclaves_badge_t);
89 #endif /* DEVELOPMENT || DEBUG */
90 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
91
92
93 /* -------------------------------------------------------------------------- */
94 #pragma mark Upcall Callouts
95
96 static tb_error_t
97 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t));
98
99 /*
100 * Tightbeam upcall callout table.
101 * Don't add inline functionality here, instead call directly into your
102 * sub-system.
103 */
104
105 static const xnuupcalls_xnuupcalls__server_s exclaves_tightbeam_upcalls = {
106 /* BEGIN IGNORE CODESTYLE */
107 /* Uncrustify doesn't deal well with Blocks. */
108 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
109 return exclaves_helloupcall(arg, completion);
110 },
111
112 .alloc = ^(const uint32_t npages, xnuupcalls_pagekind_s kind,
113 tb_error_t (^completion)(xnuupcalls_pagelist_s)) {
114 return exclaves_memory_upcall_legacy_alloc(npages, kind, completion);
115 },
116
117 .free = ^(const uint32_t pages[_Nonnull EXCLAVES_MEMORY_MAX_REQUEST],
118 const uint32_t npages, const xnuupcalls_pagekind_s kind,
119 tb_error_t (^completion)(void)) {
120 return exclaves_memory_upcall_legacy_free(pages, npages, kind, completion);
121 },
122
123 .root = ^(const uint8_t exclaveid[_Nonnull 32],
124 tb_error_t (^completion)(xnuupcalls_xnuupcalls_root__result_s)) {
125 return exclaves_storage_upcall_legacy_root(exclaveid, completion);
126 },
127
128 .open = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
129 const uint8_t name[_Nonnull 256],
130 tb_error_t (^completion)(xnuupcalls_xnuupcalls_open__result_s)) {
131 return exclaves_storage_upcall_legacy_open(fstag, rootid, name, completion);
132 },
133
134 .close = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
135 tb_error_t (^completion)(xnuupcalls_xnuupcalls_close__result_s)) {
136 return exclaves_storage_upcall_legacy_close(fstag, fileid, completion);
137 },
138
139 .create = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
140 const uint8_t name[_Nonnull 256],
141 tb_error_t (^completion)(xnuupcalls_xnuupcalls_create__result_s)) {
142 return exclaves_storage_upcall_legacy_create(fstag, rootid, name, completion);
143 },
144
145 .read = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
146 const struct xnuupcalls_iodesc_s *descriptor,
147 tb_error_t (^completion)(xnuupcalls_xnuupcalls_read__result_s)) {
148 return exclaves_storage_upcall_legacy_read(fstag, fileid, descriptor, completion);
149 },
150
151 .write = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
152 const struct xnuupcalls_iodesc_s *descriptor,
153 tb_error_t (^completion)(xnuupcalls_xnuupcalls_write__result_s)) {
154 return exclaves_storage_upcall_legacy_write(fstag, fileid, descriptor, completion);
155 },
156
157 .remove = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t rootid,
158 const uint8_t name[_Nonnull 256],
159 tb_error_t (^completion)(xnuupcalls_xnuupcalls_remove__result_s)) {
160 return exclaves_storage_upcall_legacy_remove(fstag, rootid, name, completion);
161 },
162
163 .sync = ^(const enum xnuupcalls_fstag_s fstag,
164 const enum xnuupcalls_syncop_s op,
165 const uint64_t fileid,
166 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sync__result_s)) {
167 return exclaves_storage_upcall_legacy_sync(fstag, op, fileid, completion);
168 },
169
170 .readdir = ^(const enum xnuupcalls_fstag_s fstag,
171 const uint64_t fileid, const uint64_t buf,
172 const uint32_t length,
173 tb_error_t (^completion)(xnuupcalls_xnuupcalls_readdir__result_s)) {
174 return exclaves_storage_upcall_legacy_readdir(fstag, fileid, buf, length, completion);
175 },
176
177 .getsize = ^(const enum xnuupcalls_fstag_s fstag, const uint64_t fileid,
178 tb_error_t (^completion)(xnuupcalls_xnuupcalls_getsize__result_s)) {
179 return exclaves_storage_upcall_legacy_getsize(fstag, fileid, completion);
180 },
181
182 .sealstate = ^(const enum xnuupcalls_fstag_s fstag,
183 tb_error_t (^completion)(xnuupcalls_xnuupcalls_sealstate__result_s)) {
184 return exclaves_storage_upcall_legacy_sealstate(fstag, completion);
185 },
186
187 .irq_register = ^(const uint64_t id, const int32_t index,
188 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_register__result_s)) {
189 return exclaves_driverkit_upcall_legacy_irq_register(id, index, completion);
190 },
191
192 .irq_remove = ^(const uint64_t id, const int32_t index,
193 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_remove__result_s)) {
194 return exclaves_driverkit_upcall_legacy_irq_remove(id, index, completion);
195 },
196
197 .irq_enable = ^(const uint64_t id, const int32_t index,
198 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_enable__result_s)) {
199 return exclaves_driverkit_upcall_legacy_irq_enable(id, index, completion);
200 },
201
202 .irq_disable = ^(const uint64_t id, const int32_t index,
203 tb_error_t (^completion)(xnuupcalls_xnuupcalls_irq_disable__result_s)) {
204 return exclaves_driverkit_upcall_legacy_irq_disable(id, index, completion);
205 },
206
207 .timer_register = ^(const uint64_t id,
208 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_register__result_s)) {
209 return exclaves_driverkit_upcall_legacy_timer_register(id, completion);
210 },
211
212 .timer_remove = ^(const uint64_t id, const uint32_t timer_id,
213 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_remove__result_s)) {
214 return exclaves_driverkit_upcall_legacy_timer_remove(id, timer_id, completion);
215 },
216
217 .timer_enable = ^(const uint64_t id, const uint32_t timer_id,
218 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_enable__result_s)) {
219 return exclaves_driverkit_upcall_legacy_timer_enable(id, timer_id, completion);
220 },
221
222 .timer_disable = ^(const uint64_t id, const uint32_t timer_id,
223 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_disable__result_s)) {
224 return exclaves_driverkit_upcall_legacy_timer_disable(id, timer_id, completion);
225 },
226
227 .timer_set_timeout = ^(const uint64_t id, const uint32_t timer_id,
228 const struct xnuupcalls_drivertimerspecification_s *duration,
229 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_set_timeout__result_s)) {
230 return exclaves_driverkit_upcall_legacy_timer_set_timeout(id, timer_id, duration, completion);
231 },
232
233 .timer_cancel_timeout = ^(const uint64_t id, const uint32_t timer_id,
234 tb_error_t (^completion)(xnuupcalls_xnuupcalls_timer_cancel_timeout__result_s)) {
235 return exclaves_driverkit_upcall_legacy_timer_cancel_timeout(id, timer_id, completion);
236 },
237
238 .lock_wl = ^(const uint64_t id,
239 tb_error_t (^completion)(xnuupcalls_xnuupcalls_lock_wl__result_s)) {
240 return exclaves_driverkit_upcall_legacy_lock_wl(id, completion);
241 },
242
243 .unlock_wl = ^(const uint64_t id,
244 tb_error_t (^completion)(xnuupcalls_xnuupcalls_unlock_wl__result_s)) {
245 return exclaves_driverkit_upcall_legacy_unlock_wl(id, completion);
246 },
247
248 .async_notification_signal = ^(const uint64_t id,
249 const uint32_t notificationID,
250 tb_error_t (^completion)(xnuupcalls_xnuupcalls_async_notification_signal__result_s)) {
251 return exclaves_driverkit_upcall_legacy_async_notification_signal(id,
252 notificationID, completion);
253 },
254
255 .mapper_activate = ^(const uint64_t id, const uint32_t mapperIndex,
256 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_activate__result_s)) {
257 return exclaves_driverkit_upcall_legacy_mapper_activate(id,
258 mapperIndex, completion);
259 },
260
261 .mapper_deactivate = ^(const uint64_t id, const uint32_t mapperIndex,
262 tb_error_t (^completion)(xnuupcalls_xnuupcalls_mapper_deactivate__result_s)) {
263 return exclaves_driverkit_upcall_legacy_mapper_deactivate(id,
264 mapperIndex, completion);
265 },
266
267 .notification_signal = ^(const uint64_t id, const uint32_t mask,
268 tb_error_t (^completion)(xnuupcalls_xnuupcalls_notification_signal__result_s)) {
269 return exclaves_driverkit_upcall_legacy_notification_signal(id, mask,
270 completion);
271 },
272
273 .ane_setpowerstate = ^(const uint64_t id, const uint32_t desiredState,
274 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_setpowerstate__result_s)) {
275 return exclaves_driverkit_upcall_legacy_ane_setpowerstate(id, desiredState,
276 completion);
277 },
278
279 .ane_worksubmit = ^(const uint64_t id, const uint64_t requestID,
280 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
281 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_worksubmit__result_s)) {
282 return exclaves_driverkit_upcall_legacy_ane_worksubmit(id, requestID,
283 taskDescriptorCount, submitTimestamp, completion);
284 },
285
286 .ane_workbegin = ^(const uint64_t id, const uint64_t requestID,
287 const uint64_t beginTimestamp,
288 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workbegin__result_s)) {
289 return exclaves_driverkit_upcall_legacy_ane_workbegin(id, requestID,
290 beginTimestamp, completion);
291 },
292
293 .ane_workend = ^(const uint64_t id, const uint64_t requestID,
294 tb_error_t (^completion)(xnuupcalls_xnuupcalls_ane_workend__result_s)) {
295 return exclaves_driverkit_upcall_legacy_ane_workend(id, requestID, completion);
296 },
297
298 .conclave_suspend = ^(const uint32_t flags,
299 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_suspend__result_s)) {
300 return exclaves_conclave_upcall_legacy_suspend(flags, completion);
301 },
302
303 .conclave_stop = ^(const uint32_t flags,
304 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_stop__result_s)) {
305 return exclaves_conclave_upcall_legacy_stop(flags, completion);
306 },
307 .conclave_crash_info = ^(const xnuupcalls_conclavesharedbuffer_s *shared_buf,
308 const uint32_t length,
309 tb_error_t (^completion)(xnuupcalls_xnuupcalls_conclave_crash_info__result_s)) {
310 return exclaves_conclave_upcall_legacy_crash_info(shared_buf, length, completion);
311 },
312 /* END IGNORE CODESTYLE */
313 };
314
315 #pragma clang diagnostic push
316 #pragma clang diagnostic ignored "-Wunused-parameter"
317 static const xnuupcallsv2_xnuupcalls__server_s exclaves_tightbeam_upcalls_v2 = {
318 /* BEGIN IGNORE CODESTYLE */
319 /* Uncrustify doesn't deal well with Blocks. */
320 .helloupcall = ^(const uint64_t arg, tb_error_t (^completion)(uint64_t)) {
321 return exclaves_helloupcall(arg, completion);
322 },
323
324 .alloc = ^(const uint32_t npages, xnuupcallsv2_pagekind_s kind,
325 tb_error_t (^completion)(xnuupcallsv2_pagelist_s)) {
326 return exclaves_memory_upcall_alloc(npages, kind, completion);
327 },
328
329 .free = ^(const xnuupcallsv2_pagelist_s pages, const xnuupcallsv2_pagekind_s kind,
330 tb_error_t (^completion)(void)) {
331 return exclaves_memory_upcall_free(pages, kind, completion);
332 },
333
334 .root = ^(const uint8_t exclaveid[_Nonnull 32],
335 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_root__result_s)) {
336 return exclaves_storage_upcall_root(exclaveid, completion);
337 },
338
339 .open = ^(const uint32_t fstag, const uint64_t rootid,
340 const uint8_t name[_Nonnull 256],
341 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_open__result_s)) {
342 return exclaves_storage_upcall_open(fstag, rootid, name, completion);
343 },
344
345 .close = ^(const uint32_t fstag, const uint64_t fileid,
346 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_close__result_s)) {
347 return exclaves_storage_upcall_close(fstag, fileid, completion);
348 },
349
350 .create = ^(const uint32_t fstag, const uint64_t rootid,
351 const uint8_t name[_Nonnull 256],
352 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_create__result_s)) {
353 return exclaves_storage_upcall_create(fstag, rootid, name, completion);
354 },
355
356 .read = ^(const uint32_t fstag, const uint64_t fileid,
357 const struct xnuupcallsv2_iodesc_s *descriptor,
358 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_read__result_s)) {
359 return exclaves_storage_upcall_read(fstag, fileid, descriptor, completion);
360 },
361
362 .write = ^(const uint32_t fstag, const uint64_t fileid,
363 const struct xnuupcallsv2_iodesc_s *descriptor,
364 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_write__result_s)) {
365 return exclaves_storage_upcall_write(fstag, fileid, descriptor, completion);
366 },
367
368 .remove = ^(const uint32_t fstag, const uint64_t rootid,
369 const uint8_t name[_Nonnull 256],
370 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_remove__result_s)) {
371 return exclaves_storage_upcall_remove(fstag, rootid, name, completion);
372 },
373
374 .sync = ^(const uint32_t fstag,
375 const xnuupcallsv2_syncop_s * _Nonnull op,
376 const uint64_t fileid,
377 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sync__result_s)) {
378 return exclaves_storage_upcall_sync(fstag, op, fileid, completion);
379 },
380
381 .readdir = ^(const uint32_t fstag,
382 const uint64_t fileid, const uint64_t buf,
383 const uint32_t length,
384 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_readdir__result_s)) {
385 return exclaves_storage_upcall_readdir(fstag, fileid, buf, length, completion);
386 },
387
388 .getsize = ^(const uint32_t fstag, const uint64_t fileid,
389 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_getsize__result_s)) {
390 return exclaves_storage_upcall_getsize(fstag, fileid, completion);
391 },
392
393 .sealstate = ^(const uint32_t fstag,
394 tb_error_t (^completion)(xnuupcallsv2_storageupcallsprivate_sealstate__result_s)) {
395 return exclaves_storage_upcall_sealstate(fstag, completion);
396 },
397
398 .irqregister = ^(const uint64_t id, const int32_t index,
399 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqregister__result_s)) {
400 return exclaves_driverkit_upcall_irq_register(id, index, completion);
401 },
402
403 .irqremove = ^(const uint64_t id, const int32_t index,
404 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqremove__result_s)) {
405 return exclaves_driverkit_upcall_irq_remove(id, index, completion);
406 },
407
408 .irqenable = ^(const uint64_t id, const int32_t index,
409 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqenable__result_s)) {
410 return exclaves_driverkit_upcall_irq_enable(id, index, completion);
411 },
412
413 .irqdisable = ^(const uint64_t id, const int32_t index,
414 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_irqdisable__result_s)) {
415 return exclaves_driverkit_upcall_irq_disable(id, index, completion);
416 },
417
418 .timerregister = ^(const uint64_t id,
419 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerregister__result_s)) {
420 return exclaves_driverkit_upcall_timer_register(id, completion);
421 },
422
423 .timerremove = ^(const uint64_t id, const uint32_t timerid,
424 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerremove__result_s)) {
425 return exclaves_driverkit_upcall_timer_remove(id, timerid, completion);
426 },
427
428 .timerenable = ^(const uint64_t id, const uint32_t timerid,
429 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerenable__result_s)) {
430 return exclaves_driverkit_upcall_timer_enable(id, timerid, completion);
431 },
432
433 .timerdisable = ^(const uint64_t id, const uint32_t timerid,
434 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timerdisable__result_s)) {
435 return exclaves_driverkit_upcall_timer_disable(id, timerid, completion);
436 },
437
438 .timersettimeout = ^(const uint64_t id, const uint32_t timerid,
439 const struct xnuupcallsv2_drivertimerspecification_s *duration,
440 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timersettimeout__result_s)) {
441 return exclaves_driverkit_upcall_timer_set_timeout(id, timerid, duration, completion);
442 },
443
444 .timercanceltimeout = ^(const uint64_t id, const uint32_t timerid,
445 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_timercanceltimeout__result_s)) {
446 return exclaves_driverkit_upcall_timer_cancel_timeout(id, timerid, completion);
447 },
448
449 .lockworkloop = ^(const uint64_t id,
450 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_lockworkloop__result_s)) {
451 return exclaves_driverkit_upcall_lock_workloop(id, completion);
452 },
453
454 .unlockworkloop = ^(const uint64_t id,
455 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_unlockworkloop__result_s)) {
456 return exclaves_driverkit_upcall_unlock_workloop(id, completion);
457 },
458
459 .asyncnotificationsignal = ^(const uint64_t id,
460 const uint32_t notificationID,
461 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_asyncnotificationsignal__result_s)) {
462 return exclaves_driverkit_upcall_async_notification_signal(id,
463 notificationID, completion);
464 },
465
466 .mapperactivate = ^(const uint64_t id, const uint32_t mapperIndex,
467 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperactivate__result_s)) {
468 return exclaves_driverkit_upcall_mapper_activate(id,
469 mapperIndex, completion);
470 },
471
472 .mapperdeactivate = ^(const uint64_t id, const uint32_t mapperIndex,
473 tb_error_t (^completion)(xnuupcallsv2_driverupcallsprivate_mapperdeactivate__result_s)) {
474 return exclaves_driverkit_upcall_mapper_deactivate(id,
475 mapperIndex, completion);
476 },
477
478 .anesetpowerstate = ^(const uint64_t id, const uint32_t desiredState,
479 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_anesetpowerstate__result_s)) {
480 return exclaves_driverkit_upcall_ane_setpowerstate(id, desiredState,
481 completion);
482 },
483
484 .aneworksubmit = ^(const uint64_t id, const uint64_t requestID,
485 const uint32_t taskDescriptorCount, const uint64_t submitTimestamp,
486 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworksubmit__result_s)) {
487 return exclaves_driverkit_upcall_ane_worksubmit(id, requestID,
488 taskDescriptorCount, submitTimestamp, completion);
489 },
490
491 .aneworkbegin = ^(const uint64_t id, const uint64_t requestID,
492 const uint64_t beginTimestamp,
493 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkbegin__result_s)) {
494 return exclaves_driverkit_upcall_ane_workbegin(id, requestID,
495 beginTimestamp, completion);
496 },
497
498 .aneworkend = ^(const uint64_t id, const uint64_t requestID,
499 tb_error_t (^completion)(xnuupcallsv2_aneupcallsprivate_aneworkend__result_s)) {
500 return exclaves_driverkit_upcall_ane_workend(id, requestID, completion);
501 },
502
503 .notificationsignal = ^(const uint64_t id, const uint32_t mask,
504 tb_error_t (^completion)(xnuupcallsv2_notificationupcallsprivate_notificationsignal__result_s)) {
505 return exclaves_driverkit_upcall_notification_signal(id, mask,
506 completion);
507 },
508
509 .suspend = ^(const uint32_t flags,
510 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_suspend__result_s)) {
511 return exclaves_conclave_upcall_suspend(flags, completion);
512 },
513
514 .stop = ^(const uint32_t flags,
515 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_stop__result_s)) {
516 return exclaves_conclave_upcall_stop(flags, completion);
517 },
518 .crashinfo = ^(const xnuupcallsv2_conclavesharedbuffer_s *shared_buf,
519 const uint32_t length,
520 tb_error_t (^completion)(xnuupcallsv2_conclaveupcallsprivate_crashinfo__result_s)) {
521 return exclaves_conclave_upcall_crash_info(shared_buf, length, completion);
522 },
523 /* END IGNORE CODESTYLE */
524 };
525 #pragma clang diagnostic pop
526
527 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)528 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
529 exclaves_upcall_handler_t upcall_handler)
530 {
531 assert3u(upcall_id, <, NUM_XNUPROXY_UPCALLS);
532 assert(upcall_handler != NULL);
533
534 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
535 assert(exclaves_upcall_handlers[upcall_id].handler == NULL);
536
537 exclaves_upcall_handlers[upcall_id] =
538 (exclaves_upcall_handler_registration_t){
539 .handler = upcall_handler,
540 .context = upcall_context,
541 };
542
543 return KERN_SUCCESS;
544 }
545
546 kern_return_t
exclaves_upcall_init(void)547 exclaves_upcall_init(void)
548 {
549 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
550
551 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
552 #if DEVELOPMENT || DEBUG
553 kern_return_t kr;
554 kr = exclaves_register_upcall_handler(
555 XNUPROXY_UPCALL_HELLOUPCALL, NULL,
556 exclaves_test_hello_upcall_handler);
557 assert3u(kr, ==, KERN_SUCCESS);
558 #endif /* DEVELOPMENT || DEBUG */
559 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
560
561 tb_endpoint_t tb_upcall_ep = tb_endpoint_create_with_value(
562 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL,
563 TB_ENDPOINT_OPTIONS_NONE);
564
565 tb_endpoint_t tb_upcall_v2_ep = tb_endpoint_create_with_value(
566 TB_TRANSPORT_TYPE_XNU, EXCLAVES_ID_TIGHTBEAM_UPCALL_V2,
567 TB_ENDPOINT_OPTIONS_NONE);
568
569 #pragma clang diagnostic push
570 #pragma clang diagnostic ignored "-Wcast-qual" /* FIXME: rdar://103647654 */
571 tb_error_t error = xnuupcalls_xnuupcalls__server_start(tb_upcall_ep,
572 (xnuupcalls_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls);
573 tb_error_t error2 = xnuupcallsv2_xnuupcalls__server_start(tb_upcall_v2_ep,
574 (xnuupcallsv2_xnuupcalls__server_s *)&exclaves_tightbeam_upcalls_v2);
575 #pragma clang diagnostic pop
576
577 return (error == TB_ERROR_SUCCESS && error2 == TB_ERROR_SUCCESS) ? KERN_SUCCESS : KERN_FAILURE;
578 }
579
580 /* Unslid pointers defining the range of code which triggers upcall handlers */
581 uintptr_t exclaves_upcall_range_start;
582 uintptr_t exclaves_upcall_range_end;
583
584 __startup_func
585 static void
initialize_exclaves_upcall_range(void)586 initialize_exclaves_upcall_range(void)
587 {
588 exclaves_upcall_range_start = VM_KERNEL_UNSLIDE(&exclaves_upcall_start_label);
589 assert3u(exclaves_upcall_range_start, !=, 0);
590 exclaves_upcall_range_end = VM_KERNEL_UNSLIDE(&exclaves_upcall_end_label);
591 assert3u(exclaves_upcall_range_end, !=, 0);
592 }
593 STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, initialize_exclaves_upcall_range);
594
595 bool
exclaves_upcall_in_range(uintptr_t addr,bool slid)596 exclaves_upcall_in_range(uintptr_t addr, bool slid)
597 {
598 return slid ?
599 exclaves_in_range(addr, (uintptr_t)&exclaves_upcall_start_label, (uintptr_t)&exclaves_upcall_end_label) :
600 exclaves_in_range(addr, exclaves_upcall_range_start, exclaves_upcall_range_end);
601 }
602
603 OS_NOINLINE
604 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)605 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
606 {
607 kern_return_t kr = KERN_INVALID_CAPABILITY;
608
609 __assert_only thread_t thread = current_thread();
610 assert3u(thread->th_exclaves_state & TH_EXCLAVES_UPCALL, !=, 0);
611
612 exclaves_tag_t tag = Exclaves_L4_GetMessageTag();
613
614 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
615 exclaves_badge_t badge = XNUPROXY_CR_UPCALL_BADGE(ipcb);
616
617 exclaves_upcall_handler_registration_t upcall_handler = {};
618
619 if (upcall_id < NUM_XNUPROXY_UPCALLS) {
620 upcall_handler = exclaves_upcall_handlers[upcall_id];
621 }
622 if (upcall_handler.handler) {
623 __asm__ volatile ( "EXCLAVES_UPCALL_START:\n\t");
624 kr = upcall_handler.handler(upcall_handler.context, &tag, badge);
625 __asm__ volatile ("EXCLAVES_UPCALL_END:\n\t");
626 Exclaves_L4_SetMessageTag(tag);
627 }
628
629 return kr;
630 }
631
632 /* -------------------------------------------------------------------------- */
633 #pragma mark Testing
634
635
636 static tb_error_t
637 exclaves_helloupcall(const uint64_t arg, tb_error_t (^completion)(uint64_t))
638 {
639 #if DEVELOPMENT || DEBUG
640 exclaves_debug_printf(show_test_output,
641 "%s: Hello Tightbeam Upcall!\n", __func__);
642 tb_error_t ret = completion(~arg);
643 task_stop_conclave_upcall();
644 STACKSHOT_TESTPOINT(TP_UPCALL);
645 /* Emit kdebug event for kperf sampling testing */
646 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
647 return ret;
648 #else
649 (void)arg;
650 (void)completion;
651
652 return TB_ERROR_SUCCESS;
653 #endif /* DEVELOPMENT || DEBUG */
654 }
655
656 #if DEVELOPMENT || DEBUG
657
658 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
659 static kern_return_t
exclaves_test_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)660 exclaves_test_upcall_handler(void *context, exclaves_tag_t *tag,
661 exclaves_badge_t badge)
662 {
663 #pragma unused(context, badge)
664 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
665 assert(ipcb != NULL);
666
667 Exclaves_L4_Word_t mrs = Exclaves_L4_MessageTag_Mrs(*tag);
668 assert(mrs < Exclaves_L4_IpcBuffer_Mrs);
669 Exclaves_L4_Word_t crs = Exclaves_L4_MessageTag_Crs(*tag);
670 assert(crs == 0);
671 Exclaves_L4_Word_t label = Exclaves_L4_MessageTag_Label(*tag);
672
673 /* setup test reply message */
674 *tag = Exclaves_L4_MessageTag(mrs, 0, ~label, Exclaves_L4_False);
675 for (int i = 0; i < mrs; i++) {
676 Exclaves_L4_SetMessageMr(i, ~Exclaves_L4_GetMessageMr(i));
677 }
678
679 return KERN_SUCCESS;
680 }
681 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
682
683 static int
exclaves_hello_upcall_test(__unused int64_t in,int64_t * out)684 exclaves_hello_upcall_test(__unused int64_t in, int64_t *out)
685 {
686 tb_error_t tb_result;
687 exclaveschelloserver_tests_s client;
688 const unsigned long request = 0xdecafbadfeedfaceul;
689
690 if (exclaves_get_status() != EXCLAVES_STATUS_AVAILABLE) {
691 exclaves_debug_printf(show_test_output,
692 "%s: SKIPPED: Exclaves not available\n", __func__);
693 *out = -1;
694 return 0;
695 }
696
697 exclaves_debug_printf(show_test_output, "%s: STARTING\n", __func__);
698
699 tb_endpoint_t ep = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU,
700 EXCLAVES_ID_HELLO_EXCLAVE_EP, TB_ENDPOINT_OPTIONS_NONE);
701
702 tb_result = exclaveschelloserver_tests__init(&client, ep);
703 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
704
705 tb_result = exclaveschelloserver_tests_default_upcall(&client, request, ^(exclaveschelloserver_result_s result) {
706 assert3u(tb_result, ==, TB_ERROR_SUCCESS);
707 assert3u(result.result, ==, 1);
708 assert3u(result.reply, ==, ((request >> 32) | (request << 32)));
709 });
710
711 exclaves_debug_printf(show_test_output, "%s: SUCCESS\n", __func__);
712 *out = 1;
713
714 return KERN_SUCCESS;
715 }
716 SYSCTL_TEST_REGISTER(exclaves_hello_upcall_test, exclaves_hello_upcall_test);
717
718 #ifdef EXCLAVES_OBSOLETE_UPCALL_TESTING
719 static kern_return_t
exclaves_test_hello_upcall_handler(void * context,exclaves_tag_t * tag,exclaves_badge_t badge)720 exclaves_test_hello_upcall_handler(void *context, exclaves_tag_t *tag,
721 exclaves_badge_t badge)
722 {
723 /* HelloUpcall test handler */
724 assert(context == NULL);
725 exclaves_debug_printf(show_test_output, "%s: Hello Upcall!\n", __func__);
726 task_stop_conclave_upcall();
727 STACKSHOT_TESTPOINT(TP_UPCALL);
728 /* Emit kdebug event for kperf sampling testing */
729 KDBG(BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, 0xaa));
730 return exclaves_test_upcall_handler(context, tag, badge);
731 }
732 #endif // EXCLAVES_OBSOLETE_UPCALL_TESTING
733
734 #endif /* DEVELOPMENT || DEBUG */
735
736 #endif /* __has_include(<Tightbeam/tightbeam.h>) */
737
738 #else /* CONFIG_EXCLAVES */
739
740 kern_return_t
exclaves_call_upcall_handler(exclaves_id_t upcall_id)741 exclaves_call_upcall_handler(exclaves_id_t upcall_id)
742 {
743 (void)upcall_id;
744 return KERN_NOT_SUPPORTED;
745 }
746
747 kern_return_t
exclaves_register_upcall_handler(exclaves_id_t upcall_id,void * upcall_context,exclaves_upcall_handler_t upcall_handler)748 exclaves_register_upcall_handler(exclaves_id_t upcall_id, void *upcall_context,
749 exclaves_upcall_handler_t upcall_handler)
750 {
751 (void)upcall_id;
752 (void)upcall_context;
753 (void)upcall_handler;
754
755 return KERN_NOT_SUPPORTED;
756 }
757
758 #endif /* CONFIG_EXCLAVES */
759