1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30
31 #if CONFIG_EXCLAVES
32
33 #if CONFIG_SPTM
34 #include <arm64/sptm/sptm.h>
35 #else
36 #error Invalid configuration
37 #endif /* CONFIG_SPTM */
38
39 #include <libkern/section_keywords.h>
40 #include <kern/assert.h>
41 #include <kern/locks.h>
42 #include <kern/thread.h>
43 #include <stddef.h>
44
45 #include "exclaves_boot.h"
46 #include "exclaves_debug.h"
47 #include "exclaves_frame_mint.h"
48 #include "exclaves_upcalls.h"
49 #include "exclaves_internal.h"
50
51 /* Lock around single-threaded exclaves boot */
52 LCK_MTX_DECLARE(exclaves_boot_lock, &exclaves_lck_grp);
53
54 /* Boot status. */
55 __enum_closed_decl(exclaves_boot_status_t, uint32_t, {
56 EXCLAVES_BS_NOT_DEFINED = 0,
57 EXCLAVES_BS_NOT_SUPPORTED = 1,
58 EXCLAVES_BS_NOT_STARTED = 2,
59 EXCLAVES_BS_BOOTED_EXCLAVECORE = 3,
60 EXCLAVES_BS_BOOTED_EXCLAVEKIT = 4,
61 EXCLAVES_BS_BOOTED_FAILURE = 5,
62 });
63
64 /* Atomic so that it can be safely checked outside the boot lock. */
65 static os_atomic(exclaves_boot_status_t) exclaves_boot_status = EXCLAVES_BS_NOT_DEFINED;
66
67 static thread_t exclaves_boot_thread = THREAD_NULL;
68
69 extern exclaves_boot_task_entry_t exclaves_boot_task_entries[]
70 __SECTION_START_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
71
72 extern exclaves_boot_task_entry_t exclaves_boot_task_entries_end[]
73 __SECTION_END_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
74
75 static int
ebt_cmp(const void * e1,const void * e2)76 ebt_cmp(const void *e1, const void *e2)
77 {
78 const struct exclaves_boot_task_entry *a = e1;
79 const struct exclaves_boot_task_entry *b = e2;
80
81 if (a->ebt_rank > b->ebt_rank) {
82 return 1;
83 }
84
85 if (a->ebt_rank < b->ebt_rank) {
86 return -1;
87 }
88
89 return 0;
90 }
91
92 static void
exclaves_boot_tasks(void)93 exclaves_boot_tasks(void)
94 {
95 const size_t count =
96 exclaves_boot_task_entries_end - exclaves_boot_task_entries;
97 assert3u(count, >, 0);
98
99 __assert_only const size_t size =
100 ((uintptr_t)exclaves_boot_task_entries_end -
101 (uintptr_t)exclaves_boot_task_entries);
102 assert3u(size % sizeof(exclaves_boot_task_entry_t), ==, 0);
103
104 /*
105 * exclaves_boot_task_entries is in _DATA_CONST, make a stack copy so it
106 * can be sorted.
107 */
108 exclaves_boot_task_entry_t boot_tasks[count];
109 memcpy(boot_tasks, exclaves_boot_task_entries, count * sizeof(boot_tasks[0]));
110
111 extern void qsort(void *, size_t, size_t,
112 int (*)(const void *, const void *));
113 qsort(boot_tasks, count, sizeof(boot_tasks[0]), ebt_cmp);
114
115 for (size_t i = 0; i < count; i++) {
116 exclaves_debug_printf(show_progress,
117 "exclaves: boot task started, %s\n", boot_tasks[i].ebt_name);
118
119 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
120 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_START, i);
121 kern_return_t ret = boot_tasks[i].ebt_func();
122 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
123 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_END);
124
125 exclaves_debug_printf(show_progress,
126 "exclaves: boot task done, %s\n", boot_tasks[i].ebt_name);
127
128 if (ret != KERN_SUCCESS) {
129 panic("exclaves: boot task failed: %s (%p)",
130 boot_tasks[i].ebt_name, &boot_tasks[i]);
131 }
132 }
133 }
134
135 /*
136 * Check early in boot if the secure world has bootstrapped, and update the
137 * boot status if not.
138 */
139 __startup_func
140 static void
exclaves_check_sk(void)141 exclaves_check_sk(void)
142 {
143 const bool sk_bootstrapped = SPTMArgs->sk_bootstrapped;
144
145 if (sk_bootstrapped) {
146 os_atomic_store(&exclaves_boot_status,
147 EXCLAVES_BS_NOT_STARTED, relaxed);
148 } else {
149 os_atomic_store(&exclaves_boot_status,
150 EXCLAVES_BS_NOT_SUPPORTED, relaxed);
151 }
152 }
153 STARTUP(EXCLAVES, STARTUP_RANK_FIRST, exclaves_check_sk);
154
155 static void
exclaves_boot_status_wait(const exclaves_boot_status_t status)156 exclaves_boot_status_wait(const exclaves_boot_status_t status)
157 {
158 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
159
160 while (os_atomic_load(&exclaves_boot_status, relaxed) < status) {
161 lck_mtx_sleep(&exclaves_boot_lock, LCK_SLEEP_DEFAULT,
162 (event_t)(uintptr_t)&exclaves_boot_status, THREAD_UNINT);
163 }
164 }
165
166 static void
exclaves_boot_status_wake(void)167 exclaves_boot_status_wake(void)
168 {
169 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
170 thread_wakeup((event_t)(uintptr_t)&exclaves_boot_status);
171 }
172
173 static void
exclaves_boot_status_set(const exclaves_boot_status_t status)174 exclaves_boot_status_set(const exclaves_boot_status_t status)
175 {
176 assert3u(status, >, os_atomic_load(&exclaves_boot_status, relaxed));
177
178 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
179
180 /*
181 * 'release' here to ensure that the status update is only seen after
182 * any boot update. exclaves_boot_status is loaded with 'acquire' in
183 * exclaves_boot_wait() without holding the lock so the mutex alone
184 * isn't sufficient for ordering.
185 */
186 os_atomic_store(&exclaves_boot_status, status, release);
187 exclaves_boot_status_wake();
188 }
189
190 static bool
exclaves_boot_status_is_supported(exclaves_boot_status_t status)191 exclaves_boot_status_is_supported(exclaves_boot_status_t status)
192 {
193 assert3u(status, !=, EXCLAVES_BS_NOT_DEFINED);
194
195 return status != EXCLAVES_BS_NOT_DEFINED &&
196 status != EXCLAVES_BS_NOT_SUPPORTED;
197 }
198
199 static kern_return_t
exclaves_boot_exclavecore(void)200 exclaves_boot_exclavecore(void)
201 {
202 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
203
204 kern_return_t kr = KERN_FAILURE;
205 const exclaves_boot_status_t status =
206 os_atomic_load(&exclaves_boot_status, relaxed);
207
208 if (!exclaves_boot_status_is_supported(status)) {
209 return KERN_NOT_SUPPORTED;
210 }
211
212 /*
213 * This should only ever happen if there's a userspace bug which causes
214 * boot to be called twice and cause a race.
215 */
216 if (status != EXCLAVES_BS_NOT_STARTED) {
217 return KERN_INVALID_ARGUMENT;
218 }
219
220 /* Initialize xnu upcall server. */
221 kr = exclaves_upcall_init();
222 if (kr != KERN_SUCCESS) {
223 panic("Exclaves upcall early init failed");
224 }
225
226 /* Early boot. */
227 extern kern_return_t exclaves_boot_early(void);
228 kr = exclaves_boot_early();
229 if (kr != KERN_SUCCESS) {
230 /*
231 * If exclaves failed to boot, there's not much that can be done other
232 * than panic.
233 */
234 panic("exclaves early boot failed");
235 }
236
237 /*
238 * At this point it should be possible to make tightbeam calls/configure
239 * endpoints etc.
240 */
241 exclaves_boot_thread = current_thread();
242 exclaves_boot_tasks();
243 exclaves_boot_thread = THREAD_NULL;
244
245 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVECORE);
246
247 return kr;
248 }
249
250 static kern_return_t
exclaves_boot_exclavekit(void)251 exclaves_boot_exclavekit(void)
252 {
253 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
254
255 /* This should require an entitlement at some point. */
256
257 const exclaves_boot_status_t status =
258 os_atomic_load(&exclaves_boot_status, relaxed);
259
260 if (!exclaves_boot_status_is_supported(status)) {
261 return KERN_NOT_SUPPORTED;
262 }
263
264 switch (status) {
265 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
266 return KERN_SUCCESS;
267
268 case EXCLAVES_BS_BOOTED_FAILURE:
269 return KERN_FAILURE;
270
271 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
272 break;
273
274 default:
275 return KERN_INVALID_ARGUMENT;
276 }
277
278 /*
279 * Treat a failure to boot exclavekit as a transition to the
280 * EXCLAVES_BS_BOOTED_FAILURE state and return a failure. On RELEASE we
281 * simply panic as exclavekit is required.
282 */
283 kern_return_t kr = exclaves_frame_mint_populate();
284 if (kr != KERN_SUCCESS) {
285 exclaves_requirement_assert(EXCLAVES_R_EXCLAVEKIT,
286 "failed to populate frame mint");
287 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE);
288 return KERN_FAILURE;
289 }
290
291 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVEKIT);
292
293 return KERN_SUCCESS;
294 }
295
296 kern_return_t
exclaves_boot(exclaves_boot_stage_t boot_stage)297 exclaves_boot(exclaves_boot_stage_t boot_stage)
298 {
299 lck_mtx_lock(&exclaves_boot_lock);
300
301 kern_return_t kr = KERN_FAILURE;
302
303 switch (boot_stage) {
304 case EXCLAVES_BOOT_STAGE_EXCLAVECORE:
305 kr = exclaves_boot_exclavecore();
306 break;
307
308 case EXCLAVES_BOOT_STAGE_EXCLAVEKIT:
309 kr = exclaves_boot_exclavekit();
310 break;
311
312 default:
313 kr = KERN_INVALID_ARGUMENT;
314 break;
315 }
316
317 lck_mtx_unlock(&exclaves_boot_lock);
318
319 return kr;
320 }
321
322 /*
323 * This returns once the specified boot stage has been reached. If exclaves are
324 * unavailable, it returns immediately with KERN_NOT_SUPPORTED.
325 * Make it NOINLINE so it shows up in backtraces.
326 */
327 kern_return_t OS_NOINLINE
exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)328 exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)
329 {
330 assert(desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVECORE ||
331 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVEKIT);
332
333 /* Look up the equivalent status for the specified boot stage. */
334 const exclaves_boot_status_t desired_boot_status =
335 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVECORE ?
336 EXCLAVES_BS_BOOTED_EXCLAVECORE: EXCLAVES_BS_BOOTED_EXCLAVEKIT;
337
338 /*
339 * See comment in exclaves_boot_status_set() for why this is an acquire.
340 */
341 const exclaves_boot_status_t current_boot_status =
342 os_atomic_load(&exclaves_boot_status, acquire);
343
344 if (current_boot_status >= desired_boot_status) {
345 /*
346 * Special-case the situation where the request is to wait for
347 * EXCLAVES_BOOT_STAGE_EXCLAVEKIT. EXCLAVEKIT boot can fail
348 * (unlike EXCLAVECORE boot which will panic on failure). If
349 * EXCLAVEKIT has failed, just return KERN_NOT_SUPPORTED.
350 */
351 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
352 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
353 return KERN_NOT_SUPPORTED;
354 }
355
356 return KERN_SUCCESS;
357 }
358
359 if (!exclaves_boot_status_is_supported(current_boot_status)) {
360 return KERN_NOT_SUPPORTED;
361 }
362
363 /*
364 * Allow the exclaves boot thread to pass this check during exclavecore
365 * boot. This allows exclaves boot tasks to make TB calls etc during
366 * exclavecore boot.
367 */
368 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVECORE &&
369 current_thread() == exclaves_boot_thread) {
370 return KERN_SUCCESS;
371 }
372
373 /*
374 * Otherwise, wait until exclaves has booted to the requested stage or
375 * failed to boot.
376 */
377 lck_mtx_lock(&exclaves_boot_lock);
378 exclaves_boot_status_wait(desired_boot_status);
379 lck_mtx_unlock(&exclaves_boot_lock);
380
381 /*
382 * At this point there are two possibilities. Success or EXCLAVEKIT has
383 * failed (EXCLAVECORE can never fail as it panics on failure).
384 */
385 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
386 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
387 return KERN_NOT_SUPPORTED;
388 }
389
390 return KERN_SUCCESS;
391 }
392
393 /*
394 * This returns AVAILABLE once EXCLAVES_BOOT_STAGE_EXCLAVECORE has completed. This is to
395 * maintain backwards compatibility with existing code.
396 */
397 exclaves_status_t
exclaves_get_status(void)398 exclaves_get_status(void)
399 {
400 kern_return_t kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_EXCLAVECORE);
401 assert(kr == KERN_SUCCESS || kr == KERN_NOT_SUPPORTED);
402
403 if (kr == KERN_SUCCESS) {
404 return EXCLAVES_STATUS_AVAILABLE;
405 }
406
407 return EXCLAVES_STATUS_NOT_SUPPORTED;
408 }
409
410 const char *
exclaves_get_boot_status_string(void)411 exclaves_get_boot_status_string(void)
412 {
413 exclaves_boot_status_t boot_status = os_atomic_load(&exclaves_boot_status, relaxed);
414
415 switch (boot_status) {
416 case EXCLAVES_BS_NOT_DEFINED:
417 return "NOT_DEFINED";
418 case EXCLAVES_BS_NOT_SUPPORTED:
419 return "NOT_SUPPORTED";
420 case EXCLAVES_BS_NOT_STARTED:
421 return "NOT_STARTED";
422 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
423 return "BOOTED_EXCLAVECORE";
424 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
425 return "BOOTED_EXCLAVEKIT";
426 case EXCLAVES_BS_BOOTED_FAILURE:
427 return "BOOTED_FAILURE";
428 default:
429 assertf(false, "unknown boot status %u", boot_status);
430 return "UNKNOWN";
431 }
432 }
433
434 exclaves_boot_stage_t
exclaves_get_boot_stage(void)435 exclaves_get_boot_stage(void)
436 {
437 exclaves_boot_status_t status =
438 os_atomic_load(&exclaves_boot_status, relaxed);
439
440 switch (status) {
441 case EXCLAVES_BS_NOT_STARTED:
442 case EXCLAVES_BS_NOT_SUPPORTED:
443 case EXCLAVES_BS_NOT_DEFINED:
444 return EXCLAVES_BOOT_STAGE_NONE;
445
446 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
447 return EXCLAVES_BOOT_STAGE_EXCLAVECORE;
448
449 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
450 return EXCLAVES_BOOT_STAGE_EXCLAVEKIT;
451
452 case EXCLAVES_BS_BOOTED_FAILURE:
453 return EXCLAVES_BOOT_STAGE_FAILED;
454
455 default:
456 panic("unknown boot status: %u", status);
457 }
458 }
459
460 bool
exclaves_boot_supported(void)461 exclaves_boot_supported(void)
462 {
463 exclaves_boot_status_t status;
464
465 status = os_atomic_load(&exclaves_boot_status, relaxed);
466 return exclaves_boot_status_is_supported(status);
467 }
468
469 #else /* CONFIG_EXCLAVES */
470
471 exclaves_status_t
exclaves_get_status(void)472 exclaves_get_status(void)
473 {
474 return EXCLAVES_STATUS_NOT_SUPPORTED;
475 }
476
477 exclaves_boot_stage_t
exclaves_get_boot_stage(void)478 exclaves_get_boot_stage(void)
479 {
480 return EXCLAVES_BOOT_STAGE_NONE;
481 }
482
483 bool
exclaves_boot_supported(void)484 exclaves_boot_supported(void)
485 {
486 return false;
487 }
488
489 #endif /* CONFIG_EXCLAVES */
490