1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30
31 #if CONFIG_EXCLAVES
32
33 #if CONFIG_SPTM
34 #include <arm64/sptm/sptm.h>
35 #else
36 #error Invalid configuration
37 #endif /* CONFIG_SPTM */
38
39 #include <libkern/section_keywords.h>
40 #include <kern/assert.h>
41 #include <kern/locks.h>
42 #include <kern/thread.h>
43 #include <stddef.h>
44
45 #include "exclaves_boot.h"
46 #include "exclaves_debug.h"
47 #include "exclaves_frame_mint.h"
48 #include "exclaves_upcalls.h"
49 #include "exclaves_internal.h"
50
51 /* Lock around single-threaded exclaves boot */
52 LCK_MTX_DECLARE(exclaves_boot_lock, &exclaves_lck_grp);
53
54 /* Boot status. */
55 __enum_closed_decl(exclaves_boot_status_t, uint32_t, {
56 EXCLAVES_BS_NOT_DEFINED = 0,
57 EXCLAVES_BS_NOT_SUPPORTED = 1,
58 EXCLAVES_BS_NOT_STARTED = 2,
59 EXCLAVES_BS_BOOTED_EXCLAVECORE = 3,
60 EXCLAVES_BS_BOOTED_EXCLAVEKIT = 4,
61 EXCLAVES_BS_BOOTED_FAILURE = 5,
62 });
63
64 /* Atomic so that it can be safely checked outside the boot lock. */
65 static os_atomic(exclaves_boot_status_t) exclaves_boot_status = EXCLAVES_BS_NOT_DEFINED;
66
67 static thread_t exclaves_boot_thread = THREAD_NULL;
68
69 extern exclaves_boot_task_entry_t exclaves_boot_task_entries[]
70 __SECTION_START_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
71
72 extern exclaves_boot_task_entry_t exclaves_boot_task_entries_end[]
73 __SECTION_END_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
74
75 static int
ebt_cmp(const void * e1,const void * e2)76 ebt_cmp(const void *e1, const void *e2)
77 {
78 const struct exclaves_boot_task_entry *a = e1;
79 const struct exclaves_boot_task_entry *b = e2;
80
81 if (a->ebt_rank > b->ebt_rank) {
82 return 1;
83 }
84
85 if (a->ebt_rank < b->ebt_rank) {
86 return -1;
87 }
88
89 return 0;
90 }
91
92 static void
exclaves_boot_tasks(void)93 exclaves_boot_tasks(void)
94 {
95 const size_t count =
96 exclaves_boot_task_entries_end - exclaves_boot_task_entries;
97 assert3u(count, >, 0);
98
99 __assert_only const size_t size =
100 ((uintptr_t)exclaves_boot_task_entries_end -
101 (uintptr_t)exclaves_boot_task_entries);
102 assert3u(size % sizeof(exclaves_boot_task_entry_t), ==, 0);
103
104 /*
105 * exclaves_boot_task_entries is in _DATA_CONST, make a stack copy so it
106 * can be sorted.
107 */
108 exclaves_boot_task_entry_t boot_tasks[count];
109 memcpy(boot_tasks, exclaves_boot_task_entries, count * sizeof(boot_tasks[0]));
110
111 extern void qsort(void *, size_t, size_t,
112 int (*)(const void *, const void *));
113 qsort(boot_tasks, count, sizeof(boot_tasks[0]), ebt_cmp);
114
115 for (size_t i = 0; i < count; i++) {
116 exclaves_debug_printf(show_progress,
117 "exclaves: boot task started, %s\n", boot_tasks[i].ebt_name);
118
119 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
120 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_START, i);
121 kern_return_t ret = boot_tasks[i].ebt_func();
122 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
123 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_END);
124
125 exclaves_debug_printf(show_progress,
126 "exclaves: boot task done, %s\n", boot_tasks[i].ebt_name);
127
128 if (ret != KERN_SUCCESS) {
129 panic("exclaves: boot task failed: %s (%p)",
130 boot_tasks[i].ebt_name, &boot_tasks[i]);
131 }
132 }
133 }
134
135 /*
136 * Check early in boot if the secure world has bootstrapped, and update the
137 * boot status if not.
138 */
139 __startup_func
140 static void
exclaves_check_sk(void)141 exclaves_check_sk(void)
142 {
143 const bool sk_bootstrapped = SPTMArgs->sk_bootstrapped;
144
145 if (sk_bootstrapped) {
146 os_atomic_store(&exclaves_boot_status,
147 EXCLAVES_BS_NOT_STARTED, relaxed);
148 } else {
149 os_atomic_store(&exclaves_boot_status,
150 EXCLAVES_BS_NOT_SUPPORTED, relaxed);
151 }
152 }
153 STARTUP(EXCLAVES, STARTUP_RANK_FIRST, exclaves_check_sk);
154
155 static void
exclaves_boot_status_wait(const exclaves_boot_status_t status)156 exclaves_boot_status_wait(const exclaves_boot_status_t status)
157 {
158 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
159
160 while (os_atomic_load(&exclaves_boot_status, relaxed) < status) {
161 lck_mtx_sleep(&exclaves_boot_lock, LCK_SLEEP_DEFAULT,
162 (event_t)(uintptr_t)&exclaves_boot_status, THREAD_UNINT);
163 }
164 }
165
166 static void
exclaves_boot_status_wake(void)167 exclaves_boot_status_wake(void)
168 {
169 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
170 thread_wakeup((event_t)(uintptr_t)&exclaves_boot_status);
171 }
172
173 static void
exclaves_boot_status_set(const exclaves_boot_status_t status)174 exclaves_boot_status_set(const exclaves_boot_status_t status)
175 {
176 assert3u(status, >, os_atomic_load(&exclaves_boot_status, relaxed));
177
178 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
179
180 /*
181 * 'release' here to ensure that the status update is only seen after
182 * any boot update. exclaves_boot_status is loaded with 'acquire' in
183 * exclaves_boot_wait() without holding the lock so the mutex alone
184 * isn't sufficient for ordering.
185 */
186 os_atomic_store(&exclaves_boot_status, status, release);
187 exclaves_boot_status_wake();
188 }
189
190 static bool
exclaves_boot_status_is_supported(exclaves_boot_status_t status)191 exclaves_boot_status_is_supported(exclaves_boot_status_t status)
192 {
193 assert3u(status, !=, EXCLAVES_BS_NOT_DEFINED);
194
195 return status != EXCLAVES_BS_NOT_DEFINED &&
196 status != EXCLAVES_BS_NOT_SUPPORTED;
197 }
198
199 static kern_return_t
exclaves_boot_exclavecore(void)200 exclaves_boot_exclavecore(void)
201 {
202 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
203
204 kern_return_t kr = KERN_FAILURE;
205 const exclaves_boot_status_t status =
206 os_atomic_load(&exclaves_boot_status, relaxed);
207
208 if (!exclaves_boot_status_is_supported(status)) {
209 return KERN_NOT_SUPPORTED;
210 }
211
212 /*
213 * This should only ever happen if there's a userspace bug which causes
214 * boot to be called twice and cause a race.
215 */
216 if (status != EXCLAVES_BS_NOT_STARTED) {
217 return KERN_INVALID_ARGUMENT;
218 }
219
220 /* Initialize xnu upcall server. */
221 kr = exclaves_upcall_init();
222 if (kr != KERN_SUCCESS) {
223 panic("Exclaves upcall early init failed");
224 }
225
226 /* Early boot. */
227 extern kern_return_t exclaves_boot_early(void);
228 kr = exclaves_boot_early();
229 if (kr != KERN_SUCCESS) {
230 /*
231 * If exclaves failed to boot, there's not much that can be done other
232 * than panic.
233 */
234 panic("exclaves early boot failed");
235 }
236
237 /*
238 * At this point it should be possible to make tightbeam calls/configure
239 * endpoints etc.
240 */
241 exclaves_boot_thread = current_thread();
242 exclaves_boot_tasks();
243 exclaves_boot_thread = THREAD_NULL;
244
245 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVECORE);
246
247 return kr;
248 }
249
250 static kern_return_t
exclaves_boot_exclavekit(void)251 exclaves_boot_exclavekit(void)
252 {
253 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
254
255 /* This should require an entitlement at some point. */
256
257 const exclaves_boot_status_t status =
258 os_atomic_load(&exclaves_boot_status, relaxed);
259
260 if (!exclaves_boot_status_is_supported(status)) {
261 return KERN_NOT_SUPPORTED;
262 }
263
264 switch (status) {
265 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
266 return KERN_SUCCESS;
267
268 case EXCLAVES_BS_BOOTED_FAILURE:
269 return KERN_FAILURE;
270
271 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
272 break;
273
274 default:
275 return KERN_INVALID_ARGUMENT;
276 }
277
278 /*
279 * If framebank initialization fails, treat a failure to boot exclavekit
280 * as a transition to the EXCLAVES_BS_BOOTED_FAILURE state
281 * and return a failure. On RELEASE we simply panic as exclavekit is required.
282 */
283 kern_return_t kr = exclaves_frame_mint_populate();
284 if (kr != KERN_SUCCESS) {
285 exclaves_requirement_assert(EXCLAVES_R_FRAMEBANK,
286 "failed to populate frame mint");
287 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE);
288 return KERN_FAILURE;
289 }
290
291 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVEKIT);
292
293 return KERN_SUCCESS;
294 }
295
296 kern_return_t
exclaves_boot(exclaves_boot_stage_t boot_stage)297 exclaves_boot(exclaves_boot_stage_t boot_stage)
298 {
299 lck_mtx_lock(&exclaves_boot_lock);
300
301 kern_return_t kr = KERN_FAILURE;
302
303 switch (boot_stage) {
304 case EXCLAVES_BOOT_STAGE_EXCLAVECORE:
305 kr = exclaves_boot_exclavecore();
306 break;
307
308 case EXCLAVES_BOOT_STAGE_EXCLAVEKIT:
309 if (!exclaves_requirement_is_relaxed(EXCLAVES_R_EXCLAVEKIT)) {
310 kr = exclaves_boot_exclavekit();
311 } else {
312 /*
313 * If booting exclavekit was skipped due to a relaxed requirement,
314 * treat that as a transiton to the EXCLAVES_BS_BOOTED_FAILURE
315 * state and return a failure. On RELEASE we simply panic
316 * as exclavekit is required.
317 */
318 exclaves_requirement_assert(EXCLAVES_R_EXCLAVEKIT, "booting exclavekit skipped");
319 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE);
320 }
321 break;
322
323 default:
324 kr = KERN_INVALID_ARGUMENT;
325 break;
326 }
327
328 lck_mtx_unlock(&exclaves_boot_lock);
329
330 return kr;
331 }
332
333 /*
334 * This returns once the specified boot stage has been reached. If exclaves are
335 * unavailable, it returns immediately with KERN_NOT_SUPPORTED.
336 * Make it NOINLINE so it shows up in backtraces.
337 */
338 kern_return_t OS_NOINLINE
exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)339 exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)
340 {
341 assert(desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVECORE ||
342 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVEKIT);
343
344 /* Look up the equivalent status for the specified boot stage. */
345 const exclaves_boot_status_t desired_boot_status =
346 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVECORE ?
347 EXCLAVES_BS_BOOTED_EXCLAVECORE: EXCLAVES_BS_BOOTED_EXCLAVEKIT;
348
349 /*
350 * See comment in exclaves_boot_status_set() for why this is an acquire.
351 */
352 const exclaves_boot_status_t current_boot_status =
353 os_atomic_load(&exclaves_boot_status, acquire);
354
355 if (current_boot_status >= desired_boot_status) {
356 /*
357 * Special-case the situation where the request is to wait for
358 * EXCLAVES_BOOT_STAGE_EXCLAVEKIT. EXCLAVEKIT boot can fail
359 * (unlike EXCLAVECORE boot which will panic on failure). If
360 * EXCLAVEKIT has failed, just return KERN_NOT_SUPPORTED.
361 */
362 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
363 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
364 return KERN_NOT_SUPPORTED;
365 }
366
367 return KERN_SUCCESS;
368 }
369
370 if (!exclaves_boot_status_is_supported(current_boot_status)) {
371 return KERN_NOT_SUPPORTED;
372 }
373
374 /*
375 * Allow the exclaves boot thread to pass this check during exclavecore
376 * boot. This allows exclaves boot tasks to make TB calls etc during
377 * exclavecore boot.
378 */
379 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVECORE &&
380 current_thread() == exclaves_boot_thread) {
381 return KERN_SUCCESS;
382 }
383
384 /*
385 * Otherwise, wait until exclaves has booted to the requested stage or
386 * failed to boot.
387 */
388 lck_mtx_lock(&exclaves_boot_lock);
389 exclaves_boot_status_wait(desired_boot_status);
390 lck_mtx_unlock(&exclaves_boot_lock);
391
392 /*
393 * At this point there are two possibilities. Success or EXCLAVEKIT has
394 * failed (EXCLAVECORE can never fail as it panics on failure).
395 */
396 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
397 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
398 return KERN_NOT_SUPPORTED;
399 }
400
401 return KERN_SUCCESS;
402 }
403
404 /*
405 * This returns AVAILABLE once EXCLAVES_BOOT_STAGE_EXCLAVECORE has completed. This is to
406 * maintain backwards compatibility with existing code.
407 */
408 exclaves_status_t
exclaves_get_status(void)409 exclaves_get_status(void)
410 {
411 kern_return_t kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_EXCLAVECORE);
412 assert(kr == KERN_SUCCESS || kr == KERN_NOT_SUPPORTED);
413
414 if (kr == KERN_SUCCESS) {
415 return EXCLAVES_STATUS_AVAILABLE;
416 }
417
418 return EXCLAVES_STATUS_NOT_SUPPORTED;
419 }
420
421 const char *
exclaves_get_boot_status_string(void)422 exclaves_get_boot_status_string(void)
423 {
424 exclaves_boot_status_t boot_status = os_atomic_load(&exclaves_boot_status, relaxed);
425
426 switch (boot_status) {
427 case EXCLAVES_BS_NOT_DEFINED:
428 return "NOT_DEFINED";
429 case EXCLAVES_BS_NOT_SUPPORTED:
430 return "NOT_SUPPORTED";
431 case EXCLAVES_BS_NOT_STARTED:
432 return "NOT_STARTED";
433 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
434 return "BOOTED_EXCLAVECORE";
435 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
436 return "BOOTED_EXCLAVEKIT";
437 case EXCLAVES_BS_BOOTED_FAILURE:
438 return "BOOTED_FAILURE";
439 default:
440 assertf(false, "unknown boot status %u", boot_status);
441 return "UNKNOWN";
442 }
443 }
444
445 exclaves_boot_stage_t
exclaves_get_boot_stage(void)446 exclaves_get_boot_stage(void)
447 {
448 exclaves_boot_status_t status =
449 os_atomic_load(&exclaves_boot_status, relaxed);
450
451 switch (status) {
452 case EXCLAVES_BS_NOT_STARTED:
453 case EXCLAVES_BS_NOT_SUPPORTED:
454 case EXCLAVES_BS_NOT_DEFINED:
455 return EXCLAVES_BOOT_STAGE_NONE;
456
457 case EXCLAVES_BS_BOOTED_EXCLAVECORE:
458 return EXCLAVES_BOOT_STAGE_EXCLAVECORE;
459
460 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
461 return EXCLAVES_BOOT_STAGE_EXCLAVEKIT;
462
463 case EXCLAVES_BS_BOOTED_FAILURE:
464 return EXCLAVES_BOOT_STAGE_FAILED;
465
466 default:
467 panic("unknown boot status: %u", status);
468 }
469 }
470
471 bool
exclaves_boot_supported(void)472 exclaves_boot_supported(void)
473 {
474 exclaves_boot_status_t status;
475
476 status = os_atomic_load(&exclaves_boot_status, relaxed);
477 return exclaves_boot_status_is_supported(status);
478 }
479
480 #else /* CONFIG_EXCLAVES */
481
482 exclaves_status_t
exclaves_get_status(void)483 exclaves_get_status(void)
484 {
485 return EXCLAVES_STATUS_NOT_SUPPORTED;
486 }
487
488 exclaves_boot_stage_t
exclaves_get_boot_stage(void)489 exclaves_get_boot_stage(void)
490 {
491 return EXCLAVES_BOOT_STAGE_NONE;
492 }
493
494 bool
exclaves_boot_supported(void)495 exclaves_boot_supported(void)
496 {
497 return false;
498 }
499
500 #endif /* CONFIG_EXCLAVES */
501