1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/exclaves.h>
30
31 #if CONFIG_EXCLAVES
32
33 #if CONFIG_SPTM
34 #include <arm64/sptm/sptm.h>
35 #else
36 #error Invalid configuration
37 #endif /* CONFIG_SPTM */
38
39 #include <libkern/section_keywords.h>
40 #include <kern/assert.h>
41 #include <kern/locks.h>
42 #include <kern/thread.h>
43 #include <stddef.h>
44
45 #include "exclaves_boot.h"
46 #include "exclaves_debug.h"
47 #include "exclaves_frame_mint.h"
48 #include "exclaves_upcalls.h"
49 #include "exclaves_internal.h"
50
51 /* Lock around single-threaded exclaves boot */
52 LCK_MTX_DECLARE(exclaves_boot_lock, &exclaves_lck_grp);
53
54 /* Boot status. */
55 __enum_closed_decl(exclaves_boot_status_t, uint32_t, {
56 EXCLAVES_BS_NOT_DEFINED = 0,
57 EXCLAVES_BS_NOT_SUPPORTED = 1,
58 EXCLAVES_BS_NOT_STARTED = 2,
59 EXCLAVES_BS_BOOTED_STAGE_2 = 3,
60 EXCLAVES_BS_BOOTED_EXCLAVEKIT = 4,
61 EXCLAVES_BS_BOOTED_FAILURE = 5,
62 });
63
64 /* Atomic so that it can be safely checked outside the boot lock. */
65 static os_atomic(exclaves_boot_status_t) exclaves_boot_status = EXCLAVES_BS_NOT_DEFINED;
66
67 static thread_t exclaves_boot_thread = THREAD_NULL;
68
69 extern exclaves_boot_task_entry_t exclaves_boot_task_entries[]
70 __SECTION_START_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
71
72 extern exclaves_boot_task_entry_t exclaves_boot_task_entries_end[]
73 __SECTION_END_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
74
75 static int
ebt_cmp(const void * e1,const void * e2)76 ebt_cmp(const void *e1, const void *e2)
77 {
78 const struct exclaves_boot_task_entry *a = e1;
79 const struct exclaves_boot_task_entry *b = e2;
80
81 if (a->ebt_rank > b->ebt_rank) {
82 return 1;
83 }
84
85 if (a->ebt_rank < b->ebt_rank) {
86 return -1;
87 }
88
89 return 0;
90 }
91
92 static void
exclaves_boot_tasks(void)93 exclaves_boot_tasks(void)
94 {
95 const size_t count =
96 exclaves_boot_task_entries_end - exclaves_boot_task_entries;
97 assert3u(count, >, 0);
98
99 __assert_only const size_t size =
100 ((uintptr_t)exclaves_boot_task_entries_end -
101 (uintptr_t)exclaves_boot_task_entries);
102 assert3u(size % sizeof(exclaves_boot_task_entry_t), ==, 0);
103
104 /*
105 * exclaves_boot_task_entries is in _DATA_CONST, make a stack copy so it
106 * can be sorted.
107 */
108 exclaves_boot_task_entry_t boot_tasks[count];
109 memcpy(boot_tasks, exclaves_boot_task_entries, count * sizeof(boot_tasks[0]));
110
111 extern void qsort(void *, size_t, size_t,
112 int (*)(const void *, const void *));
113 qsort(boot_tasks, count, sizeof(boot_tasks[0]), ebt_cmp);
114
115 for (size_t i = 0; i < count; i++) {
116 exclaves_debug_printf(show_progress,
117 "exclaves: boot task started, %s\n", boot_tasks[i].ebt_name);
118
119 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
120 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_START, i);
121 kern_return_t ret = boot_tasks[i].ebt_func();
122 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
123 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_END);
124
125 exclaves_debug_printf(show_progress,
126 "exclaves: boot task done, %s\n", boot_tasks[i].ebt_name);
127
128 if (ret != KERN_SUCCESS) {
129 panic("exclaves: boot task failed: %s (%p)",
130 boot_tasks[i].ebt_name, &boot_tasks[i]);
131 }
132 }
133 }
134
135 /*
136 * Check early in boot if the secure world has bootstrapped, and update the
137 * boot status if not.
138 */
139 __startup_func
140 static void
exclaves_check_sk(void)141 exclaves_check_sk(void)
142 {
143 const bool sk_bootstrapped = SPTMArgs->sk_bootstrapped;
144
145 if (sk_bootstrapped) {
146 os_atomic_store(&exclaves_boot_status,
147 EXCLAVES_BS_NOT_STARTED, relaxed);
148 } else {
149 os_atomic_store(&exclaves_boot_status,
150 EXCLAVES_BS_NOT_SUPPORTED, relaxed);
151 }
152 }
153 STARTUP(EXCLAVES, STARTUP_RANK_FIRST, exclaves_check_sk);
154
155 static void
exclaves_boot_status_wait(const exclaves_boot_status_t status)156 exclaves_boot_status_wait(const exclaves_boot_status_t status)
157 {
158 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
159
160 while (os_atomic_load(&exclaves_boot_status, relaxed) < status) {
161 lck_mtx_sleep(&exclaves_boot_lock, LCK_SLEEP_DEFAULT,
162 (event_t)(uintptr_t)&exclaves_boot_status, THREAD_UNINT);
163 }
164 }
165
166 static void
exclaves_boot_status_wake(void)167 exclaves_boot_status_wake(void)
168 {
169 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
170 thread_wakeup((event_t)(uintptr_t)&exclaves_boot_status);
171 }
172
173 static void
exclaves_boot_status_set(const exclaves_boot_status_t status)174 exclaves_boot_status_set(const exclaves_boot_status_t status)
175 {
176 assert3u(status, >, os_atomic_load(&exclaves_boot_status, relaxed));
177
178 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
179
180 /*
181 * 'release' here to ensure that the status update is only seen after
182 * any boot update. exclaves_boot_status is loaded with 'acquire' in
183 * exclaves_boot_wait() without holding the lock so the mutex alone
184 * isn't sufficient for ordering.
185 */
186 os_atomic_store(&exclaves_boot_status, status, release);
187 exclaves_boot_status_wake();
188 }
189
190 static bool
exclaves_boot_status_is_supported(exclaves_boot_status_t status)191 exclaves_boot_status_is_supported(exclaves_boot_status_t status)
192 {
193 assert3u(status, !=, EXCLAVES_BS_NOT_DEFINED);
194
195 return status != EXCLAVES_BS_NOT_DEFINED &&
196 status != EXCLAVES_BS_NOT_SUPPORTED;
197 }
198
199 static kern_return_t
exclaves_boot_stage_2(void)200 exclaves_boot_stage_2(void)
201 {
202 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
203
204 kern_return_t kr = KERN_FAILURE;
205 const exclaves_boot_status_t status =
206 os_atomic_load(&exclaves_boot_status, relaxed);
207
208 if (!exclaves_boot_status_is_supported(status)) {
209 return KERN_NOT_SUPPORTED;
210 }
211
212 /*
213 * This should only ever happen if there's a userspace bug which causes
214 * boot to be called twice and cause a race.
215 */
216 if (status != EXCLAVES_BS_NOT_STARTED) {
217 return KERN_INVALID_ARGUMENT;
218 }
219
220 /* Initialize xnu upcall server. */
221 kr = exclaves_upcall_init();
222 if (kr != KERN_SUCCESS) {
223 panic("Exclaves upcall early init failed");
224 }
225
226 /* Early boot. */
227 extern kern_return_t exclaves_boot_early(void);
228 kr = exclaves_boot_early();
229 if (kr != KERN_SUCCESS) {
230 /*
231 * If exclaves failed to boot, there's not much that can be done other
232 * than panic.
233 */
234 panic("Exclaves stage2 boot failed");
235 }
236
237 /*
238 * At this point it should be possible to make tightbeam calls/configure
239 * endpoints etc.
240 */
241 exclaves_boot_thread = current_thread();
242 exclaves_boot_tasks();
243 exclaves_boot_thread = THREAD_NULL;
244
245 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_STAGE_2);
246
247 return kr;
248 }
249
250 static kern_return_t
exclaves_boot_exclavekit(void)251 exclaves_boot_exclavekit(void)
252 {
253 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
254
255 /* This should require an entitlement at some point. */
256
257 const exclaves_boot_status_t status =
258 os_atomic_load(&exclaves_boot_status, relaxed);
259
260 if (!exclaves_boot_status_is_supported(status)) {
261 return KERN_NOT_SUPPORTED;
262 }
263
264 /* Should only be called after stage2 boot. */
265 if (status != EXCLAVES_BS_BOOTED_STAGE_2) {
266 return KERN_INVALID_ARGUMENT;
267 }
268
269 /*
270 * Treat a failure to boot exclavekit as a transition to the
271 * EXCLAVES_BS_BOOTED_FAILURE state and return a failure. On RELEASE we
272 * simply panic as exclavekit is required.
273 */
274 kern_return_t kr = exclaves_frame_mint_populate();
275 if (kr != KERN_SUCCESS) {
276 exclaves_requirement_assert(EXCLAVES_R_EXCLAVEKIT,
277 "failed to populate frame mint");
278 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE);
279 return KERN_FAILURE;
280 }
281
282 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVEKIT);
283
284 return KERN_SUCCESS;
285 }
286
287 kern_return_t
exclaves_boot(exclaves_boot_stage_t boot_stage)288 exclaves_boot(exclaves_boot_stage_t boot_stage)
289 {
290 lck_mtx_lock(&exclaves_boot_lock);
291
292 kern_return_t kr = KERN_FAILURE;
293
294 switch (boot_stage) {
295 case EXCLAVES_BOOT_STAGE_2:
296 kr = exclaves_boot_stage_2();
297 break;
298
299 case EXCLAVES_BOOT_STAGE_EXCLAVEKIT:
300 kr = exclaves_boot_exclavekit();
301 break;
302
303 default:
304 kr = KERN_INVALID_ARGUMENT;
305 break;
306 }
307
308 lck_mtx_unlock(&exclaves_boot_lock);
309
310 return kr;
311 }
312
313 /*
314 * This returns once the specified boot stage has been reached. If exclaves are
315 * unavailable, it returns immediately with KERN_NOT_SUPPORTED.
316 * Make it NOINLINE so it shows up in backtraces.
317 */
318 kern_return_t OS_NOINLINE
exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)319 exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)
320 {
321 assert(desired_boot_stage == EXCLAVES_BOOT_STAGE_2 ||
322 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVEKIT);
323
324 /* Look up the equivalent status for the specified boot stage. */
325 const exclaves_boot_status_t desired_boot_status =
326 desired_boot_stage == EXCLAVES_BOOT_STAGE_2 ?
327 EXCLAVES_BS_BOOTED_STAGE_2 : EXCLAVES_BS_BOOTED_EXCLAVEKIT;
328
329 /*
330 * See comment in exclaves_boot_status_set() for why this is an acquire.
331 */
332 const exclaves_boot_status_t current_boot_status =
333 os_atomic_load(&exclaves_boot_status, acquire);
334
335 if (current_boot_status >= desired_boot_status) {
336 /*
337 * Special-case the situation where the request is to wait for
338 * EXCLAVES_BOOT_STAGE_EXCLAVEKIT. EXCLAVEKIT boot can fail
339 * (unlike STAGE_2 boot which will panic on failure). If
340 * EXCLAVEKIT has failed, just return KERN_NOT_SUPPORTED.
341 */
342 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
343 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
344 return KERN_NOT_SUPPORTED;
345 }
346
347 return KERN_SUCCESS;
348 }
349
350 if (!exclaves_boot_status_is_supported(current_boot_status)) {
351 return KERN_NOT_SUPPORTED;
352 }
353
354 /*
355 * Allow the exclaves boot thread to pass this check during stage2
356 * boot. This allows exclaves boot tasks to make TB calls etc during
357 * stage2 boot.
358 */
359 if (desired_boot_status == EXCLAVES_BS_BOOTED_STAGE_2 &&
360 current_thread() == exclaves_boot_thread) {
361 return KERN_SUCCESS;
362 }
363
364 /*
365 * Otherwise, wait until exclaves has booted to the requested stage or
366 * failed to boot.
367 */
368 lck_mtx_lock(&exclaves_boot_lock);
369 exclaves_boot_status_wait(desired_boot_status);
370 lck_mtx_unlock(&exclaves_boot_lock);
371
372 /*
373 * At this point there are two possibilities. Success or EXCLAVEKIT has
374 * failed (STAGE_2 can never fail as it panics on failure).
375 */
376 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
377 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
378 return KERN_NOT_SUPPORTED;
379 }
380
381 return KERN_SUCCESS;
382 }
383
384 /*
385 * This returns AVAILABLE once EXCLAVES_BOOT_STAGE_2 has completed. This is to
386 * maintain backwards compatibility with existing code.
387 */
388 exclaves_status_t
exclaves_get_status(void)389 exclaves_get_status(void)
390 {
391 kern_return_t kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_2);
392 assert(kr == KERN_SUCCESS || kr == KERN_NOT_SUPPORTED);
393
394 if (kr == KERN_SUCCESS) {
395 return EXCLAVES_STATUS_AVAILABLE;
396 }
397
398 return EXCLAVES_STATUS_NOT_SUPPORTED;
399 }
400
401 const char *
exclaves_get_boot_status_string(void)402 exclaves_get_boot_status_string(void)
403 {
404 exclaves_boot_status_t boot_status = os_atomic_load(&exclaves_boot_status, relaxed);
405
406 switch (boot_status) {
407 case EXCLAVES_BS_NOT_DEFINED:
408 return "NOT_DEFINED";
409 case EXCLAVES_BS_NOT_SUPPORTED:
410 return "NOT_SUPPORTED";
411 case EXCLAVES_BS_NOT_STARTED:
412 return "NOT_STARTED";
413 case EXCLAVES_BS_BOOTED_STAGE_2:
414 return "BOOTED_STAGE_2";
415 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
416 return "BOOTED_EXCLAVEKIT";
417 case EXCLAVES_BS_BOOTED_FAILURE:
418 return "BOOTED_FAILURE";
419 default:
420 assertf(false, "unknown boot status %u", boot_status);
421 return "UNKNOWN";
422 }
423 }
424
425 exclaves_boot_stage_t
exclaves_get_boot_stage(void)426 exclaves_get_boot_stage(void)
427 {
428 exclaves_boot_status_t status =
429 os_atomic_load(&exclaves_boot_status, relaxed);
430
431 switch (status) {
432 case EXCLAVES_BS_NOT_STARTED:
433 case EXCLAVES_BS_NOT_SUPPORTED:
434 case EXCLAVES_BS_NOT_DEFINED:
435 return EXCLAVES_BOOT_STAGE_NONE;
436
437 case EXCLAVES_BS_BOOTED_STAGE_2:
438 return EXCLAVES_BOOT_STAGE_2;
439
440 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
441 return EXCLAVES_BOOT_STAGE_EXCLAVEKIT;
442
443 case EXCLAVES_BS_BOOTED_FAILURE:
444 return EXCLAVES_BOOT_STAGE_FAILED;
445
446 default:
447 panic("unknown boot status: %u", status);
448 }
449 }
450
451 bool
exclaves_boot_supported(void)452 exclaves_boot_supported(void)
453 {
454 exclaves_boot_status_t status;
455
456 status = os_atomic_load(&exclaves_boot_status, relaxed);
457 return exclaves_boot_status_is_supported(status);
458 }
459
460 #else /* CONFIG_EXCLAVES */
461
462 exclaves_status_t
exclaves_get_status(void)463 exclaves_get_status(void)
464 {
465 return EXCLAVES_STATUS_NOT_SUPPORTED;
466 }
467
468 exclaves_boot_stage_t
exclaves_get_boot_stage(void)469 exclaves_get_boot_stage(void)
470 {
471 return EXCLAVES_BOOT_STAGE_NONE;
472 }
473
474 bool
exclaves_boot_supported(void)475 exclaves_boot_supported(void)
476 {
477 return false;
478 }
479
480 #endif /* CONFIG_EXCLAVES */
481