1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if CONFIG_EXCLAVES
30
31 #include <stdint.h>
32 #include <stdbool.h>
33
34 #include <arm/misc_protos.h>
35
36 #include <mach/exclaves.h>
37 #include <mach/kern_return.h>
38
39 #include <string.h>
40
41 #include <kern/assert.h>
42 #include <kern/bits.h>
43 #include <kern/queue.h>
44 #include <kern/kalloc.h>
45 #include <kern/locks.h>
46 #include <kern/task.h>
47 #include <kern/thread_call.h>
48
49 #include <vm/pmap.h>
50
51 #include <kern/ipc_kobject.h>
52
53 #include <os/hash.h>
54
55 #include <mach/mach_traps.h>
56 #include <mach/mach_port.h>
57
58 #include <sys/event.h>
59 #include <sys/reason.h>
60
61 #include "exclaves_aoe.h"
62 #include "exclaves_conclave.h"
63 #include "exclaves_debug.h"
64 #include "exclaves_memory.h"
65 #include "exclaves_resource.h"
66 #include "exclaves_sensor.h"
67 #include "exclaves_shared_memory.h"
68 #include "exclaves_xnuproxy.h"
69
70 #include "kern/exclaves.tightbeam.h"
71
72 static LCK_GRP_DECLARE(resource_lck_grp, "exclaves_resource");
73 static kern_return_t
74 exclaves_update_state_machine_locked(exclaves_resource_t *resource);
75
76 /*
77 * A cache of service ids in the kernel domain
78 */
79 static bitmap_t
80 kernel_service_bitmap[BITMAP_LEN(CONCLAVE_SERVICE_MAX)] = {0};
81
82 /*
83 * Exclave Resources
84 *
85 * Exclaves provide a fixed static set of resources available to XNU. Some
86 * examples of types of resources:
87 * - Conclave managers
88 * - Services
89 * - Named buffers
90 * - Audio buffers
91 * ...
92 *
93 * Each resource has a name, a type and a corresponding identifier which is
94 * shared between XNU and Exclaves. Resources are scoped by what entities are
95 * allowed to access them.
96 * Resources are discovered during boot and made available in a two-level table
97 * scheme. The root table collects resources by their scope, with the
98 * second-level tables listing the actual resources.
99 *
100 *
101 * Root Table
102 * ┌────────────────────────────┐
103 * │ ┌────────────────────────┐ │
104 * │ │ "com.apple.kernel" │─┼─────┐
105 * │ └────────────────────────┘ │ │
106 * │ ┌────────────────────────┐ │ │
107 * │ │"com.apple.conclave.a" │─┼─┐ │
108 * │ └────────────────────────┘ │ │ │
109 * │ ┌────────────────────────┐ │ │ │
110 * │ │"com.apple.conclave.b" │ │ │ │
111 * │ └────────────────────────┘ │ │ │
112 * │ ┌────────────────────────┐ │ │ │
113 * │ │ "com.apple.driver.a" │ │ │ │
114 * │ └────────────────────────┘ │ │ │
115 * │ ... │ │ │
116 * │ │ │ │
117 * └────────────────────────────┘ │ │
118 * ┌─────────────────────────┘ │
119 * │ │
120 * │ ┌─────────────────────────┘
121 * │ │
122 * │ │
123 * │ │
124 * │ └──▶ "com.apple.kernel"
125 * │ ┌─────────────────────────────────────────────────────┐
126 * │ │┌───────────────────────┬──────────────────┬────────┐│
127 * │ ││"com.apple.conclave.a" │ CONCLAVE_MANAGER │ 0x1234 ││
128 * │ │└───────────────────────┴──────────────────┴────────┘│
129 * │ │┌───────────────────────┬──────────────────┬────────┐│
130 * │ ││"com.apple.conclave.b" │ CONCLAVE_MANAGER │ 0x7654 ││
131 * │ │└───────────────────────┴──────────────────┴────────┘│
132 * │ │ │
133 * │ │ ... │
134 * │ └─────────────────────────────────────────────────────┘
135 * │
136 * └─────▶ "com.apple.conclave.a"
137 * ┌─────────────────────────────────────────────────────┐
138 * │┌───────────────────────┬──────────────────┬────────┐│
139 * ││ "audio_buf" │ AUDIO_BUFFER │ 0x9999 ││
140 * │└───────────────────────┴──────────────────┴────────┘│
141 * │┌───────────────────────┬──────────────────┬────────┐│
142 * ││ "service_x" │ SERVICE │ 0x1111 ││
143 * │└───────────────────────┴──────────────────┴────────┘│
144 * │┌───────────────────────┬──────────────────┬────────┐│
145 * ││ "named_buffer_x" │ NAMED_BUFFER │0x66565 ││
146 * │└───────────────────────┴──────────────────┴────────┘│
147 * │ ... │
148 * └─────────────────────────────────────────────────────┘
149 *
150 * ...
151 *
152 *
153 * Resources can be looked up by first finding the root table entry (the
154 * "domain") and then searching for the identifier in that domain.
155 * For example to lookup the conclave manager ID for "com.apple.conclave.a",
156 * the "com.apple.kernel" domain would be found and then within that domain, the
157 * search would continue using the conclave name and the CONCLAVE_MANAGER type.
158 * Every conclave domain has a corresponding CONCLAVE_MANAGER resource in the
159 * "com.apple.kernel" domain.
160 */
161
162 /* -------------------------------------------------------------------------- */
163 #pragma mark Hash Table
164
165 #define TABLE_LEN 64
166
167 /*
168 * A table item is what ends up being stored in the hash table. It has a key and
169 * a value.
170 */
171 typedef struct {
172 const void *i_key;
173 size_t i_key_len;
174 void *i_value;
175
176 queue_chain_t i_chain;
177 } table_item_t;
178
179 /*
180 * The hash table consists of an array of buckets (queues). The hashing function
181 * will choose in which bucket a particular item belongs.
182 */
183 typedef struct {
184 queue_head_t *t_buckets;
185 size_t t_buckets_count;
186 } table_t;
187
188 /*
189 * Given a key, return the corresponding bucket.
190 */
191 static queue_head_t *
get_bucket(table_t * table,const void * key,size_t key_len)192 get_bucket(table_t *table, const void *key, size_t key_len)
193 {
194 const uint32_t idx = os_hash_jenkins(key, key_len) &
195 (table->t_buckets_count - 1);
196 return &table->t_buckets[idx];
197 }
198
199 /*
200 * Insert a new table item associated with 'key' into a table.
201 */
202 static void
table_put(table_t * table,const void * key,size_t key_len,table_item_t * item)203 table_put(table_t *table, const void *key, size_t key_len, table_item_t *item)
204 {
205 assert3p(item->i_chain.next, ==, NULL);
206 assert3p(item->i_chain.prev, ==, NULL);
207 assert3p(item->i_value, !=, NULL);
208
209 queue_head_t *head = get_bucket(table, key, key_len);
210 enqueue(head, &item->i_chain);
211 }
212
213 /*
214 * Iterate through all items matching 'key' calling cb for each.
215 */
216 static void
217 table_get(table_t *table, const void *key, size_t key_len, bool (^cb)(void *))
218 {
219 const queue_head_t *head = get_bucket(table, key, key_len);
220 table_item_t *elem = NULL;
221
222 assert3p(head, !=, NULL);
223
qe_foreach_element(elem,head,i_chain)224 qe_foreach_element(elem, head, i_chain) {
225 if (elem->i_key_len == key_len &&
226 memcmp(elem->i_key, key, elem->i_key_len) == 0) {
227 if (cb(elem->i_value)) {
228 return;
229 }
230 }
231 }
232
233 return;
234 }
235
236 /*
237 * Initialize the queues.
238 */
239 static void
table_init(table_t * table)240 table_init(table_t *table)
241 {
242 assert3u(table->t_buckets_count & (table->t_buckets_count - 1), ==, 0);
243
244 /* Initialise each bucket. */
245 for (size_t i = 0; i < table->t_buckets_count; i++) {
246 queue_init(&table->t_buckets[i]);
247 }
248 }
249
250 /*
251 * Allocate a new table with the specified number of buckets.
252 */
253 static table_t *
table_alloc(size_t nbuckets)254 table_alloc(size_t nbuckets)
255 {
256 assert3u(nbuckets, >, 0);
257 assert3u(nbuckets & (nbuckets - 1), ==, 0);
258
259 table_t *table = kalloc_type(table_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
260
261 table->t_buckets_count = nbuckets;
262 table->t_buckets = kalloc_type(queue_head_t, nbuckets,
263 Z_WAITOK | Z_ZERO | Z_NOFAIL);
264
265 return table;
266 }
267
268 static void
269 table_iterate(table_t *table,
270 bool (^cb)(const void *key, size_t key_len, void *value))
271 {
272 for (size_t i = 0; i < table->t_buckets_count; i++) {
273 const queue_head_t *head = &table->t_buckets[i];
274 table_item_t *elem = NULL;
275
qe_foreach_element(elem,head,i_chain)276 qe_foreach_element(elem, head, i_chain) {
277 if (cb(elem->i_key, elem->i_key_len, elem->i_value)) {
278 return;
279 }
280 }
281 }
282 }
283
284
285 /* -------------------------------------------------------------------------- */
286 #pragma mark Root Table
287
288 /*
289 * The root table is a hash table which contains an entry for every top-level
290 * domain.
291 * Domains scope resources. For example a conclave domain will contain a list of
292 * services available in that conclave. The kernel itself gets its own domain
293 * which holds conclave managers and other resources the kernel communicates
294 * with directly.
295 */
296 table_t root_table = {
297 .t_buckets = (queue_chain_t *)(queue_chain_t[TABLE_LEN]){},
298 .t_buckets_count = TABLE_LEN,
299 };
300
301 /*
302 * Entries in the root table. Each itself a table containing resources available
303 * in that domain.
304 */
305 typedef struct {
306 char d_name[EXCLAVES_RESOURCE_NAME_MAX];
307 table_t *d_table_name;
308 table_t *d_table_id;
309 } exclaves_resource_domain_t;
310
311 static exclaves_resource_domain_t *
lookup_domain(const char * domain_name)312 lookup_domain(const char *domain_name)
313 {
314 __block exclaves_resource_domain_t *domain = NULL;
315 table_get(&root_table, domain_name, strlen(domain_name), ^bool (void *data) {
316 domain = data;
317 return true;
318 });
319
320 return domain;
321 }
322
323 static void
324 iterate_domains(bool (^cb)(exclaves_resource_domain_t *))
325 {
326 table_iterate(&root_table,
327 ^(__unused const void *key, __unused size_t key_len, void *value) {
328 exclaves_resource_domain_t *domain = value;
329 return cb(domain);
330 });
331 }
332
333 static void
334 iterate_resources(exclaves_resource_domain_t *domain,
335 bool (^cb)(exclaves_resource_t *))
336 {
337 table_iterate(domain->d_table_name,
338 ^(__unused const void *key, __unused size_t key_len, void *value) {
339 exclaves_resource_t *resource = value;
340 return cb(resource);
341 });
342 }
343
344 static exclaves_resource_t *
lookup_resource_by_name(exclaves_resource_domain_t * domain,const char * name,xnuproxy_resourcetype_s type)345 lookup_resource_by_name(exclaves_resource_domain_t *domain, const char *name,
346 xnuproxy_resourcetype_s type)
347 {
348 __block exclaves_resource_t *resource = NULL;
349 table_get(domain->d_table_name, name, strlen(name), ^bool (void *data) {
350 exclaves_resource_t *tmp = data;
351 if (tmp->r_type == type) {
352 resource = data;
353 return true;
354 }
355 return false;
356 });
357
358 return resource;
359 }
360
361 static exclaves_resource_t *
lookup_resource_by_id(exclaves_resource_domain_t * domain,uint64_t id,xnuproxy_resourcetype_s type)362 lookup_resource_by_id(exclaves_resource_domain_t *domain, uint64_t id,
363 xnuproxy_resourcetype_s type)
364 {
365 __block exclaves_resource_t *resource = NULL;
366
367 table_get(domain->d_table_id, &id, sizeof(id), ^bool (void *data) {
368 exclaves_resource_t *tmp = data;
369 if (tmp->r_type == type) {
370 resource = data;
371 return true;
372 }
373 return false;
374 });
375
376 return resource;
377 }
378
379 static exclaves_resource_domain_t *
exclaves_resource_domain_alloc(const char * scope)380 exclaves_resource_domain_alloc(const char *scope)
381 {
382 assert3u(strlen(scope), >, 0);
383 assert3u(strlen(scope), <=, EXCLAVES_RESOURCE_NAME_MAX);
384
385 exclaves_resource_domain_t *domain = kalloc_type(
386 exclaves_resource_domain_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
387 (void) strlcpy(domain->d_name, scope,
388 sizeof(domain->d_name));
389
390 domain->d_table_name = table_alloc(TABLE_LEN);
391 table_init(domain->d_table_name);
392
393 domain->d_table_id = table_alloc(TABLE_LEN);
394 table_init(domain->d_table_id);
395
396 table_item_t *item = kalloc_type(table_item_t,
397 Z_WAITOK | Z_ZERO | Z_NOFAIL);
398 item->i_key = domain->d_name;
399 item->i_key_len = strlen(domain->d_name);
400 item->i_value = domain;
401
402 table_put(&root_table, scope, strlen(scope), item);
403
404 return domain;
405 }
406
407 static void
exclaves_resource_insert_name_table(xnuproxy_resourcetype_s type __unused,const char * name,exclaves_resource_domain_t * domain,exclaves_resource_t * resource)408 exclaves_resource_insert_name_table(xnuproxy_resourcetype_s type __unused, const char *name,
409 exclaves_resource_domain_t *domain, exclaves_resource_t *resource)
410 {
411 table_item_t *name_item = kalloc_type(table_item_t,
412 Z_WAITOK | Z_ZERO | Z_NOFAIL);
413
414 name_item->i_key = resource->r_name;
415 name_item->i_key_len = strlen(resource->r_name);
416 name_item->i_value = resource;
417
418 assertf(lookup_resource_by_name(domain, name, type) == NULL,
419 "Duplicate entry in exclaves resource table for \"%s\" , \"%s\"", domain->d_name, name);
420 table_put(domain->d_table_name, name, strlen(name), name_item);
421 }
422
423 static void
exclaves_resource_insert_id_table(xnuproxy_resourcetype_s type,uint64_t id,exclaves_resource_t * resource)424 exclaves_resource_insert_id_table(xnuproxy_resourcetype_s type, uint64_t id,
425 exclaves_resource_t *resource)
426 {
427 switch (type) {
428 case XNUPROXY_RESOURCETYPE_NOTIFICATION: {
429 /* Stick the newly created resource into the ID table. */
430 table_item_t *id_item = kalloc_type(table_item_t,
431 Z_WAITOK | Z_ZERO | Z_NOFAIL);
432 id_item->i_key = &resource->r_id;
433 id_item->i_key_len = sizeof(resource->r_id);
434 id_item->i_value = resource;
435
436 /*
437 * Globally unique notification ids are added to the kernel domain for
438 * lookup while signalling
439 */
440 exclaves_resource_domain_t *kernel_domain = lookup_domain(EXCLAVES_DOMAIN_KERNEL);
441 table_put(kernel_domain->d_table_id, &id, sizeof(id), id_item);
442
443 break;
444 }
445
446 default:
447 break;
448 }
449 }
450
451 static exclaves_resource_t *
exclaves_resource_alloc(xnuproxy_resourcetype_s type,const char * name,uint64_t id,exclaves_resource_domain_t * domain,bool connected)452 exclaves_resource_alloc(xnuproxy_resourcetype_s type, const char *name, uint64_t id,
453 exclaves_resource_domain_t *domain, bool connected)
454 {
455 if (type == XNUPROXY_RESOURCETYPE_NOTIFICATION) {
456 exclaves_resource_t *resource = exclaves_notification_lookup_by_id(id);
457 if (resource != NULL) {
458 /*
459 * Name entry should refer to the resource associated with the
460 * already present id
461 */
462 exclaves_resource_insert_name_table(type, name, domain, resource);
463 return NULL;
464 }
465 }
466
467 exclaves_resource_t *resource = kalloc_type(exclaves_resource_t,
468 Z_WAITOK | Z_ZERO | Z_NOFAIL);
469
470 resource->r_type = type;
471 resource->r_id = id;
472 resource->r_active = false;
473 resource->r_connected = connected;
474 os_atomic_store(&resource->r_usecnt, 0, relaxed);
475
476 /*
477 * Each resource has an associated kobject of type
478 * IKOT_EXCLAVES_RESOURCE.
479 */
480 ipc_port_t port = ipc_kobject_alloc_port(resource,
481 IKOT_EXCLAVES_RESOURCE, IPC_KOBJECT_ALLOC_NONE);
482 resource->r_port = port;
483
484 lck_mtx_init(&resource->r_mutex, &resource_lck_grp, NULL);
485
486 (void) strlcpy(resource->r_name, name, sizeof(resource->r_name));
487
488 /*
489 * Add the resource to the name table, used for lookup
490 */
491 exclaves_resource_insert_name_table(type, name, domain, resource);
492
493 /*
494 * Some types also need to lookup by id in addition to looking up by
495 * name.
496 */
497 exclaves_resource_insert_id_table(type, id, resource);
498
499 return resource;
500 }
501
502 /* -------------------------------------------------------------------------- */
503 #pragma mark Exclaves Resources
504
505 static void exclaves_resource_no_senders(ipc_port_t port,
506 mach_port_mscount_t mscount);
507
508 IPC_KOBJECT_DEFINE(IKOT_EXCLAVES_RESOURCE,
509 .iko_op_movable_send = true,
510 .iko_op_stable = true,
511 .iko_op_no_senders = exclaves_resource_no_senders);
512
513 static void exclaves_conclave_init(exclaves_resource_t *resource);
514 static void exclaves_notification_init(exclaves_resource_t *resource);
515 static void exclaves_resource_sensor_reset(exclaves_resource_t *resource);
516 static void exclaves_resource_shared_memory_unmap(exclaves_resource_t *resource);
517 static void exclaves_resource_audio_memory_unmap(exclaves_resource_t *resource);
518
519 static void
populate_conclave_services(void)520 populate_conclave_services(void)
521 {
522 /* BEGIN IGNORE CODESTYLE */
523 iterate_domains(^(exclaves_resource_domain_t *domain) {
524
525 const bool is_kernel_domain =
526 (strcmp(domain->d_name, EXCLAVES_DOMAIN_KERNEL) == 0 ||
527 strcmp(domain->d_name, EXCLAVES_DOMAIN_DARWIN) == 0);
528
529 exclaves_resource_t *cm = exclaves_resource_lookup_by_name(
530 EXCLAVES_DOMAIN_KERNEL, domain->d_name,
531 XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
532
533 iterate_resources(domain, ^(exclaves_resource_t *resource) {
534 if (resource->r_type != XNUPROXY_RESOURCETYPE_SERVICE) {
535 return (bool)false;
536 }
537
538 if (cm != NULL) {
539 conclave_resource_t *c = &cm->r_conclave;
540 if (exclaves_is_forwarding_resource(resource)) {
541 return (bool)false;
542 }
543 assert3u(resource->r_id, <, CONCLAVE_SERVICE_MAX);
544 bitmap_set(c->c_service_bitmap, (uint32_t)resource->r_id);
545 return (bool)false;
546 }
547
548 if (is_kernel_domain) {
549 bitmap_set(kernel_service_bitmap,
550 (uint32_t)resource->r_id);
551 return (bool)false;
552
553 }
554
555 /*
556 * Ignore services that are in unknown domains. This can
557 * happen if a conclave manager doesn't have a populated
558 * endpoint (for example during bringup).
559 */
560 return (bool)false;
561 });
562
563 return (bool)false;
564 });
565 /* END IGNORE CODESTYLE */
566 }
567
568 /*
569 * The aoe_service_table is a hash table which contains a map of aoe service to
570 * conclave.
571 */
572 static table_t aoe_service_table = {
573 .t_buckets = (queue_chain_t *)(queue_chain_t[TABLE_LEN]){},
574 .t_buckets_count = TABLE_LEN,
575 };
576
577 exclaves_resource_t *
exclaves_conclave_lookup_by_aoeserviceid(uint64_t id)578 exclaves_conclave_lookup_by_aoeserviceid(uint64_t id)
579 {
580 __block exclaves_resource_t *resource = NULL;
581 table_get(&aoe_service_table, &id, sizeof(id), ^bool (void *data) {
582 resource = data;
583 return true;
584 });
585
586 /* Ignore entries not marked connected. */
587 if (resource == NULL || !resource->r_connected) {
588 return NULL;
589 }
590
591 return resource;
592 }
593
594 static void
populate_aoeservice_to_conclave(void)595 populate_aoeservice_to_conclave(void)
596 {
597 table_init(&aoe_service_table);
598
599 /* BEGIN IGNORE CODESTYLE */
600 iterate_domains(^(exclaves_resource_domain_t *domain) {
601
602 exclaves_resource_t *cm = exclaves_resource_lookup_by_name(
603 EXCLAVES_DOMAIN_KERNEL, domain->d_name,
604 XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
605 if (cm == NULL) {
606 return (bool)false;
607 }
608
609 iterate_resources(domain, ^(exclaves_resource_t *resource) {
610 if (resource->r_type != XNUPROXY_RESOURCETYPE_ALWAYSONEXCLAVESSERVICE) {
611 return (bool)false;
612 }
613
614 /* Found an ALWAYSONEXCLAVESSERVICE, add an entry to the map. */
615
616 /* Assert that there's no existing entry. */
617 assert3p(exclaves_conclave_lookup_by_aoeserviceid(resource->r_id),
618 ==, NULL);
619
620 /* Stick the newly created resource into the table. */
621 table_item_t *item = kalloc_type(table_item_t,
622 Z_WAITOK | Z_ZERO | Z_NOFAIL);
623
624 item->i_key = &resource->r_id;
625 item->i_key_len = sizeof(resource->r_id);
626 item->i_value = cm;
627
628 table_put(&aoe_service_table, &resource->r_id,
629 sizeof(resource->r_id), item);
630
631 return (bool)false;
632 });
633
634 return (bool)false;
635 });
636 /* END IGNORE CODESTYLE */
637 }
638
639 /*
640 * Discover all the static exclaves resources populating the resource tables as
641 * we go.
642 */
643 kern_return_t
exclaves_resource_init(void)644 exclaves_resource_init(void)
645 {
646 /* Initialize the root table. */
647 table_init(&root_table);
648
649 /* BEGIN IGNORE CODESTYLE */
650 kern_return_t kr = exclaves_xnuproxy_resource_info(
651 ^(const char *name, const char *scope,
652 xnuproxy_resourcetype_s type, uint64_t id, bool connected) {
653 /*
654 * Every resource is scoped to a specific domain, find the
655 * domain (or create one if it doesn't exist).
656 */
657 exclaves_resource_domain_t *domain = lookup_domain(scope);
658 if (domain == NULL) {
659 domain = exclaves_resource_domain_alloc(scope);
660 }
661
662 /* Allocate a new resource in the domain. */
663 exclaves_resource_t *resource = exclaves_resource_alloc(type,
664 name, id, domain, connected);
665
666 if (!resource) {
667 assert3u(type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
668 return;
669 }
670
671 /*
672 * Type specific initialization.
673 */
674 switch (type) {
675 case XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER:
676 exclaves_conclave_init(resource);
677 break;
678
679 case XNUPROXY_RESOURCETYPE_NOTIFICATION:
680 exclaves_notification_init(resource);
681 break;
682
683 default:
684 break;
685 }
686
687
688 });
689 /* END IGNORE CODESTYLE */
690
691 if (kr != KERN_SUCCESS) {
692 return kr;
693 }
694
695 /* Populate the conclave service ID bitmaps. */
696 populate_conclave_services();
697
698 /* Build a map of AOE service -> conclave. */
699 populate_aoeservice_to_conclave();
700
701 return KERN_SUCCESS;
702 }
703
704 exclaves_resource_t *
exclaves_resource_lookup_by_name(const char * domain_name,const char * name,xnuproxy_resourcetype_s type)705 exclaves_resource_lookup_by_name(const char *domain_name, const char *name,
706 xnuproxy_resourcetype_s type)
707 {
708 assert3u(strlen(domain_name), >, 0);
709 assert3u(strlen(name), >, 0);
710
711 exclaves_resource_domain_t *domain = lookup_domain(domain_name);
712 if (domain == NULL) {
713 return NULL;
714 }
715
716 exclaves_resource_t *r = lookup_resource_by_name(domain, name, type);
717
718 if (r == NULL) {
719 return NULL;
720 }
721
722 /*
723 * Ignore exclave resources that are not connected
724 */
725 if (!r->r_connected) {
726 switch (r->r_type) {
727 case XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER:
728 case XNUPROXY_RESOURCETYPE_SERVICE:
729 if (exclaves_is_forwarding_resource(r)) {
730 break;
731 }
732 default:
733 return NULL;
734 }
735 }
736
737 return r;
738 }
739
740 static exclaves_resource_t *
exclaves_resource_lookup_by_id(const char * domain_name,uint64_t id,xnuproxy_resourcetype_s type)741 exclaves_resource_lookup_by_id(const char *domain_name, uint64_t id,
742 xnuproxy_resourcetype_s type)
743 {
744 assert3u(strlen(domain_name), >, 0);
745
746 exclaves_resource_domain_t *domain = lookup_domain(domain_name);
747 if (domain == NULL) {
748 return NULL;
749 }
750
751 exclaves_resource_t *r = lookup_resource_by_id(domain, id, type);
752
753 /* Ignore entries not marked connected. */
754 if (r == NULL || !r->r_connected) {
755 return NULL;
756 }
757
758 return r;
759 }
760
761 const char *
exclaves_resource_name(const exclaves_resource_t * resource)762 exclaves_resource_name(const exclaves_resource_t *resource)
763 {
764 return resource->r_name;
765 }
766
767 /*
768 * Notes on use-count management
769 * For the most part everything is done under the resource lock.
770 * In some cases, it's necessary to grab/release a use count without
771 * holding the lock - for example the realtime audio paths doing copyin/copyout
772 * of named buffers/audio buffers.
773 * To prevent against races, initialization/de-initialization should always
774 * recheck the use-count under the lock.
775 */
776 uint32_t
exclaves_resource_retain(exclaves_resource_t * resource)777 exclaves_resource_retain(exclaves_resource_t *resource)
778 {
779 uint32_t orig =
780 os_atomic_inc_orig(&resource->r_usecnt, relaxed);
781 assert3u(orig, <, UINT32_MAX);
782
783 return orig;
784 }
785
786 void
exclaves_resource_release(exclaves_resource_t * resource)787 exclaves_resource_release(exclaves_resource_t *resource)
788 {
789 /*
790 * Drop the use count without holding the lock (this path may be called
791 * by RT threads and should be RT-safe).
792 */
793 uint32_t orig = os_atomic_dec_orig(&resource->r_usecnt, release);
794 assert3u(orig, !=, 0);
795 if (orig != 1) {
796 return;
797 }
798
799 /*
800 * Now grab the lock. The RT-safe paths calling this function shouldn't
801 * end up here unless there's a bug or mis-behaving user code (like
802 * deallocating an in-use mach port).
803 */
804 lck_mtx_lock(&resource->r_mutex);
805
806 /*
807 * Re-check the use count - as a second user of the resource
808 * may have snuck in in the meantime.
809 */
810 if (os_atomic_load(&resource->r_usecnt, acquire) > 0) {
811 lck_mtx_unlock(&resource->r_mutex);
812 return;
813 }
814
815 switch (resource->r_type) {
816 case XNUPROXY_RESOURCETYPE_SENSOR:
817 exclaves_resource_sensor_reset(resource);
818 break;
819
820 case XNUPROXY_RESOURCETYPE_SHAREDMEMORY:
821 exclaves_resource_shared_memory_unmap(resource);
822 break;
823
824 case XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY:
825 exclaves_resource_audio_memory_unmap(resource);
826 break;
827
828 default:
829 break;
830 }
831
832 lck_mtx_unlock(&resource->r_mutex);
833 }
834
835 kern_return_t
exclaves_resource_from_port_name(ipc_space_t space,mach_port_name_t name,exclaves_resource_t ** out)836 exclaves_resource_from_port_name(ipc_space_t space, mach_port_name_t name,
837 exclaves_resource_t **out)
838 {
839 kern_return_t kr = KERN_SUCCESS;
840 ipc_port_t port = IPC_PORT_NULL;
841
842 if (!MACH_PORT_VALID(name)) {
843 return KERN_INVALID_NAME;
844 }
845
846 kr = ipc_port_translate_send(space, name, &port);
847 if (kr != KERN_SUCCESS) {
848 return kr;
849 }
850
851 /* port is locked */
852 assert(IP_VALID(port));
853
854 exclaves_resource_t *resource = ipc_kobject_get_stable(port,
855 IKOT_EXCLAVES_RESOURCE);
856
857 /* The port is valid, but doesn't denote an exclaves resource. */
858 if (resource == NULL) {
859 ip_mq_unlock(port);
860 return KERN_INVALID_CAPABILITY;
861 }
862
863 /* Grab a reference while the port is good and the ipc lock is held. */
864 __assert_only uint32_t orig = exclaves_resource_retain(resource);
865 assert3u(orig, >, 0);
866
867 ip_mq_unlock(port);
868 *out = resource;
869
870 return KERN_SUCCESS;
871 }
872
873 /*
874 * Consumes a reference to the resource. On success the resource is reference is
875 * associated with the lifetime of the port.
876 */
877 kern_return_t
exclaves_resource_create_port_name(exclaves_resource_t * resource,ipc_space_t space,mach_port_name_t * name)878 exclaves_resource_create_port_name(exclaves_resource_t *resource, ipc_space_t space,
879 mach_port_name_t *name)
880 {
881 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), >, 0);
882
883 /*
884 * make a send right and donate our reference for
885 * exclaves_resource_no_senders if this is the first send right
886 */
887 if (!ipc_kobject_make_send_lazy_alloc_port(&resource->r_port,
888 resource, IKOT_EXCLAVES_RESOURCE)) {
889 exclaves_resource_release(resource);
890 }
891
892 *name = ipc_port_copyout_send(resource->r_port, space);
893 if (!MACH_PORT_VALID(*name)) {
894 /*
895 * ipc_port_copyout_send() releases the send right on failure
896 * (possibly calling exclaves_resource_no_senders() in the
897 * process).
898 */
899 return KERN_RESOURCE_SHORTAGE;
900 }
901
902 return KERN_SUCCESS;
903 }
904
905 static void
exclaves_resource_no_senders(ipc_port_t port,__unused mach_port_mscount_t mscount)906 exclaves_resource_no_senders(ipc_port_t port,
907 __unused mach_port_mscount_t mscount)
908 {
909 exclaves_resource_t *resource = ipc_kobject_get_stable(port,
910 IKOT_EXCLAVES_RESOURCE);
911
912 exclaves_resource_release(resource);
913 }
914
915 /* -------------------------------------------------------------------------- */
916 #pragma mark Conclave Manager
917
918 static void
exclaves_conclave_init(exclaves_resource_t * resource)919 exclaves_conclave_init(exclaves_resource_t *resource)
920 {
921 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
922
923 tb_client_connection_t connection = NULL;
924 __assert_only kern_return_t kr = exclaves_conclave_launcher_init(resource->r_id,
925 &connection);
926 assert3u(kr, ==, KERN_SUCCESS);
927
928 conclave_resource_t *conclave = &resource->r_conclave;
929
930 conclave->c_control = connection;
931 conclave->c_state = CONCLAVE_S_NONE;
932 conclave->c_request = CONCLAVE_R_NONE;
933 conclave->c_active_downcall = false;
934 conclave->c_active_stopcall = false;
935 conclave->c_downcall_thread = THREAD_NULL;
936 conclave->c_task = TASK_NULL;
937
938 queue_init(&conclave->c_aoe_q);
939 }
940
941 kern_return_t
exclaves_conclave_attach(const char * name,task_t task)942 exclaves_conclave_attach(const char *name, task_t task)
943 {
944 assert3p(task, !=, TASK_NULL);
945
946 exclaves_resource_t *resource = exclaves_resource_lookup_by_name(
947 EXCLAVES_DOMAIN_KERNEL, name, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
948 if (resource == NULL) {
949 /* Just return success here. The conclave launch will fail. */
950 return KERN_SUCCESS;
951 }
952 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
953
954 conclave_resource_t *conclave = &resource->r_conclave;
955
956 lck_mtx_lock(&resource->r_mutex);
957
958 if (conclave->c_state != CONCLAVE_S_NONE) {
959 lck_mtx_unlock(&resource->r_mutex);
960 return KERN_INVALID_ARGUMENT;
961 }
962
963 task_reference(task);
964
965 task->conclave = resource;
966
967 conclave->c_task = task;
968 conclave->c_state = CONCLAVE_S_ATTACHED;
969
970 lck_mtx_unlock(&resource->r_mutex);
971
972 return KERN_SUCCESS;
973 }
974
975 kern_return_t
exclaves_conclave_detach(exclaves_resource_t * resource,task_t task)976 exclaves_conclave_detach(exclaves_resource_t *resource, task_t task)
977 {
978 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
979
980 conclave_resource_t *conclave = &resource->r_conclave;
981
982 lck_mtx_lock(&resource->r_mutex);
983
984 while (conclave->c_active_downcall) {
985 conclave->c_active_detach = true;
986 assert3p(conclave->c_downcall_thread, !=, THREAD_NULL);
987 lck_mtx_sleep_with_inheritor(&resource->r_mutex,
988 LCK_SLEEP_DEFAULT,
989 (event_t)&conclave->c_active_downcall,
990 conclave->c_downcall_thread,
991 THREAD_UNINT,
992 TIMEOUT_WAIT_FOREVER);
993 conclave->c_active_detach = false;
994 }
995
996 if (conclave->c_state != CONCLAVE_S_ATTACHED &&
997 conclave->c_state != CONCLAVE_S_STOPPED) {
998 panic("Task %p trying to detach a conclave %p but it is in a "
999 "weird state", task, conclave);
1000 }
1001
1002 assert3u(conclave->c_active_downcall, ==, 0);
1003 assert3u(conclave->c_active_stopcall, ==, 0);
1004 assert3p(conclave->c_downcall_thread, ==, THREAD_NULL);
1005 assert3u(conclave->c_request, ==, CONCLAVE_R_NONE);
1006 assert3p(task->conclave, !=, NULL);
1007 assert3p(resource, ==, task->conclave);
1008
1009 /* Cleanup any residual AOE state. */
1010 exclaves_aoe_teardown();
1011
1012 task->conclave = NULL;
1013 conclave->c_task = TASK_NULL;
1014
1015 conclave->c_state = CONCLAVE_S_NONE;
1016
1017 lck_mtx_unlock(&resource->r_mutex);
1018
1019 task_deallocate(task);
1020
1021 return KERN_SUCCESS;
1022 }
1023
1024 kern_return_t
exclaves_conclave_inherit(exclaves_resource_t * resource,task_t old_task,task_t new_task)1025 exclaves_conclave_inherit(exclaves_resource_t *resource, task_t old_task,
1026 task_t new_task)
1027 {
1028 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1029
1030 conclave_resource_t *conclave = &resource->r_conclave;
1031
1032 lck_mtx_lock(&resource->r_mutex);
1033
1034 assert3u(conclave->c_state, !=, CONCLAVE_S_NONE);
1035
1036 assert3p(new_task->conclave, ==, NULL);
1037 assert3p(old_task->conclave, !=, NULL);
1038 assert3p(resource, ==, old_task->conclave);
1039
1040 /* Only allow inheriting the conclave if it has not yet started. */
1041 if (conclave->c_state != CONCLAVE_S_ATTACHED ||
1042 conclave->c_active_downcall ||
1043 conclave->c_active_stopcall) {
1044 lck_mtx_unlock(&resource->r_mutex);
1045 return KERN_FAILURE;
1046 }
1047
1048 old_task->conclave = NULL;
1049
1050 task_reference(new_task);
1051 new_task->conclave = resource;
1052
1053 conclave->c_task = new_task;
1054
1055 lck_mtx_unlock(&resource->r_mutex);
1056 task_deallocate(old_task);
1057
1058 return KERN_SUCCESS;
1059 }
1060
1061 bool
exclaves_conclave_is_attached(const exclaves_resource_t * resource)1062 exclaves_conclave_is_attached(const exclaves_resource_t *resource)
1063 {
1064 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1065 const conclave_resource_t *conclave = &resource->r_conclave;
1066
1067 return conclave->c_state == CONCLAVE_S_ATTACHED;
1068 }
1069
1070 kern_return_t
exclaves_conclave_launch(exclaves_resource_t * resource)1071 exclaves_conclave_launch(exclaves_resource_t *resource)
1072 {
1073 kern_return_t kr;
1074 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1075
1076 conclave_resource_t *conclave = &resource->r_conclave;
1077
1078 if (exclaves_boot_wait(EXCLAVES_BOOT_STAGE_EXCLAVEKIT) != KERN_SUCCESS) {
1079 /*
1080 * This should only ever happen if the EXCLAVEKIT requirement was
1081 * relaxed.
1082 */
1083 if (exclaves_requirement_is_relaxed(EXCLAVES_R_EXCLAVEKIT) ||
1084 exclaves_requirement_is_relaxed(EXCLAVES_R_FRAMEBANK)) {
1085 exclaves_debug_printf(show_errors,
1086 "exclaves: requirement was relaxed, ignoring error: failed to boot to exclavekit\n");
1087 } else {
1088 panic("exclaves: requirement failed: failed to boot to exclavekit\n");
1089 }
1090 return KERN_NOT_SUPPORTED;
1091 }
1092
1093 lck_mtx_lock(&resource->r_mutex);
1094
1095 if (conclave->c_state != CONCLAVE_S_ATTACHED ||
1096 conclave->c_active_downcall ||
1097 conclave->c_active_stopcall) {
1098 lck_mtx_unlock(&resource->r_mutex);
1099 return KERN_FAILURE;
1100 }
1101
1102 conclave->c_request |= CONCLAVE_R_LAUNCH_REQUESTED;
1103 kr = exclaves_update_state_machine_locked(resource);
1104 return kr;
1105 }
1106
1107 bool
exclaves_is_forwarding_resource(exclaves_resource_t * resource)1108 exclaves_is_forwarding_resource(exclaves_resource_t *resource)
1109 {
1110 if (resource->r_type == XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER ||
1111 resource->r_type == XNUPROXY_RESOURCETYPE_SERVICE) {
1112 if (resource->r_id > EXCLAVES_FORWARDING_RESOURCE_ID_BASE) {
1113 return true;
1114 }
1115 }
1116 return false;
1117 }
1118
1119 void
exclaves_conclave_prepare_teardown(task_t task __unused)1120 exclaves_conclave_prepare_teardown(task_t task __unused)
1121 {
1122 /* We explicitly do not handle HAS_ARM_FEAT_SME here because it's always
1123 * handled on exclaves_enter
1124 */
1125 }
1126
1127 static kern_return_t
exclaves_update_state_machine_locked(exclaves_resource_t * resource)1128 exclaves_update_state_machine_locked(exclaves_resource_t *resource)
1129 {
1130 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1131
1132 conclave_resource_t *conclave = &resource->r_conclave;
1133 conclave_state_t pending_state = CONCLAVE_S_NONE;
1134 kern_return_t ret;
1135
1136 while (1) {
1137 bool stop_call = false;
1138 /* Check if there are pending requests */
1139 if (conclave->c_request & CONCLAVE_R_LAUNCH_REQUESTED) {
1140 conclave->c_request &= ~CONCLAVE_R_LAUNCH_REQUESTED;
1141 assert3u(conclave->c_active_downcall, ==, 0);
1142 conclave->c_active_downcall = true;
1143 conclave->c_downcall_thread = current_thread();
1144 pending_state = CONCLAVE_S_RUNNING;
1145 lck_mtx_unlock(&resource->r_mutex);
1146
1147 ret = exclaves_conclave_launcher_launch(conclave->c_control);
1148 assert3u(ret, ==, KERN_SUCCESS);
1149 } else if (conclave->c_request & CONCLAVE_R_SUSPEND_REQUESTED) {
1150 task_t task = conclave->c_task;
1151 int suspend_count;
1152 bool suspend;
1153
1154 task_lock(task);
1155 suspend_count = task->suspend_count;
1156 task_unlock(task);
1157
1158 suspend = (suspend_count > 0) ? true : false;
1159 conclave->c_request &= ~CONCLAVE_R_SUSPEND_REQUESTED;
1160
1161 /* Check the state to see if downcall is needed */
1162 if (suspend && conclave->c_state != CONCLAVE_S_RUNNING) {
1163 continue;
1164 }
1165
1166 if (!suspend && conclave->c_state != CONCLAVE_S_SUSPENDED) {
1167 continue;
1168 }
1169
1170 assert3u(conclave->c_active_downcall, ==, 0);
1171 conclave->c_active_downcall = true;
1172 conclave->c_downcall_thread = current_thread();
1173 pending_state = suspend ? CONCLAVE_S_SUSPENDED : CONCLAVE_S_RUNNING;
1174 lck_mtx_unlock(&resource->r_mutex);
1175
1176 ret = exclaves_conclave_launcher_suspend(conclave->c_control,
1177 suspend);
1178 } else if (conclave->c_request & CONCLAVE_R_STOP_REQUESTED) {
1179 conclave->c_request &= ~CONCLAVE_R_STOP_REQUESTED;
1180
1181 /* Check the state to see if downcall is needed */
1182 if (conclave->c_state != CONCLAVE_S_RUNNING &&
1183 conclave->c_state != CONCLAVE_S_SUSPENDED) {
1184 continue;
1185 }
1186 assert3u(conclave->c_active_downcall, ==, 0);
1187 conclave->c_active_downcall = true;
1188 conclave->c_downcall_thread = current_thread();
1189 conclave->c_active_stopcall = true;
1190 stop_call = true;
1191 pending_state = CONCLAVE_S_STOPPED;
1192 lck_mtx_unlock(&resource->r_mutex);
1193
1194 ret = exclaves_conclave_launcher_stop(conclave->c_control,
1195 CONCLAVE_LAUNCHER_CONCLAVESTOPREASON_EXIT);
1196 assert3u(ret, ==, KERN_SUCCESS);
1197 } else {
1198 lck_mtx_unlock(&resource->r_mutex);
1199 break;
1200 }
1201
1202 lck_mtx_lock(&resource->r_mutex);
1203 assert3u(conclave->c_active_downcall, ==, 1);
1204 assert3p(conclave->c_downcall_thread, ==, current_thread());
1205 conclave->c_active_downcall = false;
1206 conclave->c_downcall_thread = THREAD_NULL;
1207 if (stop_call) {
1208 conclave->c_active_stopcall = false;
1209 }
1210 if (conclave->c_active_detach) {
1211 wakeup_all_with_inheritor((event_t)&conclave->c_active_downcall, THREAD_AWAKENED);
1212 }
1213
1214 /* Bail out if active stopcall is going on */
1215 if (conclave->c_active_stopcall || conclave->c_state == CONCLAVE_S_STOPPED) {
1216 lck_mtx_unlock(&resource->r_mutex);
1217 break;
1218 }
1219
1220 conclave->c_state = pending_state;
1221 }
1222 return KERN_SUCCESS;
1223 }
1224
1225 /*
1226 * Return the domain associated with the current conclave.
1227 * If not joined to a conclave, return the KERNEL domain. This implies that the
1228 * calling task is sufficiently privileged.
1229 */
1230 const char *
exclaves_conclave_get_domain(exclaves_resource_t * resource)1231 exclaves_conclave_get_domain(exclaves_resource_t *resource)
1232 {
1233 if (resource != NULL) {
1234 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1235 return resource->r_name;
1236 }
1237
1238 if (!exclaves_has_priv(current_task(), EXCLAVES_PRIV_KERNEL_DOMAIN)) {
1239 exclaves_requirement_assert(EXCLAVES_R_CONCLAVE_RESOURCES,
1240 "no conclave manager present");
1241 }
1242
1243 return EXCLAVES_DOMAIN_KERNEL;
1244 }
1245
1246 kern_return_t
exclaves_conclave_stop(exclaves_resource_t * resource,bool gather_crash_bt __unused)1247 exclaves_conclave_stop(exclaves_resource_t *resource, bool gather_crash_bt __unused)
1248 {
1249 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1250
1251 conclave_resource_t *conclave = &resource->r_conclave;
1252
1253 lck_mtx_lock(&resource->r_mutex);
1254
1255 /* Bailout if active stopcall in progress */
1256 if (conclave->c_active_stopcall || conclave->c_state == CONCLAVE_S_STOPPED) {
1257 lck_mtx_unlock(&resource->r_mutex);
1258 return KERN_SUCCESS;
1259 }
1260
1261 /* Arm stop requested if downcall in progress */
1262 if (conclave->c_active_downcall) {
1263 conclave->c_request |= CONCLAVE_R_STOP_REQUESTED;
1264 lck_mtx_unlock(&resource->r_mutex);
1265 return KERN_SUCCESS;
1266 }
1267
1268 if (conclave->c_state == CONCLAVE_S_ATTACHED) {
1269 /* Change the state to stopped if the conclave was never started */
1270 conclave->c_state = CONCLAVE_S_STOPPED;
1271
1272 /* Suspend might be requested, clear it as well */
1273 conclave->c_request = CONCLAVE_R_NONE;
1274 lck_mtx_unlock(&resource->r_mutex);
1275 return KERN_SUCCESS;
1276 }
1277
1278 conclave->c_request |= CONCLAVE_R_STOP_REQUESTED;
1279 kern_return_t kr = exclaves_update_state_machine_locked(resource);
1280
1281 return kr;
1282 }
1283
1284 kern_return_t
exclaves_conclave_suspend(exclaves_resource_t * resource)1285 exclaves_conclave_suspend(exclaves_resource_t *resource)
1286 {
1287 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1288
1289 conclave_resource_t *conclave = &resource->r_conclave;
1290
1291 lck_mtx_lock(&resource->r_mutex);
1292
1293 /* Bailout if active stopcall in progress */
1294 if (conclave->c_active_stopcall || conclave->c_state == CONCLAVE_S_STOPPED) {
1295 lck_mtx_unlock(&resource->r_mutex);
1296 return KERN_SUCCESS;
1297 }
1298
1299 /* Arm suspend requested if downcall in progress */
1300 if (conclave->c_active_downcall) {
1301 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1302 lck_mtx_unlock(&resource->r_mutex);
1303 return KERN_SUCCESS;
1304 }
1305
1306 if (conclave->c_state == CONCLAVE_S_ATTACHED) {
1307 /* Conclave is not yet launched, just arm suspend requested and bailout */
1308 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1309 lck_mtx_unlock(&resource->r_mutex);
1310 return KERN_SUCCESS;
1311 } else if (conclave->c_state == CONCLAVE_S_SUSPENDED) {
1312 lck_mtx_unlock(&resource->r_mutex);
1313 return KERN_SUCCESS;
1314 }
1315
1316 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1317 kern_return_t kr = exclaves_update_state_machine_locked(resource);
1318
1319 return kr;
1320 }
1321
1322 kern_return_t
exclaves_conclave_resume(exclaves_resource_t * resource)1323 exclaves_conclave_resume(exclaves_resource_t *resource)
1324 {
1325 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1326
1327 conclave_resource_t *conclave = &resource->r_conclave;
1328
1329 lck_mtx_lock(&resource->r_mutex);
1330
1331 /* Bailout if active stopcall in progress */
1332 if (conclave->c_active_stopcall || conclave->c_state == CONCLAVE_S_STOPPED) {
1333 lck_mtx_unlock(&resource->r_mutex);
1334 return KERN_SUCCESS;
1335 }
1336
1337 /* Arm suspend requested if downcall in progress */
1338 if (conclave->c_active_downcall) {
1339 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1340 lck_mtx_unlock(&resource->r_mutex);
1341 return KERN_SUCCESS;
1342 }
1343
1344 if (conclave->c_state == CONCLAVE_S_ATTACHED) {
1345 /* Conclave is not yet launched, just arm suspend requested and bailout */
1346 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1347 lck_mtx_unlock(&resource->r_mutex);
1348 return KERN_SUCCESS;
1349 } else if (conclave->c_state == CONCLAVE_S_RUNNING) {
1350 lck_mtx_unlock(&resource->r_mutex);
1351 return KERN_SUCCESS;
1352 }
1353
1354 conclave->c_request |= CONCLAVE_R_SUSPEND_REQUESTED;
1355 kern_return_t kr = exclaves_update_state_machine_locked(resource);
1356
1357 return kr;
1358 }
1359
1360 kern_return_t
exclaves_conclave_stop_upcall(exclaves_resource_t * resource)1361 exclaves_conclave_stop_upcall(exclaves_resource_t *resource)
1362 {
1363 assert3p(resource, !=, NULL);
1364 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1365
1366 conclave_resource_t *conclave = &resource->r_conclave;
1367 thread_t thread = current_thread();
1368
1369 lck_mtx_lock(&resource->r_mutex);
1370
1371 if (conclave->c_state == CONCLAVE_S_STOPPED || conclave->c_active_stopcall) {
1372 lck_mtx_unlock(&resource->r_mutex);
1373 return KERN_SUCCESS;
1374 }
1375
1376 conclave->c_active_stopcall = true;
1377 thread->th_exclaves_state |= TH_EXCLAVES_STOP_UPCALL_PENDING;
1378 lck_mtx_unlock(&resource->r_mutex);
1379
1380 return KERN_SUCCESS;
1381 }
1382
1383 kern_return_t
exclaves_conclave_stop_upcall_complete(exclaves_resource_t * resource,task_t task)1384 exclaves_conclave_stop_upcall_complete(exclaves_resource_t *resource, task_t task)
1385 {
1386 assert3p(resource, !=, NULL);
1387 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1388
1389 conclave_resource_t *conclave = &resource->r_conclave;
1390 thread_t thread = current_thread();
1391
1392 thread->th_exclaves_state &= ~TH_EXCLAVES_STOP_UPCALL_PENDING;
1393
1394 int flags = PX_DEBUG_NO_HONOR | PX_NO_EXCEPTION_UTHREAD;
1395 exception_info_t info = {
1396 .os_reason = OS_REASON_GUARD,
1397 .exception_type = EXC_GUARD,
1398 .mx_code = GUARD_REASON_EXCLAVES,
1399 .mx_subcode = 0
1400 };
1401
1402 exit_with_exclave_exception(get_bsdtask_info(task), info, flags);
1403
1404 lck_mtx_lock(&resource->r_mutex);
1405
1406 conclave->c_active_stopcall = false;
1407 conclave->c_state = CONCLAVE_S_STOPPED;
1408 conclave->c_request = CONCLAVE_R_NONE;
1409
1410 lck_mtx_unlock(&resource->r_mutex);
1411 return KERN_SUCCESS;
1412 }
1413
1414 bool
exclaves_conclave_has_service(exclaves_resource_t * resource,uint64_t id)1415 exclaves_conclave_has_service(exclaves_resource_t *resource, uint64_t id)
1416 {
1417 assert3u(id, <, CONCLAVE_SERVICE_MAX);
1418
1419 if (resource == NULL) {
1420 /* There's no conclave, fallback to the kernel domain. */
1421 if (!exclaves_has_priv(current_task(), EXCLAVES_PRIV_KERNEL_DOMAIN)) {
1422 exclaves_requirement_assert(EXCLAVES_R_CONCLAVE_RESOURCES,
1423 "no conclave manager present");
1424 }
1425 return bitmap_test(kernel_service_bitmap, (uint32_t)id);
1426 }
1427
1428 assert3p(resource, !=, NULL);
1429 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_CONCLAVEMANAGER);
1430
1431 conclave_resource_t *conclave = &resource->r_conclave;
1432
1433 return bitmap_test(conclave->c_service_bitmap, (uint32_t)id);
1434 }
1435
1436 /* -------------------------------------------------------------------------- */
1437 #pragma mark Sensors
1438
1439 static void
exclaves_resource_sensor_reset(exclaves_resource_t * resource)1440 exclaves_resource_sensor_reset(exclaves_resource_t *resource)
1441 {
1442 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SENSOR);
1443 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), ==, 0);
1444 LCK_MTX_ASSERT(&resource->r_mutex, LCK_MTX_ASSERT_OWNED);
1445
1446 exclaves_sensor_status_t status;
1447
1448 for (int i = 0; i < resource->r_sensor.s_startcount; i++) {
1449 __assert_only kern_return_t kr = exclaves_sensor_stop(
1450 (exclaves_sensor_type_t)resource->r_id, 0, &status);
1451 assert3u(kr, !=, KERN_INVALID_ARGUMENT);
1452 }
1453
1454 resource->r_sensor.s_startcount = 0;
1455 }
1456
1457 kern_return_t
exclaves_resource_sensor_open(const char * domain,const char * id_name,exclaves_resource_t ** out)1458 exclaves_resource_sensor_open(const char *domain, const char *id_name,
1459 exclaves_resource_t **out)
1460 {
1461 assert3p(out, !=, NULL);
1462
1463 exclaves_resource_t *sensor = exclaves_resource_lookup_by_name(domain,
1464 id_name, XNUPROXY_RESOURCETYPE_SENSOR);
1465
1466 if (sensor == NULL) {
1467 return KERN_NOT_FOUND;
1468 }
1469
1470 assert3u(sensor->r_type, ==, XNUPROXY_RESOURCETYPE_SENSOR);
1471
1472 lck_mtx_lock(&sensor->r_mutex);
1473 exclaves_resource_retain(sensor);
1474 lck_mtx_unlock(&sensor->r_mutex);
1475
1476 *out = sensor;
1477
1478 return KERN_SUCCESS;
1479 }
1480
1481 kern_return_t
exclaves_resource_sensor_start(exclaves_resource_t * resource,uint64_t flags,exclaves_sensor_status_t * status)1482 exclaves_resource_sensor_start(exclaves_resource_t *resource, uint64_t flags,
1483 exclaves_sensor_status_t *status)
1484 {
1485 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SENSOR);
1486
1487 lck_mtx_lock(&resource->r_mutex);
1488 if (resource->r_sensor.s_startcount == UINT64_MAX) {
1489 lck_mtx_unlock(&resource->r_mutex);
1490 return KERN_INVALID_ARGUMENT;
1491 }
1492
1493 kern_return_t kr = exclaves_sensor_start(
1494 (exclaves_sensor_type_t)resource->r_id, flags, status);
1495 if (kr == KERN_SUCCESS) {
1496 resource->r_sensor.s_startcount += 1;
1497 }
1498 lck_mtx_unlock(&resource->r_mutex);
1499 return kr;
1500 }
1501
1502 kern_return_t
exclaves_resource_sensor_status(exclaves_resource_t * resource,uint64_t flags,exclaves_sensor_status_t * status)1503 exclaves_resource_sensor_status(exclaves_resource_t *resource, uint64_t flags,
1504 exclaves_sensor_status_t *status)
1505 {
1506 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SENSOR);
1507
1508 lck_mtx_lock(&resource->r_mutex);
1509 kern_return_t kr = exclaves_sensor_status(
1510 (exclaves_sensor_type_t)resource->r_id, flags, status);
1511 lck_mtx_unlock(&resource->r_mutex);
1512
1513 return kr;
1514 }
1515
1516 kern_return_t
exclaves_resource_sensor_stop(exclaves_resource_t * resource,uint64_t flags,exclaves_sensor_status_t * status)1517 exclaves_resource_sensor_stop(exclaves_resource_t *resource, uint64_t flags,
1518 exclaves_sensor_status_t *status)
1519 {
1520 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SENSOR);
1521
1522 lck_mtx_lock(&resource->r_mutex);
1523 if (resource->r_sensor.s_startcount == 0) {
1524 lck_mtx_unlock(&resource->r_mutex);
1525 return KERN_INVALID_ARGUMENT;
1526 }
1527
1528 kern_return_t kr = exclaves_sensor_stop(
1529 (exclaves_sensor_type_t)resource->r_id, flags, status);
1530 if (kr == KERN_SUCCESS) {
1531 resource->r_sensor.s_startcount -= 1;
1532 }
1533 lck_mtx_unlock(&resource->r_mutex);
1534
1535 return kr;
1536 }
1537
1538 /* -------------------------------------------------------------------------- */
1539 #pragma mark Notifications
1540
1541 static void
exclaves_notification_init(exclaves_resource_t * resource)1542 exclaves_notification_init(exclaves_resource_t *resource)
1543 {
1544 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1545 exclaves_notification_t *notification = &resource->r_notification;
1546 klist_init(¬ification->notification_klist);
1547 }
1548
1549 static int
filt_exclaves_notification_attach(struct knote * kn,__unused struct kevent_qos_s * kev)1550 filt_exclaves_notification_attach(struct knote *kn, __unused struct kevent_qos_s *kev)
1551 {
1552 int error = 0;
1553 exclaves_resource_t *exclaves_resource = NULL;
1554 kern_return_t kr = exclaves_resource_from_port_name(current_space(), (mach_port_name_t)kn->kn_id, &exclaves_resource);
1555 if (kr != KERN_SUCCESS) {
1556 error = ENOENT;
1557 goto out;
1558 }
1559 assert3p(exclaves_resource, !=, NULL);
1560 if (exclaves_resource->r_type != XNUPROXY_RESOURCETYPE_NOTIFICATION) {
1561 exclaves_resource_release(exclaves_resource);
1562 error = EINVAL;
1563 goto out;
1564 }
1565
1566 lck_mtx_lock(&exclaves_resource->r_mutex);
1567
1568 if (kn->kn_exclaves_resource != NULL) {
1569 lck_mtx_unlock(&exclaves_resource->r_mutex);
1570 exclaves_resource_release(exclaves_resource);
1571 error = EBUSY;
1572 goto out;
1573 }
1574
1575 /* kn_exclaves_resource consumes the ref. */
1576 kn->kn_exclaves_resource = exclaves_resource;
1577 KNOTE_ATTACH(&exclaves_resource->r_notification.notification_klist, kn);
1578 lck_mtx_unlock(&exclaves_resource->r_mutex);
1579
1580 error = 0;
1581 out:
1582 return error;
1583 }
1584
1585 static void
filt_exclaves_notification_detach(struct knote * kn)1586 filt_exclaves_notification_detach(struct knote *kn)
1587 {
1588 exclaves_resource_t *exclaves_resource = kn->kn_exclaves_resource;
1589
1590 if (exclaves_resource != NULL) {
1591 assert3u(exclaves_resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1592 lck_mtx_lock(&exclaves_resource->r_mutex);
1593 kn->kn_exclaves_resource = NULL;
1594 KNOTE_DETACH(&exclaves_resource->r_notification.notification_klist, kn);
1595 lck_mtx_unlock(&exclaves_resource->r_mutex);
1596
1597 exclaves_resource_release(exclaves_resource);
1598 }
1599 }
1600
1601 static int
filt_exclaves_notification_event(struct knote * kn,long hint)1602 filt_exclaves_notification_event(struct knote *kn, long hint)
1603 {
1604 /* ALWAYS CALLED WITH exclaves_resource mutex held */
1605 exclaves_resource_t *exclaves_resource __assert_only = kn->kn_exclaves_resource;
1606 LCK_MTX_ASSERT(&exclaves_resource->r_mutex, LCK_MTX_ASSERT_OWNED);
1607
1608 /*
1609 * if the user is interested in this event, record it.
1610 */
1611 if (kn->kn_sfflags & hint) {
1612 kn->kn_fflags |= hint;
1613 }
1614
1615 /* if we have any matching state, activate the knote */
1616 if (kn->kn_fflags != 0) {
1617 return FILTER_ACTIVE;
1618 } else {
1619 return 0;
1620 }
1621 }
1622
1623 static int
filt_exclaves_notification_touch(struct knote * kn,struct kevent_qos_s * kev)1624 filt_exclaves_notification_touch(struct knote *kn, struct kevent_qos_s *kev)
1625 {
1626 int result;
1627 exclaves_resource_t *exclaves_resource = kn->kn_exclaves_resource;
1628 assert3p(exclaves_resource, !=, NULL);
1629 assert3u(exclaves_resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1630
1631 lck_mtx_lock(&exclaves_resource->r_mutex);
1632 /* accept new mask and mask off output events no long interesting */
1633 kn->kn_sfflags = kev->fflags;
1634 kn->kn_fflags &= kn->kn_sfflags;
1635 if (kn->kn_fflags != 0) {
1636 result = FILTER_ACTIVE;
1637 } else {
1638 result = 0;
1639 }
1640 lck_mtx_unlock(&exclaves_resource->r_mutex);
1641
1642 return result;
1643 }
1644
1645 static int
filt_exclaves_notification_process(struct knote * kn,struct kevent_qos_s * kev)1646 filt_exclaves_notification_process(struct knote *kn, struct kevent_qos_s *kev)
1647 {
1648 int result = 0;
1649 exclaves_resource_t *exclaves_resource = kn->kn_exclaves_resource;
1650 assert3p(exclaves_resource, !=, NULL);
1651 assert3u(exclaves_resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1652
1653 lck_mtx_lock(&exclaves_resource->r_mutex);
1654 if (kn->kn_fflags) {
1655 knote_fill_kevent(kn, kev, 0);
1656 result = FILTER_ACTIVE;
1657 }
1658 lck_mtx_unlock(&exclaves_resource->r_mutex);
1659 return result;
1660 }
1661
1662 SECURITY_READ_ONLY_EARLY(struct filterops) exclaves_notification_filtops = {
1663 .f_attach = filt_exclaves_notification_attach,
1664 .f_detach = filt_exclaves_notification_detach,
1665 .f_event = filt_exclaves_notification_event,
1666 .f_touch = filt_exclaves_notification_touch,
1667 .f_process = filt_exclaves_notification_process,
1668 };
1669
1670 kern_return_t
exclaves_notification_create(const char * domain,const char * name,exclaves_resource_t ** out)1671 exclaves_notification_create(const char *domain, const char *name,
1672 exclaves_resource_t **out)
1673 {
1674 assert3p(out, !=, NULL);
1675
1676 exclaves_resource_t *resource = exclaves_resource_lookup_by_name(domain,
1677 name, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1678
1679 if (resource == NULL) {
1680 return KERN_NOT_FOUND;
1681 }
1682 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1683
1684 lck_mtx_lock(&resource->r_mutex);
1685 exclaves_resource_retain(resource);
1686 lck_mtx_unlock(&resource->r_mutex);
1687
1688 *out = resource;
1689
1690 return KERN_SUCCESS;
1691 }
1692
1693 kern_return_t
exclaves_notification_signal(exclaves_resource_t * exclaves_resource,long event_mask)1694 exclaves_notification_signal(exclaves_resource_t *exclaves_resource, long event_mask)
1695 {
1696 assert3p(exclaves_resource, !=, NULL);
1697 assert3u(exclaves_resource->r_type, ==, XNUPROXY_RESOURCETYPE_NOTIFICATION);
1698
1699 lck_mtx_lock(&exclaves_resource->r_mutex);
1700 KNOTE(&exclaves_resource->r_notification.notification_klist, event_mask);
1701 lck_mtx_unlock(&exclaves_resource->r_mutex);
1702
1703 return KERN_SUCCESS;
1704 }
1705
1706 exclaves_resource_t *
exclaves_notification_lookup_by_id(uint64_t id)1707 exclaves_notification_lookup_by_id(uint64_t id)
1708 {
1709 return exclaves_resource_lookup_by_id(EXCLAVES_DOMAIN_KERNEL, id,
1710 XNUPROXY_RESOURCETYPE_NOTIFICATION);
1711 }
1712
1713 uint64_t
exclaves_service_lookup(const char * domain,const char * name)1714 exclaves_service_lookup(const char *domain, const char *name)
1715 {
1716 assert3p(domain, !=, NULL);
1717 assert3p(name, !=, NULL);
1718
1719 exclaves_resource_t *resource = exclaves_resource_lookup_by_name(domain,
1720 name, XNUPROXY_RESOURCETYPE_SERVICE);
1721 if (resource == NULL) {
1722 return EXCLAVES_INVALID_ID;
1723 }
1724
1725 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SERVICE);
1726 return resource->r_id;
1727 }
1728
1729 /* -------------------------------------------------------------------------- */
1730 #pragma mark Shared Memory
1731
1732 kern_return_t
exclaves_resource_shared_memory_copyin(exclaves_resource_t * resource,user_addr_t buffer,mach_vm_size_t size1,mach_vm_size_t offset1,mach_vm_size_t size2,mach_vm_size_t offset2)1733 exclaves_resource_shared_memory_copyin(exclaves_resource_t *resource,
1734 user_addr_t buffer, mach_vm_size_t size1, mach_vm_size_t offset1,
1735 mach_vm_size_t size2, mach_vm_size_t offset2)
1736 {
1737 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), >, 0);
1738 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SHAREDMEMORY);
1739
1740 mach_vm_size_t umax = 0;
1741
1742 if (buffer == USER_ADDR_NULL || size1 == 0) {
1743 return KERN_INVALID_ARGUMENT;
1744 }
1745
1746 shared_memory_resource_t *sm = &resource->r_shared_memory;
1747 assert3p(sm->sm_addr, !=, NULL);
1748 assert3u(sm->sm_size, !=, 0);
1749
1750 if (os_add_overflow(offset1, size1, &umax) || umax > sm->sm_size) {
1751 return KERN_INVALID_ARGUMENT;
1752 }
1753
1754 if (os_add_overflow(offset2, size2, &umax) || umax > sm->sm_size) {
1755 return KERN_INVALID_ARGUMENT;
1756 }
1757
1758 if ((sm->sm_perm & EXCLAVES_BUFFER_PERM_WRITE) == 0) {
1759 return KERN_PROTECTION_FAILURE;
1760 }
1761
1762 if (copyin(buffer, sm->sm_addr + offset1, size1) != 0) {
1763 return KERN_FAILURE;
1764 }
1765
1766 if (copyin(buffer + size1, sm->sm_addr + offset2, size2) != 0) {
1767 return KERN_FAILURE;
1768 }
1769
1770 return KERN_SUCCESS;
1771 }
1772
1773 kern_return_t
exclaves_resource_shared_memory_copyout(exclaves_resource_t * resource,user_addr_t buffer,mach_vm_size_t size1,mach_vm_size_t offset1,mach_vm_size_t size2,mach_vm_size_t offset2)1774 exclaves_resource_shared_memory_copyout(exclaves_resource_t *resource,
1775 user_addr_t buffer, mach_vm_size_t size1, mach_vm_size_t offset1,
1776 mach_vm_size_t size2, mach_vm_size_t offset2)
1777 {
1778 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), >, 0);
1779 assert(resource->r_type == XNUPROXY_RESOURCETYPE_SHAREDMEMORY ||
1780 resource->r_type == XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
1781
1782 mach_vm_size_t umax = 0;
1783
1784 if (buffer == USER_ADDR_NULL || size1 == 0) {
1785 return KERN_INVALID_ARGUMENT;
1786 }
1787
1788 shared_memory_resource_t *sm = &resource->r_shared_memory;
1789 assert3p(sm->sm_addr, !=, NULL);
1790 assert3u(sm->sm_size, !=, 0);
1791
1792 if (os_add_overflow(offset1, size1, &umax) || umax > sm->sm_size) {
1793 return KERN_INVALID_ARGUMENT;
1794 }
1795
1796 if (os_add_overflow(offset2, size2, &umax) || umax > sm->sm_size) {
1797 return KERN_INVALID_ARGUMENT;
1798 }
1799
1800 if ((sm->sm_perm & EXCLAVES_BUFFER_PERM_READ) == 0) {
1801 return KERN_PROTECTION_FAILURE;
1802 }
1803
1804 if (copyout(sm->sm_addr + offset1, buffer, size1) != 0) {
1805 return KERN_FAILURE;
1806 }
1807
1808 if (copyout(sm->sm_addr + offset2, buffer + size1, size2) != 0) {
1809 return KERN_FAILURE;
1810 }
1811
1812 return KERN_SUCCESS;
1813 }
1814
1815 /* The lower 32bits contain the endpoint id. */
1816 static uint32_t
audio_memory_get_endpoint(exclaves_resource_t * resource)1817 audio_memory_get_endpoint(exclaves_resource_t *resource)
1818 {
1819 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
1820 return resource->r_id << 32 >> 32;
1821 }
1822
1823 /* The upper 32bits of the id contain the buffer id. */
1824 static uint32_t
audio_memory_get_buffer_id(exclaves_resource_t * resource)1825 audio_memory_get_buffer_id(exclaves_resource_t *resource)
1826 {
1827 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
1828 return resource->r_id >> 32;
1829 }
1830
1831 static kern_return_t
shared_memory_map(exclaves_resource_t * resource,size_t size,exclaves_buffer_perm_t perm)1832 shared_memory_map(exclaves_resource_t *resource, size_t size,
1833 exclaves_buffer_perm_t perm)
1834 {
1835 assert(resource->r_type == XNUPROXY_RESOURCETYPE_SHAREDMEMORY ||
1836 resource->r_type == XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
1837
1838 /*
1839 * It is expected that shared memory is either write-only or read-only.
1840 * This is enforced through the userspace APIs (inbound or outbound buffers
1841 * respectively).
1842 */
1843 assert(perm == EXCLAVES_BUFFER_PERM_READ ||
1844 perm == EXCLAVES_BUFFER_PERM_WRITE);
1845
1846 kern_return_t kr = KERN_FAILURE;
1847
1848 /* round size up to nearest page */
1849 mach_vm_offset_t rounded_size = 0;
1850 if (size == 0 || mach_vm_round_page_overflow(size, &rounded_size)) {
1851 return KERN_INVALID_ARGUMENT;
1852 }
1853 const size_t page_count = rounded_size / PAGE_SIZE;
1854
1855 lck_mtx_lock(&resource->r_mutex);
1856
1857 __block shared_memory_resource_t *sm = &resource->r_shared_memory;
1858
1859 /*
1860 * If already active, bump the use count, check that the perms and size
1861 * are compatible and return. Checking the use count is insufficient
1862 * here as this can race with with a non-locked use count release.
1863 */
1864 if (resource->r_active) {
1865 /*
1866 * Both the permissions and size must match.
1867 */
1868 if (sm->sm_size < rounded_size || sm->sm_perm != perm) {
1869 lck_mtx_unlock(&resource->r_mutex);
1870 return KERN_INVALID_ARGUMENT;
1871 }
1872
1873 exclaves_resource_retain(resource);
1874 lck_mtx_unlock(&resource->r_mutex);
1875 return KERN_SUCCESS;
1876 }
1877
1878 /* This is lazily initialised and never de-initialised. */
1879 if (sm->sm_client.connection == NULL) {
1880 uint64_t endpoint = resource->r_type == XNUPROXY_RESOURCETYPE_SHAREDMEMORY ?
1881 resource->r_id :
1882 audio_memory_get_endpoint(resource);
1883
1884 kr = exclaves_shared_memory_init(endpoint, &sm->sm_client);
1885 if (kr != KERN_SUCCESS) {
1886 lck_mtx_unlock(&resource->r_mutex);
1887 return kr;
1888 }
1889 }
1890
1891 const sharedmemorybase_perms_s sm_perm = perm == EXCLAVES_BUFFER_PERM_WRITE ?
1892 SHAREDMEMORYBASE_PERMS_READWRITE : SHAREDMEMORYBASE_PERMS_READONLY;
1893 sharedmemorybase_mapping_s mapping = 0;
1894 kr = exclaves_shared_memory_setup(&sm->sm_client, sm_perm, 0,
1895 page_count, &mapping);
1896 if (kr != KERN_SUCCESS) {
1897 lck_mtx_unlock(&resource->r_mutex);
1898 return kr;
1899 }
1900
1901 /*
1902 * From this point on exclaves_shared_memory_teardown() must be called
1903 * if something goes wrong so that the buffer will be properly unmapped.
1904 */
1905 sm->sm_size = rounded_size;
1906 sm->sm_perm = perm;
1907 sm->sm_addr = NULL;
1908
1909 /*
1910 * The shared buffer is now accessible by xnu. Discover the layout of
1911 * the memory and map it into the kernel.
1912 */
1913 uint32_t *pages = kalloc_type(uint32_t, page_count,
1914 Z_WAITOK | Z_ZERO | Z_NOFAIL);
1915 __block uint32_t idx = 0;
1916 /* BEGIN IGNORE CODESTYLE */
1917 kr = exclaves_shared_memory_iterate(&sm->sm_client, &mapping, 0,
1918 page_count, ^(uint64_t pa) {
1919 assert3u(pa & PAGE_MASK, ==, 0);
1920 assert3u(idx, <, page_count);
1921
1922 pages[idx++] = (uint32_t)atop(pa);
1923 });
1924 /* END IGNORE CODESTYLE */
1925
1926 if (kr != KERN_SUCCESS) {
1927 kfree_type(uint32_t, page_count, pages);
1928 exclaves_shared_memory_teardown(&sm->sm_client, &mapping);
1929 lck_mtx_unlock(&resource->r_mutex);
1930 return KERN_FAILURE;
1931 }
1932
1933 assert3u(idx, ==, page_count);
1934
1935 const vm_prot_t prot = (perm & EXCLAVES_BUFFER_PERM_WRITE) != 0 ?
1936 VM_PROT_READ | VM_PROT_WRITE :
1937 VM_PROT_READ;
1938 kr = exclaves_memory_map((uint32_t)page_count, pages, prot, &sm->sm_addr);
1939 kfree_type(uint32_t, page_count, pages);
1940 if (kr != KERN_SUCCESS) {
1941 exclaves_shared_memory_teardown(&sm->sm_client, &mapping);
1942 lck_mtx_unlock(&resource->r_mutex);
1943 return KERN_FAILURE;
1944 }
1945
1946 sm->sm_mapping = mapping;
1947
1948 exclaves_resource_retain(resource);
1949 resource->r_active = true;
1950
1951 lck_mtx_unlock(&resource->r_mutex);
1952
1953 return KERN_SUCCESS;
1954 }
1955
1956 kern_return_t
exclaves_resource_shared_memory_map(const char * domain,const char * name,size_t size,exclaves_buffer_perm_t perm,exclaves_resource_t ** out)1957 exclaves_resource_shared_memory_map(const char *domain, const char *name, size_t size,
1958 exclaves_buffer_perm_t perm, exclaves_resource_t **out)
1959 {
1960 assert3p(out, !=, NULL);
1961
1962 exclaves_resource_t *resource = exclaves_resource_lookup_by_name(domain,
1963 name, XNUPROXY_RESOURCETYPE_SHAREDMEMORY);
1964 if (resource == NULL) {
1965 return KERN_NOT_FOUND;
1966 }
1967 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_SHAREDMEMORY);
1968
1969 kern_return_t kr = shared_memory_map(resource, size, perm);
1970 if (kr != KERN_SUCCESS) {
1971 return kr;
1972 }
1973
1974 *out = resource;
1975 return KERN_SUCCESS;
1976 }
1977
1978
1979 static void
exclaves_resource_shared_memory_unmap(exclaves_resource_t * resource)1980 exclaves_resource_shared_memory_unmap(exclaves_resource_t *resource)
1981 {
1982 assert(resource->r_type == XNUPROXY_RESOURCETYPE_SHAREDMEMORY ||
1983 resource->r_type == XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
1984 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), ==, 0);
1985 LCK_MTX_ASSERT(&resource->r_mutex, LCK_MTX_ASSERT_OWNED);
1986
1987 shared_memory_resource_t *sm = &resource->r_shared_memory;
1988
1989 if (sm->sm_addr != NULL) {
1990 __assert_only kern_return_t kr =
1991 exclaves_memory_unmap(sm->sm_addr, sm->sm_size);
1992 assert3u(kr, ==, KERN_SUCCESS);
1993 sm->sm_addr = NULL;
1994 sm->sm_size = 0;
1995 }
1996
1997 kern_return_t kr = exclaves_shared_memory_teardown(&sm->sm_client,
1998 &sm->sm_mapping);
1999 if (kr != KERN_SUCCESS) {
2000 exclaves_debug_printf(show_errors,
2001 "exclaves: failed to teardown shared memory: %s, \n",
2002 resource->r_name);
2003 return;
2004 }
2005
2006 bzero(&resource->r_shared_memory, sizeof(resource->r_shared_memory));
2007
2008 resource->r_active = false;
2009 }
2010
2011 char *
exclaves_resource_shared_memory_get_buffer(exclaves_resource_t * resource,size_t * buffer_len)2012 exclaves_resource_shared_memory_get_buffer(exclaves_resource_t *resource,
2013 size_t *buffer_len)
2014 {
2015 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), >, 0);
2016 assert(resource->r_type == XNUPROXY_RESOURCETYPE_SHAREDMEMORY ||
2017 resource->r_type == XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
2018
2019 shared_memory_resource_t *sm = &resource->r_shared_memory;
2020 assert3p(sm->sm_addr, !=, NULL);
2021 assert3u(sm->sm_size, !=, 0);
2022
2023 if (buffer_len != NULL) {
2024 *buffer_len = sm->sm_size;
2025 }
2026
2027 return sm->sm_addr;
2028 }
2029
2030 /* -------------------------------------------------------------------------- */
2031 #pragma mark Arbitrated Audio Memory
2032
2033 kern_return_t
exclaves_resource_audio_memory_map(const char * domain,const char * name,size_t size,exclaves_resource_t ** out)2034 exclaves_resource_audio_memory_map(const char *domain, const char *name,
2035 size_t size, exclaves_resource_t **out)
2036 {
2037 assert3p(out, !=, NULL);
2038
2039 exclaves_resource_t *resource = exclaves_resource_lookup_by_name(domain,
2040 name, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
2041 if (resource == NULL) {
2042 return KERN_NOT_FOUND;
2043 }
2044 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
2045
2046 kern_return_t kr = shared_memory_map(resource, size,
2047 EXCLAVES_BUFFER_PERM_READ);
2048 if (kr != KERN_SUCCESS) {
2049 return kr;
2050 }
2051
2052 *out = resource;
2053 return KERN_SUCCESS;
2054 }
2055
2056 static void
exclaves_resource_audio_memory_unmap(exclaves_resource_t * resource)2057 exclaves_resource_audio_memory_unmap(exclaves_resource_t *resource)
2058 {
2059 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
2060 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), ==, 0);
2061 LCK_MTX_ASSERT(&resource->r_mutex, LCK_MTX_ASSERT_OWNED);
2062
2063 exclaves_resource_shared_memory_unmap(resource);
2064 }
2065
2066 kern_return_t
exclaves_resource_audio_memory_copyout(exclaves_resource_t * resource,user_addr_t buffer,mach_vm_size_t size1,mach_vm_size_t offset1,mach_vm_size_t size2,mach_vm_size_t offset2,user_addr_t ustatus)2067 exclaves_resource_audio_memory_copyout(exclaves_resource_t *resource,
2068 user_addr_t buffer, mach_vm_size_t size1, mach_vm_size_t offset1,
2069 mach_vm_size_t size2, mach_vm_size_t offset2, user_addr_t ustatus)
2070 {
2071 assert3u(os_atomic_load(&resource->r_usecnt, relaxed), >, 0);
2072 assert3u(resource->r_type, ==, XNUPROXY_RESOURCETYPE_ARBITRATEDAUDIOMEMORY);
2073
2074 kern_return_t kr = KERN_FAILURE;
2075 exclaves_sensor_status_t status;
2076 const uint32_t id = audio_memory_get_buffer_id(resource);
2077
2078 kr = exclaves_sensor_copy(id, size1, offset1, size2, offset2, &status);
2079 if (kr != KERN_SUCCESS) {
2080 return kr;
2081 }
2082
2083 kr = exclaves_resource_shared_memory_copyout(resource, buffer,
2084 size1, offset1, size2, offset2);
2085 if (kr != KERN_SUCCESS) {
2086 return kr;
2087 }
2088
2089 if (ustatus != 0 &&
2090 copyout(&status, ustatus, sizeof(status)) != 0) {
2091 return KERN_FAILURE;
2092 }
2093
2094 return KERN_SUCCESS;
2095 }
2096
2097 #pragma mark AOE Service
2098
2099 void
2100 exclaves_resource_aoeservice_iterate(const char *domain_name,
2101 bool (^cb)(exclaves_resource_t *))
2102 {
2103 assert3u(strlen(domain_name), >, 0);
2104
2105 exclaves_resource_domain_t *domain = lookup_domain(domain_name);
2106 if (domain == NULL) {
2107 return;
2108 }
2109
2110 iterate_resources(domain, ^(exclaves_resource_t *resource) {
2111 if (resource->r_type != XNUPROXY_RESOURCETYPE_ALWAYSONEXCLAVESSERVICE) {
2112 return (bool)false;
2113 }
2114
2115 return cb(resource);
2116 });
2117 }
2118
2119 #endif /* CONFIG_EXCLAVES */
2120