1 /*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if CONFIG_EXCLAVES
30
31 #include <vm/pmap.h>
32
33 #include <vm/vm_page_internal.h>
34 #include <vm/vm_object_xnu.h>
35 #include <vm/vm_pageout_xnu.h>
36 #include <vm/vm_kern_xnu.h>
37 #include <vm/vm_map_xnu.h>
38 #include <vm/vm_memory_entry_xnu.h>
39 #include <vm/vm_protos.h>
40
41 #include <mach/mach_vm.h>
42 #include <mach/mach_host.h>
43
44 #include <device/device_port.h>
45
46 #include <kern/ipc_kobject.h>
47
48 #include <libkern/coreanalytics/coreanalytics.h>
49 #include <kern/ledger.h>
50
51 #include "exclaves_memory.h"
52
53 /* -------------------------------------------------------------------------- */
54 #pragma mark Accounting
55
56 typedef struct {
57 _Atomic uint64_t pages_alloced;
58 _Atomic uint64_t pages_freed;
59 _Atomic uint64_t time_allocating;
60 _Atomic uint64_t max_alloc_latency;
61 _Atomic uint64_t alloc_latency_byhighbit[16];// highbit(MCT end - MCT start)/4
62 } exclaves_allocation_statistics_t;
63
64 exclaves_allocation_statistics_t exclaves_allocation_statistics;
65
66 CA_EVENT(ca_exclaves_allocation_statistics,
67 CA_INT, pages_alloced,
68 CA_INT, pages_freed,
69 CA_INT, time_allocating,
70 CA_INT, max_alloc_latency,
71 CA_INT, alloc_latency_highbit0,
72 CA_INT, alloc_latency_highbit1,
73 CA_INT, alloc_latency_highbit2,
74 CA_INT, alloc_latency_highbit3,
75 CA_INT, alloc_latency_highbit4,
76 CA_INT, alloc_latency_highbit5,
77 CA_INT, alloc_latency_highbit6,
78 CA_INT, alloc_latency_highbit7,
79 CA_INT, alloc_latency_highbit8,
80 CA_INT, alloc_latency_highbit9,
81 CA_INT, alloc_latency_highbit10,
82 CA_INT, alloc_latency_highbit11,
83 CA_INT, alloc_latency_highbit12,
84 CA_INT, alloc_latency_highbit13,
85 CA_INT, alloc_latency_highbit14,
86 CA_INT, alloc_latency_highbit15);
87
88 void
exclaves_memory_report_accounting(void)89 exclaves_memory_report_accounting(void)
90 {
91 ca_event_t event = CA_EVENT_ALLOCATE(ca_exclaves_allocation_statistics);
92 CA_EVENT_TYPE(ca_exclaves_allocation_statistics) * e = event->data;
93
94 e->pages_alloced = os_atomic_load(&exclaves_allocation_statistics.pages_alloced, relaxed);
95 e->pages_freed = os_atomic_load(&exclaves_allocation_statistics.pages_freed, relaxed);
96 e->time_allocating = os_atomic_load(&exclaves_allocation_statistics.time_allocating, relaxed);
97 e->max_alloc_latency = os_atomic_load(&exclaves_allocation_statistics.max_alloc_latency, relaxed);
98 e->alloc_latency_highbit0 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[0], relaxed);
99 e->alloc_latency_highbit1 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[1], relaxed);
100 e->alloc_latency_highbit2 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[2], relaxed);
101 e->alloc_latency_highbit3 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[3], relaxed);
102 e->alloc_latency_highbit4 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[4], relaxed);
103 e->alloc_latency_highbit5 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[5], relaxed);
104 e->alloc_latency_highbit6 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[6], relaxed);
105 e->alloc_latency_highbit7 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[7], relaxed);
106 e->alloc_latency_highbit8 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[8], relaxed);
107 e->alloc_latency_highbit9 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[9], relaxed);
108 e->alloc_latency_highbit10 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[10], relaxed);
109 e->alloc_latency_highbit11 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[11], relaxed);
110 e->alloc_latency_highbit12 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[12], relaxed);
111 e->alloc_latency_highbit13 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[13], relaxed);
112 e->alloc_latency_highbit14 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[14], relaxed);
113 e->alloc_latency_highbit15 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[15], relaxed);
114
115 CA_EVENT_SEND(event);
116 }
117
118 static_assert(
119 (EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN == XNUUPCALLS_PAGEKIND_ROOTDOMAIN) &&
120 (EXCLAVES_MEMORY_PAGEKIND_CONCLAVE == XNUUPCALLS_PAGEKIND_CONCLAVE),
121 "xnuupcalls_pagekind_s mismatch");
122 static_assert(
123 (EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN == XNUUPCALLSV2_PAGEKIND_ROOTDOMAIN) &&
124 (EXCLAVES_MEMORY_PAGEKIND_CONCLAVE == XNUUPCALLSV2_PAGEKIND_CONCLAVE),
125 "xnuupcallsv2_pagekind_s mismatch");
126
127 static ledger_t
get_conclave_mem_ledger(exclaves_memory_pagekind_t kind)128 get_conclave_mem_ledger(exclaves_memory_pagekind_t kind)
129 {
130 ledger_t ledger;
131 switch (kind) {
132 case EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN:
133 ledger = kernel_task->ledger;
134 break;
135 case EXCLAVES_MEMORY_PAGEKIND_CONCLAVE:
136 if (current_thread()->conclave_stop_task != NULL) {
137 ledger = current_thread()->conclave_stop_task->ledger;
138 } else {
139 ledger = current_task()->ledger;
140 }
141 break;
142 default:
143 panic("Conclave Memory ledger doesn't recognize pagekind");
144 break;
145 }
146 return ledger;
147 }
148
149
150 /* -------------------------------------------------------------------------- */
151 #pragma mark Allocation/Free
152
153 void
exclaves_memory_alloc(const uint32_t npages,uint32_t * pages,const exclaves_memory_pagekind_t kind,const exclaves_memory_page_flags_t flags)154 exclaves_memory_alloc(const uint32_t npages, uint32_t *pages, const exclaves_memory_pagekind_t kind, const exclaves_memory_page_flags_t flags)
155 {
156 uint32_t pages_left = npages;
157 vm_page_t page_list = NULL;
158 vm_page_t sequestered = NULL;
159 unsigned p = 0;
160
161 uint64_t start_time = mach_continuous_approximate_time();
162 kma_flags_t kma_flags = KMA_NOFAIL;
163 vm_object_t vm_obj = exclaves_object;
164
165 #if HAS_MTE
166 /**
167 * Avoid specifying KMA_TAG if MTE has been disabled by boot arg.
168 * Otherwise, sptm_retype() will panic if asked to produce a tagged SK page
169 * without tag storage space to back it.
170 */
171 if ((flags & EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED) && is_mte_enabled) {
172 kma_flags |= KMA_TAG;
173 vm_obj = exclaves_object_tagged;
174 }
175 #else /* !HAS_MTE */
176 (void)flags;
177 #endif /* HAS_MTE */
178
179 while (pages_left) {
180 vm_page_t next;
181 vm_page_alloc_list(pages_left, kma_flags, &page_list);
182
183 vm_object_lock(vm_obj);
184 for (vm_page_t mem = page_list; mem != VM_PAGE_NULL; mem = next) {
185 next = mem->vmp_snext;
186 if (!vm_page_in_array(mem)) {
187 // avoid ml_static_mfree() pages due to 117505258
188 mem->vmp_snext = sequestered;
189 sequestered = mem;
190 continue;
191 }
192 mem->vmp_snext = NULL;
193
194 vm_page_lock_queues();
195 vm_page_wire(mem, VM_KERN_MEMORY_EXCLAVES, FALSE);
196 vm_page_unlock_queues();
197 /* Insert the page into the exclaves object */
198 vm_page_insert_wired(mem, vm_obj,
199 ptoa(VM_PAGE_GET_PHYS_PAGE(mem)),
200 VM_KERN_MEMORY_EXCLAVES);
201
202 /* Retype via SPTM to SK owned */
203 sptm_retype_params_t retype_params = {
204 .raw = SPTM_RETYPE_PARAMS_NULL
205 };
206 #if HAS_MTE
207 if (kma_flags & KMA_TAG) {
208 retype_params.sk_flags |= SPTM_SK_PAGE_FLAGS_TAGGABLE;
209 pmap_unmake_tagged_page(VM_PAGE_GET_PHYS_PAGE(mem));
210 }
211 #endif /* HAS_MTE */
212 sptm_retype(ptoa(VM_PAGE_GET_PHYS_PAGE(mem)),
213 XNU_DEFAULT, SK_DEFAULT, retype_params);
214
215 pages[p++] = VM_PAGE_GET_PHYS_PAGE(mem);
216 pages_left--;
217 }
218 vm_object_unlock(vm_obj);
219 }
220
221 vm_page_free_list(sequestered, FALSE);
222
223 uint64_t elapsed_time = mach_continuous_approximate_time() - start_time;
224
225 os_atomic_add(&exclaves_allocation_statistics.pages_alloced, npages, relaxed);
226 os_atomic_add(&exclaves_allocation_statistics.time_allocating, elapsed_time, relaxed);
227 os_atomic_max(&exclaves_allocation_statistics.max_alloc_latency, elapsed_time, relaxed);
228 os_atomic_add(&exclaves_allocation_statistics.alloc_latency_byhighbit[ffsll(elapsed_time) / 4], elapsed_time, relaxed);
229
230 ledger_t ledger = get_conclave_mem_ledger(kind);
231 kern_return_t ledger_ret = ledger_credit(ledger,
232 task_ledgers.conclave_mem,
233 (ledger_amount_t) (npages * PAGE_SIZE));
234 if (ledger_ret != KERN_SUCCESS) {
235 panic("Ledger credit failed. count %u error code %d",
236 npages,
237 ledger_ret);
238 }
239 }
240
241 void
exclaves_memory_free(const uint32_t npages,const uint32_t * pages,const exclaves_memory_pagekind_t kind,const exclaves_memory_page_flags_t flags)242 exclaves_memory_free(const uint32_t npages, const uint32_t *pages, const exclaves_memory_pagekind_t kind, const exclaves_memory_page_flags_t flags)
243 {
244 vm_object_t vm_obj = exclaves_object;
245 #if HAS_MTE
246 if (flags & EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED) {
247 vm_obj = exclaves_object_tagged;
248 }
249 #else /* !HAS_MTE */
250 (void)flags;
251 #endif /* HAS_MTE */
252
253 vm_object_lock(vm_obj);
254 for (size_t p = 0; p < npages; p++) {
255 /* Find the page in the exclaves object. */
256 vm_page_t m;
257 m = vm_page_lookup(vm_obj, ptoa(pages[p]));
258
259 /* Assert we found the page */
260 assert(m != VM_PAGE_NULL);
261
262 /* Via SPTM, verify the page type is something ownable by xnu. */
263 assert3u(sptm_get_frame_type(ptoa(VM_PAGE_GET_PHYS_PAGE(m))),
264 ==, XNU_DEFAULT);
265
266 #if HAS_MTE
267 if (vm_obj == exclaves_object_tagged) {
268 /* pmap_make_tagged_page works lazily, hence we need to mark page m as `using_mte == false` */
269 m->vmp_using_mte = false;
270 pmap_make_tagged_page(VM_PAGE_GET_PHYS_PAGE(m));
271 m->vmp_using_mte = true;
272 }
273 #endif /* HAS_MTE */
274
275 /* Free the page */
276 vm_page_lock_queues();
277 vm_page_free(m);
278 vm_page_unlock_queues();
279 }
280 vm_object_unlock(vm_obj);
281
282 os_atomic_add(&exclaves_allocation_statistics.pages_freed, npages, relaxed);
283
284 ledger_t ledger = get_conclave_mem_ledger(kind);
285 kern_return_t ledger_ret = ledger_debit(ledger,
286 task_ledgers.conclave_mem,
287 (ledger_amount_t) (npages * PAGE_SIZE));
288 if (ledger_ret != KERN_SUCCESS) {
289 panic("Ledger debit failed. count %u error code %d",
290 npages,
291 ledger_ret);
292 }
293 }
294
295 static void
validate_for_mapping(uint32_t page,vm_prot_t prot)296 validate_for_mapping(uint32_t page, vm_prot_t prot)
297 {
298 const sptm_frame_type_t type = sptm_get_frame_type(ptoa(page));
299
300 // Mapping RW and type is SK_SHARED_RW.
301 if (type == SK_SHARED_RW && (prot & VM_PROT_WRITE) != 0) {
302 return;
303 }
304
305 // Mapping RO and type is SK_SHARED_RW or SH_SHARED_RO
306 if ((type == SK_SHARED_RW || type == SK_SHARED_RO) &&
307 (prot & VM_PROT_WRITE) == 0) {
308 return;
309 }
310
311 // Mismatch of type and prot
312 panic("trying to map exclaves memory (prot: %u) "
313 "but memory is of the wrong type (%u)", prot, type);
314 }
315
316 kern_return_t
exclaves_memory_map(uint32_t npages,const uint32_t * pages,vm_prot_t prot,char ** address)317 exclaves_memory_map(uint32_t npages, const uint32_t *pages, vm_prot_t prot,
318 char **address)
319 {
320 assert3u(npages, >, 0);
321
322 kern_return_t kr = KERN_FAILURE;
323 const vm_map_kernel_flags_t vmk_flags = {
324 .vmf_fixed = false,
325 .vm_tag = VM_KERN_MEMORY_EXCLAVES_SHARED,
326 };
327 const vm_size_t size = npages * PAGE_SIZE;
328
329 memory_object_t pager = device_pager_setup((memory_object_t)NULL,
330 (uintptr_t)NULL, size, DEVICE_PAGER_COHERENT);
331 assert3p(pager, !=, NULL);
332
333 for (uint32_t i = 0; i < npages; i++) {
334 validate_for_mapping(pages[i], prot);
335
336 kr = device_pager_populate_object(pager, ptoa(i), pages[i],
337 PAGE_SIZE);
338 if (kr != KERN_SUCCESS) {
339 device_pager_deallocate(pager);
340 return kr;
341 }
342 }
343
344 ipc_port_t entry = IPC_PORT_NULL;
345 kr = mach_memory_object_memory_entry_64((host_t)1, false, size,
346 prot, pager, &entry);
347 if (kr != KERN_SUCCESS) {
348 device_pager_deallocate(pager);
349 return kr;
350 }
351
352 kr = mach_vm_map_kernel(kernel_map, (mach_vm_offset_ut *)address, size, 0, vmk_flags, entry,
353 0, FALSE, prot, prot, VM_INHERIT_DEFAULT);
354
355 mach_memory_entry_port_release(entry);
356
357 if (kr != KERN_SUCCESS) {
358 device_pager_deallocate(pager);
359 return kr;
360 }
361
362 device_pager_deallocate(pager);
363
364 /*
365 * Wire the memory so that it's paged-in up-front. This memory is
366 * already wired via exclaves_memory_alloc.
367 */
368 const vm_map_offset_ut start = *(vm_map_offset_ut *)address;
369 kr = vm_map_wire_kernel(kernel_map, start, start + size, prot,
370 VM_KERN_MEMORY_EXCLAVES_SHARED, false);
371 if (kr != KERN_SUCCESS) {
372 mach_vm_deallocate(kernel_map, start, size);
373 return kr;
374 }
375
376 return KERN_SUCCESS;
377 }
378
379 kern_return_t
exclaves_memory_unmap(char * address,size_t size)380 exclaves_memory_unmap(char *address, size_t size)
381 {
382 kern_return_t kr = KERN_FAILURE;
383
384 const vm_map_offset_ut start = (vm_map_offset_ut)address;
385 kr = vm_map_unwire(kernel_map, start, start + size, false);
386 if (kr != KERN_SUCCESS) {
387 return kr;
388 }
389
390 kr = mach_vm_deallocate(kernel_map, (mach_vm_address_t)address, size);
391 if (kr != KERN_SUCCESS) {
392 return kr;
393 }
394
395 return KERN_SUCCESS;
396 }
397
398 /* -------------------------------------------------------------------------- */
399 #pragma mark Upcalls
400
401 /* Legacy upcall handlers */
402
403 tb_error_t
404 exclaves_memory_upcall_legacy_alloc(uint32_t npages, xnuupcalls_pagekind_s kind,
405 tb_error_t (^completion)(xnuupcalls_pagelist_s))
406 {
407 xnuupcalls_pagelist_s pagelist = {};
408
409 assert3u(npages, <=, ARRAY_COUNT(pagelist.pages));
410 if (npages > ARRAY_COUNT(pagelist.pages)) {
411 panic("npages");
412 }
413
414 exclaves_memory_alloc(npages, pagelist.pages,
415 (exclaves_memory_pagekind_t) kind,
416 EXCLAVES_MEMORY_PAGE_FLAGS_NONE);
417 return completion(pagelist);
418 }
419
420 tb_error_t
421 exclaves_memory_upcall_legacy_alloc_ext(uint32_t npages, xnuupcalls_pageallocflags_s flags,
422 tb_error_t (^completion)(xnuupcalls_pagelist_s))
423 {
424 xnuupcalls_pagelist_s pagelist = {};
425 exclaves_memory_pagekind_t kind = EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN;
426 exclaves_memory_page_flags_t alloc_flags = EXCLAVES_MEMORY_PAGE_FLAGS_NONE;
427
428 assert3u(npages, <=, ARRAY_COUNT(pagelist.pages));
429 if (npages > ARRAY_COUNT(pagelist.pages)) {
430 panic("npages");
431 }
432
433 if (flags & XNUUPCALLS_PAGEALLOCFLAGS_CONCLAVE) {
434 kind = EXCLAVES_MEMORY_PAGEKIND_CONCLAVE;
435 }
436 #if HAS_MTE
437 if (flags & XNUUPCALLS_PAGEALLOCFLAGS_SEC_TRANSITION) {
438 alloc_flags |= EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED;
439 }
440 #endif /* HAS_MTE */
441 exclaves_memory_alloc(npages, pagelist.pages, kind, alloc_flags);
442 return completion(pagelist);
443 }
444
445
446 tb_error_t
447 exclaves_memory_upcall_legacy_free(const uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST],
448 uint32_t npages, const xnuupcalls_pagekind_s kind,
449 tb_error_t (^completion)(void))
450 {
451 /* Get pointer for page list paddr */
452 assert(npages <= EXCLAVES_MEMORY_MAX_REQUEST);
453 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
454 panic("npages");
455 }
456
457 exclaves_memory_free(npages, pages, (exclaves_memory_pagekind_t) kind, EXCLAVES_MEMORY_PAGE_FLAGS_NONE);
458
459 return completion();
460 }
461
462 tb_error_t
463 exclaves_memory_upcall_legacy_free_ext(const uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST],
464 uint32_t npages, const xnuupcalls_pagefreeflags_s flags,
465 tb_error_t (^completion)(void))
466 {
467 exclaves_memory_pagekind_t kind = EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN;
468 exclaves_memory_page_flags_t free_flags = EXCLAVES_MEMORY_PAGE_FLAGS_NONE;
469 /* Get pointer for page list paddr */
470 assert(npages <= EXCLAVES_MEMORY_MAX_REQUEST);
471 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
472 panic("npages");
473 }
474 if (flags & XNUUPCALLS_PAGEALLOCFLAGS_CONCLAVE) {
475 kind = EXCLAVES_MEMORY_PAGEKIND_CONCLAVE;
476 }
477 #if HAS_MTE
478 if (flags & XNUUPCALLS_PAGEFREEFLAGS_SEC_TRANSITION) {
479 free_flags |= EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED;
480 }
481 #endif /* HAS_MTE */
482
483 exclaves_memory_free(npages, pages, kind, free_flags);
484
485 return completion();
486 }
487
488 /* Upcall handlers */
489
490 tb_error_t
491 exclaves_memory_upcall_alloc(uint32_t npages, xnuupcallsv2_pagekind_s kind,
492 tb_error_t (^completion)(xnuupcallsv2_pagelist_s))
493 {
494 uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST];
495 xnuupcallsv2_pagelist_s pagelist = {};
496
497 assert3u(npages, <=, EXCLAVES_MEMORY_MAX_REQUEST);
498 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
499 panic("npages");
500 }
501
502 exclaves_memory_alloc(npages, pages,
503 (exclaves_memory_pagekind_t) kind,
504 EXCLAVES_MEMORY_PAGE_FLAGS_NONE);
505
506 u32__v_assign_unowned(&pagelist, pages, npages);
507
508 return completion(pagelist);
509 }
510
511 tb_error_t
512 exclaves_memory_upcall_alloc_ext(uint32_t npages, xnuupcallsv2_pageallocflagsv2_s flags,
513 tb_error_t (^completion)(xnuupcallsv2_pagelist_s))
514 {
515 uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST];
516 xnuupcallsv2_pagelist_s pagelist = {};
517 exclaves_memory_pagekind_t kind = EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN;
518 exclaves_memory_page_flags_t alloc_flags = EXCLAVES_MEMORY_PAGE_FLAGS_NONE;
519
520 assert3u(npages, <=, EXCLAVES_MEMORY_MAX_REQUEST);
521 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
522 panic("npages");
523 }
524
525 if (flags & XNUUPCALLSV2_PAGEALLOCFLAGSV2_CONCLAVE) {
526 kind = EXCLAVES_MEMORY_PAGEKIND_CONCLAVE;
527 }
528 #if HAS_MTE
529 if (flags & XNUUPCALLSV2_PAGEALLOCFLAGSV2_SEC_TRANSITION) {
530 alloc_flags |= EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED;
531 }
532 #endif /* HAS_MTE */
533
534 exclaves_memory_alloc(npages, pages, kind, alloc_flags);
535
536 u32__v_assign_unowned(&pagelist, pages, npages);
537
538 return completion(pagelist);
539 }
540
541
542 tb_error_t
543 exclaves_memory_upcall_free(const xnuupcallsv2_pagelist_s pages,
544 const xnuupcallsv2_pagekind_s kind, tb_error_t (^completion)(void))
545 {
546 uint32_t _pages[EXCLAVES_MEMORY_MAX_REQUEST];
547 uint32_t *pages_ptr = _pages;
548 uint32_t __block npages = 0;
549
550 u32__v_visit(&pages, ^(size_t i, const uint32_t page) {
551 if (++npages > EXCLAVES_MEMORY_MAX_REQUEST) {
552 panic("npages");
553 }
554 pages_ptr[i] = page;
555 });
556
557 exclaves_memory_free(npages, _pages, (exclaves_memory_pagekind_t) kind, EXCLAVES_MEMORY_PAGE_FLAGS_NONE);
558
559 return completion();
560 }
561
562 tb_error_t
563 exclaves_memory_upcall_free_ext(const xnuupcallsv2_pagelist_s pages,
564 const xnuupcallsv2_pagefreeflagsv2_s flags, tb_error_t (^completion)(void))
565 {
566 uint32_t _pages[EXCLAVES_MEMORY_MAX_REQUEST];
567 uint32_t *pages_ptr = _pages;
568 uint32_t __block npages = 0;
569 exclaves_memory_pagekind_t kind = EXCLAVES_MEMORY_PAGEKIND_ROOTDOMAIN;
570 exclaves_memory_page_flags_t free_flags = EXCLAVES_MEMORY_PAGE_FLAGS_NONE;
571
572 u32__v_visit(&pages, ^(size_t i, const uint32_t page) {
573 if (++npages > EXCLAVES_MEMORY_MAX_REQUEST) {
574 panic("npages");
575 }
576 pages_ptr[i] = page;
577 });
578
579 if (flags & XNUUPCALLSV2_PAGEFREEFLAGSV2_CONCLAVE) {
580 kind = EXCLAVES_MEMORY_PAGEKIND_CONCLAVE;
581 }
582 #if HAS_MTE
583 if (flags & XNUUPCALLSV2_PAGEFREEFLAGSV2_SEC_TRANSITION) {
584 free_flags |= EXCLAVES_MEMORY_PAGE_FLAGS_MTE_TAGGED;
585 }
586 #endif /* HAS_MTE */
587
588 exclaves_memory_free(npages, _pages, kind, free_flags);
589
590 return completion();
591 }
592
593 #endif /* CONFIG_EXCLAVES */
594