1 /*
2 * Copyright (c) 1998-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <IOKit/IOService.h>
30 #include <IOKit/IOInterruptEventSource.h>
31 #include <IOKit/IOTimerEventSource.h>
32 #include <IOKit/IOMapper.h>
33 #include "../Kernel/IOServicePrivate.h"
34
35 #include <Exclaves/Exclaves.h>
36
37 #if CONFIG_EXCLAVES
38 #include <mach/exclaves.h>
39 #include <Exclaves/IOService.tightbeam.h>
40
41 #define EXLOG(x...) do { \
42 if (kIOLogExclaves & gIOKitDebug) \
43 IOLog(x); \
44 } while (false)
45
46 /* Global IOExclaveProxyState lookup table */
47
48 OSDictionary *gExclaveProxyStates;
49 IORecursiveLock *gExclaveProxyStateLock;
50 const OSSymbol *gDARTMapperFunctionSetActive;
51
52 /* IOExclaveProxyState */
53
54 class IOExclaveWorkLoopAperture {
55 public:
56 IOWorkLoop *workLoop;
57 void
closeGate()58 closeGate()
59 {
60 workLoop->closeGate();
61 }
62 void
openGate()63 openGate()
64 {
65 workLoop->openGate();
66 }
67 };
68
69 #endif /* CONFIG_EXCLAVES */
70
71 struct IOService::IOExclaveProxyState {
72 IOService *service;
73 uint64_t mach_endpoint;
74 #if CONFIG_EXCLAVES
75 tb_endpoint_t tb_endpoint;
76 ioservice_ioserviceconcrete client;
77 // ExclaveDriverKit related state
78 bool edk_endpoint_exists;
79 uint64_t edk_mach_endpoint;
80 tb_endpoint_t edk_tb_endpoint;
81 ioservice_ioserviceprivate edk_client;
82 OSDictionary *exclave_interrupts;
83 IOLock *exclave_interrupts_lock;
84 OSDictionary *exclave_timers;
85 IOLock *exclave_timers_lock;
86 uint32_t nextExclaveTimerId;
87
88 // TODO: implement properly once ExclaveAperture removed
89 IOExclaveWorkLoopAperture *ewla;
90
91 IOLock * exclaveAsyncNotificationEventSourcesLock;
92 OSArray *exclaveAsyncNotificationEventSources;
93
94 // ANE specific upcalls
95 ANEUpcallSetPowerStateHandler aneSetPowerStateUpcallHandler;
96 ANEUpcallWorkHandler aneWorkSubmitUpcallHandler;
97 ANEUpcallWorkHandler aneWorkBeginUpcallHandler;
98 ANEUpcallWorkHandler aneWorkEndUpcallHandler;
99 #endif /* CONFIG_EXCLAVES */
100 };
101
102 #if CONFIG_EXCLAVES
103 class IOExclaveProxyStateWrapper : public OSObject {
104 OSDeclareFinalStructors(IOExclaveProxyStateWrapper);
105 public:
106 IOService::IOExclaveProxyState *proxyState;
107 };
108 OSDefineMetaClassAndFinalStructors(IOExclaveProxyStateWrapper, OSObject);
109 #endif /* CONFIG_EXCLAVES */
110
111 bool
exclaveStart(IOService * provider,IOExclaveProxyState ** pRef)112 IOService::exclaveStart(IOService * provider, IOExclaveProxyState ** pRef)
113 {
114 IOExclaveProxyState * ref;
115
116 ref = NULL;
117 #if CONFIG_EXCLAVES
118 int err = 1;
119 do {
120 char key[16];
121 uint64_t serviceID;
122 uint64_t mach_endpoint = 0;
123 uint64_t edk_mach_endpoint = 0;
124 bool edk_endpoint_exists = false;
125 tb_error_t tberr;
126 tb_endpoint_t tb_endpoint;
127 tb_endpoint_t edk_tb_endpoint;
128 ioservice_ioserviceconcrete client;
129 ioservice_ioserviceprivate edk_client;
130 OSObject * prop;
131 OSData * data;
132 IOWorkLoop * wl;
133 IOExclaveProxyStateWrapper * wrapper;
134
135 // exit early if Exclaves are not available
136 if (exclaves_get_status() == EXCLAVES_STATUS_NOT_SUPPORTED) {
137 break;
138 }
139
140 prop = provider->copyProperty("exclave-endpoint");
141 if ((data = OSDynamicCast(OSData, prop))) {
142 mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
143 }
144 OSSafeReleaseNULL(prop);
145
146 prop = provider->copyProperty("exclave-edk-endpoint");
147 if ((data = OSDynamicCast(OSData, prop))) {
148 edk_mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
149 edk_endpoint_exists = true;
150 }
151 OSSafeReleaseNULL(prop);
152
153 // Initialize IOServiceConcrete endpoint
154 tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
155 assert(NULL != tb_endpoint);
156 if (NULL == tb_endpoint) {
157 break;
158 }
159 tberr = ioservice_ioserviceconcrete_init(&client, tb_endpoint);
160 assert(TB_ERROR_SUCCESS == tberr);
161 if (TB_ERROR_SUCCESS != tberr) {
162 break;
163 }
164
165 if (edk_endpoint_exists) {
166 // Initialize IOServicePrivate endpoint
167 edk_tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, edk_mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
168 assert(NULL != edk_tb_endpoint);
169 if (NULL == edk_tb_endpoint) {
170 printf("%s: ERROR: Failed to create endpoint\n", __func__);
171 break;
172 }
173 tberr = ioservice_ioserviceprivate_init(&edk_client, edk_tb_endpoint);
174 assert(TB_ERROR_SUCCESS == tberr);
175 if (TB_ERROR_SUCCESS != tberr) {
176 printf("%s: ERROR: Failed to init IOServicePrivate\n", __func__);
177 break;
178 }
179 }
180
181 ref = IONewZero(IOExclaveProxyState, 1);
182 if (!ref) {
183 break;
184 }
185 ref->service = this;
186 ref->mach_endpoint = mach_endpoint;
187 ref->tb_endpoint = tb_endpoint;
188 ref->client = client;
189 ref->edk_endpoint_exists = edk_endpoint_exists;
190 if (edk_endpoint_exists) {
191 ref->edk_mach_endpoint = edk_mach_endpoint;
192 ref->edk_tb_endpoint = edk_tb_endpoint;
193 ref->edk_client = edk_client;
194 }
195 ref->exclave_interrupts = OSDictionary::withCapacity(1);
196 ref->exclave_interrupts_lock = IOLockAlloc();
197 ref->exclave_timers = OSDictionary::withCapacity(1);
198 ref->exclave_timers_lock = IOLockAlloc();
199 ref->exclaveAsyncNotificationEventSourcesLock = IOLockAlloc();
200
201 // TODO: remove once workloop aperture workaround removed
202 wl = getWorkLoop();
203 if (!wl) {
204 printf("%s ERROR: getWorkLoop failed\n", __func__);
205 break;
206 }
207 ref->ewla = IONew(IOExclaveWorkLoopAperture, 1);
208 if (!ref->ewla) {
209 printf("%s ERROR: exclaveWorkLoopAperture init failed\n", __func__);
210 break;
211 }
212 ref->ewla->workLoop = wl;
213
214 // Add proxy state to global lookup table
215 serviceID = getRegistryEntryID();
216 snprintf(key, sizeof(key), "%llu", serviceID);
217 wrapper = OSTypeAlloc(IOExclaveProxyStateWrapper);
218 wrapper->proxyState = ref;
219 IORecursiveLockLock(gExclaveProxyStateLock);
220 gExclaveProxyStates->setObject(key, wrapper);
221 IORecursiveLockUnlock(gExclaveProxyStateLock);
222
223 if (ref->edk_endpoint_exists) {
224 // Start() called after lookup table registration in case upcalls are made during exclave start().
225 // Use registry ID as exclave's upcall identifer
226 bool result;
227 tberr = ioservice_ioserviceprivate_startprivate(&edk_client, serviceID, &result);
228 if (TB_ERROR_SUCCESS != tberr || !result) {
229 printf("%s ERROR: Failed StartPrivate\n", __func__);
230 // Deregister from lookup table if start() fails
231 IORecursiveLockLock(gExclaveProxyStateLock);
232 gExclaveProxyStates->removeObject(key);
233 IORecursiveLockUnlock(gExclaveProxyStateLock);
234 wrapper->release();
235 break;
236 }
237 }
238
239 err = 0;
240 } while (false);
241
242 if (err) {
243 if (ref) {
244 OSSafeReleaseNULL(ref->exclave_interrupts);
245 if (ref->exclave_interrupts_lock) {
246 IOLockFree(ref->exclave_interrupts_lock);
247 ref->exclave_interrupts_lock = NULL;
248 }
249 OSSafeReleaseNULL(ref->exclave_timers);
250 if (ref->exclave_timers_lock) {
251 IOLockFree(ref->exclave_timers_lock);
252 ref->exclave_timers_lock = NULL;
253 }
254 if (ref->exclaveAsyncNotificationEventSourcesLock) {
255 IOLockFree(ref->exclaveAsyncNotificationEventSourcesLock);
256 ref->exclaveAsyncNotificationEventSourcesLock = NULL;
257 }
258 if (ref->ewla) {
259 IODelete(ref->ewla, IOExclaveWorkLoopAperture, 1);
260 ref->ewla = NULL;
261 }
262 IODelete(ref, IOExclaveProxyState, 1);
263 ref = NULL;
264 }
265 }
266 #endif /* CONFIG_EXCLAVES */
267
268 if (!ref) {
269 return false;
270 }
271
272 *pRef = ref;
273 return true;
274 }
275
276 uint64_t
exclaveEndpoint(IOExclaveProxyState * pRef)277 IOService::exclaveEndpoint(IOExclaveProxyState * pRef)
278 {
279 return pRef->mach_endpoint;
280 }
281
282 bool
start(IOService * provider)283 IOExclaveProxy::start(IOService * provider)
284 {
285 bool ok;
286
287 ok = exclaveStart(provider, &exclaveState);
288
289 return ok;
290 }
291
292 /* Exclave upcall handlers */
293
294 #if CONFIG_EXCLAVES
295
296 static IOService::IOExclaveProxyState *
getProxyStateFromRegistryID(uint64_t id)297 getProxyStateFromRegistryID(uint64_t id)
298 {
299 OSObject *obj = NULL;
300 IOExclaveProxyStateWrapper *wrapper = NULL;
301 char key[15];
302
303 snprintf(key, sizeof(key), "%llu", id);
304 IORecursiveLockLock(gExclaveProxyStateLock);
305 obj = gExclaveProxyStates->getObject(key);
306 IORecursiveLockUnlock(gExclaveProxyStateLock);
307 if (!obj) {
308 printf("%s ERROR: failed to find proxy state\n", __func__);
309 return NULL;
310 }
311
312 wrapper = OSDynamicCast(IOExclaveProxyStateWrapper, obj);
313 if (!wrapper) {
314 printf("%s ERROR: failed to cast IOExclaveProxyStateWrapper\n", __func__);
315 return NULL;
316 }
317
318 if (!wrapper->proxyState) {
319 printf("%s ERROR: IOExclaveProxyStateWrapper contains NULL proxy state\n", __func__);
320 return NULL;
321 }
322
323 return wrapper->proxyState;
324 }
325
326 bool
IOExclaveInterruptUpcallHandler(uint64_t id,IOExclaveInterruptUpcallArgs * args)327 IOExclaveInterruptUpcallHandler(uint64_t id, IOExclaveInterruptUpcallArgs *args)
328 {
329 assert(args);
330 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
331 if (!ref || !args) {
332 return false;
333 }
334 ref->service->retain();
335
336 bool res;
337 switch (args->type) {
338 case kIOExclaveInterruptUpcallTypeRegister:
339 // Register interrupt
340 res = ref->service->exclaveRegisterInterrupt(ref, args->index, args->data.register_args.test_irq);
341 break;
342 case kIOExclaveInterruptUpcallTypeRemove:
343 // Remove interrupt
344 res = ref->service->exclaveRemoveInterrupt(ref, args->index);
345 break;
346 case kIOExclaveInterruptUpcallTypeEnable:
347 // Enable/disable interrupt
348 res = ref->service->exclaveEnableInterrupt(ref, args->index, args->data.enable_args.enable);
349 break;
350 default:
351 res = false;
352 printf("%s ERROR: invalid upcall type\n", __func__);
353 }
354
355 if (!res) {
356 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
357 ref->service->release();
358 return false;
359 }
360
361 ref->service->release();
362 return true;
363 }
364
365 bool
IOExclaveTimerUpcallHandler(uint64_t id,IOExclaveTimerUpcallArgs * args)366 IOExclaveTimerUpcallHandler(uint64_t id, IOExclaveTimerUpcallArgs *args)
367 {
368 assert(args);
369 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
370 if (!ref || !args) {
371 return false;
372 }
373 ref->service->retain();
374
375 bool res;
376 uint32_t timer_id = args->timer_id;
377 switch (args->type) {
378 case kIOExclaveTimerUpcallTypeRegister:
379 // Register timer
380 res = ref->service->exclaveRegisterTimer(ref, &args->timer_id);
381 break;
382 case kIOExclaveTimerUpcallTypeRemove:
383 // Remove timer
384 res = ref->service->exclaveRemoveTimer(ref, timer_id);
385 break;
386 case kIOExclaveTimerUpcallTypeEnable:
387 {
388 // Enable/disable timer
389 bool enable = args->data.enable_args.enable;
390 res = ref->service->exclaveEnableTimer(ref, timer_id, enable);
391 break;
392 }
393 case kIOExclaveTimerUpcallTypeSetTimeout:
394 {
395 // Set timeout
396 uint32_t options = args->data.set_timeout_args.clock_continuous ? kIOTimeOptionsContinuous : 0;
397 AbsoluteTime duration = args->data.set_timeout_args.duration;
398 kern_return_t *kr = &args->data.set_timeout_args.kr;
399 res = ref->service->exclaveTimerSetTimeout(ref, timer_id, options, duration, 0, kr);
400 break;
401 }
402 case kIOExclaveTimerUpcallTypeCancelTimeout:
403 // Cancel timeout
404 res = ref->service->exclaveTimerCancelTimeout(ref, timer_id);
405 break;
406 default:
407 res = false;
408 printf("%s ERROR: invalid upcall type\n", __func__);
409 }
410
411 if (!res) {
412 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
413 ref->service->release();
414 return false;
415 }
416
417 ref->service->release();
418 return true;
419 }
420
421 bool
IOExclaveLockWorkloop(uint64_t id,bool lock)422 IOExclaveLockWorkloop(uint64_t id, bool lock)
423 {
424 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
425 if (!ref) {
426 return false;
427 }
428
429 // Lock or unlock workloop
430 if (lock) {
431 ref->ewla->closeGate();
432 EXLOG("%s locked workloop\n", __func__);
433 } else {
434 ref->ewla->openGate();
435 EXLOG("%s unlocked workloop\n", __func__);
436 }
437 return true;
438 }
439
440 static void
getExclaveInterruptKey(int index,char * key,size_t size)441 getExclaveInterruptKey(int index, char *key, size_t size)
442 {
443 snprintf(key, size, "%d", index);
444 }
445
446 static IOInterruptEventSource *
copyExclaveInterruptEventSource(IOService::IOExclaveProxyState * pRef,int index)447 copyExclaveInterruptEventSource(IOService::IOExclaveProxyState * pRef, int index)
448 {
449 OSObject *obj;
450 IOInterruptEventSource *ies;
451 char irqKey[5];
452
453 if (!pRef) {
454 return NULL;
455 }
456
457 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
458 IOLockAssert(pRef->exclave_interrupts_lock, kIOLockAssertOwned);
459 obj = pRef->exclave_interrupts->getObject(irqKey);
460 if (!obj) {
461 return NULL;
462 }
463
464 ies = OSDynamicCast(IOInterruptEventSource, obj);
465 if (ies) {
466 ies->retain();
467 }
468 return ies;
469 }
470
471 // TODO: Remove after testing
472 void
IOExclaveTestSignalInterrupt(thread_call_param_t arg0,__unused thread_call_param_t arg1)473 IOExclaveTestSignalInterrupt(thread_call_param_t arg0, __unused thread_call_param_t arg1)
474 {
475 EXLOG("%s called\n", __func__);
476
477 // Unpackage params
478 struct IOExclaveTestSignalInterruptParam *params = (struct IOExclaveTestSignalInterruptParam *) arg0;
479 if (params->id == -1 || params->index == -1) {
480 printf("%s: ERROR: id and irq index not initialized\n", __func__);
481 return;
482 }
483
484 uint64_t id = params->id;
485 int index = (int) params->index;
486
487 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
488 if (!ref) {
489 return;
490 }
491 ref->service->retain();
492
493 // Get interrupt
494 char irqKey[5];
495 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
496 OSObject *obj2 = ref->exclave_interrupts->getObject(irqKey);
497 if (!obj2) {
498 printf("%s: ERROR: failed to get ies\n", __func__);
499 ref->service->release();
500 return;
501 }
502
503 IOInterruptEventSource *ies = OSDynamicCast(IOInterruptEventSource, obj2);
504 if (!ies) {
505 printf("%s: ERROR: failed to cast ies\n", __func__);
506 ref->service->release();
507 return;
508 }
509
510 // Signal interrupt
511 ies->interruptOccurred(NULL, NULL, 1);
512
513 ref->service->release();
514 }
515
516 bool
IOExclaveAsyncNotificationUpcallHandler(uint64_t id,struct IOExclaveAsyncNotificationUpcallArgs * args)517 IOExclaveAsyncNotificationUpcallHandler(uint64_t id, struct IOExclaveAsyncNotificationUpcallArgs *args)
518 {
519 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
520 bool ret = false;
521 if (!ref) {
522 return false;
523 }
524
525 switch (args->type) {
526 case AsyncNotificationUpcallTypeSignal:
527 ret = ref->service->exclaveAsyncNotificationSignal(ref, args->notificationID) == kIOReturnSuccess;
528 break;
529 default:
530 ret = false;
531 break;
532 }
533 return ret;
534 }
535
536 bool
IOExclaveMapperOperationUpcallHandler(uint64_t id,IOExclaveMapperOperationUpcallArgs * args)537 IOExclaveMapperOperationUpcallHandler(uint64_t id, IOExclaveMapperOperationUpcallArgs *args)
538 {
539 assert(args);
540 IOService *provider = NULL;
541 bool res = false;
542 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
543 if (!ref) {
544 return false;
545 }
546 provider = ref->service->getProvider();
547
548 IOMapper *mapper = IOMapper::copyMapperForDeviceWithIndex(provider, (unsigned int)(args->mapperIndex));
549 if (!mapper) {
550 goto finish;
551 }
552
553 switch (args->type) {
554 case MapperActivate:
555 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(true), (void *)(false), NULL, NULL);
556 break;
557 case MapperDeactivate:
558 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(false), (void *)(false), NULL, NULL);
559 break;
560 default:
561 break;
562 }
563
564 finish:
565 return res;
566 }
567
568 bool
IOExclaveANEUpcallHandler(uint64_t id,struct IOExclaveANEUpcallArgs * args,bool * result)569 IOExclaveANEUpcallHandler(uint64_t id, struct IOExclaveANEUpcallArgs *args, bool *result)
570 {
571 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
572 bool ret = false;
573 bool _result = false;
574 if (!ref || !args) {
575 return false;
576 }
577
578 switch (args->type) {
579 case kIOExclaveANEUpcallTypeSetPowerState:
580 if (ref->aneSetPowerStateUpcallHandler) {
581 _result = (ref->aneSetPowerStateUpcallHandler)(
582 args->setpowerstate_args.desired_state
583 );
584 ret = true;
585 } else {
586 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
587 }
588 break;
589 case kIOExclaveANEUpcallTypeWorkSubmit:
590 if (ref->aneWorkSubmitUpcallHandler) {
591 _result = (ref->aneWorkSubmitUpcallHandler)(
592 args->work_args.arg0,
593 args->work_args.arg1,
594 args->work_args.arg2
595 );
596 ret = true;
597 } else {
598 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
599 }
600 break;
601 case kIOExclaveANEUpcallTypeWorkBegin:
602 if (ref->aneWorkBeginUpcallHandler) {
603 _result = (ref->aneWorkBeginUpcallHandler)(
604 args->work_args.arg0,
605 args->work_args.arg1,
606 args->work_args.arg2
607 );
608 ret = true;
609 } else {
610 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
611 }
612 break;
613 case kIOExclaveANEUpcallTypeWorkEnd:
614 if (ref->aneWorkEndUpcallHandler) {
615 _result = (ref->aneWorkEndUpcallHandler)(
616 args->work_args.arg0,
617 args->work_args.arg1,
618 args->work_args.arg2
619 );
620 ret = true;
621 } else {
622 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
623 }
624 break;
625 default:
626 ret = false;
627 break;
628 }
629
630 if (result) {
631 *result = _result;
632 }
633
634 return ret;
635 }
636
637 /* IOService exclave methods */
638
639 #endif /* CONFIG_EXCLAVES */
640
641 bool
exclaveRegisterInterrupt(IOExclaveProxyState * pRef,int index,bool noProvider=false)642 IOService::exclaveRegisterInterrupt(IOExclaveProxyState * pRef, int index, bool noProvider = false)
643 {
644 #if CONFIG_EXCLAVES
645 IOInterruptEventSource *ies = NULL;
646 IOInterruptEventSource::Action action;
647 IOWorkLoop *wl;
648 char irqKey[5];
649
650 assert(getWorkLoop());
651
652 if (!pRef) {
653 return false;
654 }
655
656 action = OSMemberFunctionCast(IOInterruptEventSource::Action,
657 this, &IOService::exclaveInterruptOccurred);
658 ies = IOInterruptEventSource::interruptEventSource(this, action, noProvider ? nullptr : getProvider(), index);
659 if (!ies) {
660 return false;
661 }
662
663 wl = getWorkLoop();
664 if (!wl) {
665 ies->release();
666 return false;
667 }
668 if (wl->addEventSource(ies) != kIOReturnSuccess) {
669 ies->release();
670 return false;
671 }
672
673 // Register IOIES in exclave proxy state
674 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
675 IOLockLock(pRef->exclave_interrupts_lock);
676 pRef->exclave_interrupts->setObject(irqKey, ies);
677 IOLockUnlock(pRef->exclave_interrupts_lock);
678 OSSafeReleaseNULL(ies);
679
680 EXLOG("%s: IRQ %d register success!\n", __func__, index);
681 return true;
682 #else /* CONFIG_EXCLAVES */
683 return false;
684 #endif /* CONFIG_EXCLAVES */
685 }
686
687 bool
exclaveRemoveInterrupt(IOExclaveProxyState * pRef,int index)688 IOService::exclaveRemoveInterrupt(IOExclaveProxyState * pRef, int index)
689 {
690 #if CONFIG_EXCLAVES
691 IOInterruptEventSource *ies;
692 IOWorkLoop *wl;
693 char irqKey[5];
694
695 assert(getWorkLoop());
696
697 if (!pRef) {
698 return false;
699 }
700 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
701
702 IOLockLock(pRef->exclave_interrupts_lock);
703
704 ies = copyExclaveInterruptEventSource(pRef, index);
705
706 if (!ies) {
707 IOLockUnlock(pRef->exclave_interrupts_lock);
708 OSSafeReleaseNULL(ies);
709 return false;
710 }
711 pRef->exclave_interrupts->removeObject(irqKey);
712 IOLockUnlock(pRef->exclave_interrupts_lock);
713
714 wl = getWorkLoop();
715 if (!wl) {
716 OSSafeReleaseNULL(ies);
717 return false;
718 }
719
720 wl->removeEventSource(ies);
721 OSSafeReleaseNULL(ies);
722
723 EXLOG("%s: IRQ %d removed successfully\n", __func__, index);
724 return true;
725 #else /* CONFIG_EXCLAVES */
726 return false;
727 #endif /* CONFIG_EXCLAVES */
728 }
729
730 bool
exclaveEnableInterrupt(IOExclaveProxyState * pRef,int index,bool enable)731 IOService::exclaveEnableInterrupt(IOExclaveProxyState * pRef, int index, bool enable)
732 {
733 #if CONFIG_EXCLAVES
734 IOInterruptEventSource *ies;
735
736 assert(getWorkLoop());
737
738 if (!pRef) {
739 return false;
740 }
741
742 IOLockLock(pRef->exclave_interrupts_lock);
743 ies = copyExclaveInterruptEventSource(pRef, index);
744 if (!ies) {
745 IOLockUnlock(pRef->exclave_interrupts_lock);
746 OSSafeReleaseNULL(ies);
747 return false;
748 }
749
750 if (enable) {
751 ies->enable();
752 } else {
753 ies->disable();
754 }
755 IOLockUnlock(pRef->exclave_interrupts_lock);
756 OSSafeReleaseNULL(ies);
757
758 EXLOG("%s: IRQ %s success!\n", __func__, enable ? "enable" : "disable");
759 return true;
760 #else /* CONFIG_EXCLAVES */
761 return false;
762 #endif /* CONFIG_EXCLAVES */
763 }
764
765
766 void
exclaveInterruptOccurred(IOInterruptEventSource * eventSource,int count)767 IOService::exclaveInterruptOccurred(IOInterruptEventSource *eventSource, int count)
768 {
769 #if CONFIG_EXCLAVES
770 tb_error_t tberr;
771 IOService::IOExclaveProxyState *ref;
772
773 if (!eventSource) {
774 printf("%s ERROR: IOInterruptEventSource is null\n", __func__);
775 return;
776 }
777
778 EXLOG("%s id 0x%llx (irq index %d)\n", __func__, getRegistryEntryID(), eventSource->getIntIndex());
779
780 ref = getProxyStateFromRegistryID(getRegistryEntryID());
781 if (!ref) {
782 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
783 return;
784 }
785
786 assert(ref->edk_endpoint_exists);
787 tberr = ioservice_ioserviceprivate_interruptoccurredprivate(&ref->edk_client, eventSource->getIntIndex(), count);
788 assert(TB_ERROR_SUCCESS == tberr);
789 if (TB_ERROR_SUCCESS != tberr) {
790 printf("%s ERROR: tightbeam call failed\n", __func__);
791 return;
792 }
793 #endif /* CONFIG_EXCLAVES */
794 }
795
796 #if CONFIG_EXCLAVES
797 static void
getExclaveTimerKey(uint32_t timer_id,char * key,size_t size)798 getExclaveTimerKey(uint32_t timer_id, char *key, size_t size)
799 {
800 snprintf(key, size, "%d", timer_id);
801 }
802
803 static IOTimerEventSource *
copyExclaveTimerEventSource(IOService::IOExclaveProxyState * pRef,uint32_t timer_id)804 copyExclaveTimerEventSource(IOService::IOExclaveProxyState * pRef, uint32_t timer_id)
805 {
806 OSObject *obj;
807 IOTimerEventSource *tes;
808 char timerKey[5];
809
810 if (!pRef) {
811 return NULL;
812 }
813
814 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
815 IOLockAssert(pRef->exclave_timers_lock, kIOLockAssertOwned);
816 obj = pRef->exclave_timers->getObject(timerKey);
817 if (!obj) {
818 return NULL;
819 }
820
821 tes = OSDynamicCast(IOTimerEventSource, obj);
822 if (tes) {
823 tes->retain();
824 }
825 return tes;
826 }
827 #endif /* CONFIG_EXCLAVES */
828
829 bool
exclaveRegisterTimer(IOExclaveProxyState * pRef,uint32_t * timer_id)830 IOService::exclaveRegisterTimer(IOExclaveProxyState * pRef, uint32_t *timer_id)
831 {
832 #if CONFIG_EXCLAVES
833 IOTimerEventSource *tes = NULL;
834 IOTimerEventSource::Action action;
835 IOWorkLoop *wl;
836 char timerKey[5];
837
838 assert(getWorkLoop());
839
840 if (!pRef || !timer_id) {
841 return false;
842 }
843
844 action = OSMemberFunctionCast(IOTimerEventSource::Action,
845 this, &IOService::exclaveTimerFired);
846 tes = IOTimerEventSource::timerEventSource(this, action);
847 if (!tes) {
848 return false;
849 }
850
851 wl = getWorkLoop();
852 if (!wl) {
853 tes->release();
854 return false;
855 }
856 if (wl->addEventSource(tes) != kIOReturnSuccess) {
857 tes->release();
858 return false;
859 }
860
861 // Register IOTES in exclave proxy state
862 IOLockLock(pRef->exclave_timers_lock);
863 *timer_id = pRef->nextExclaveTimerId++;
864 getExclaveTimerKey(*timer_id, timerKey, sizeof(timerKey));
865 pRef->exclave_timers->setObject(timerKey, tes);
866 IOLockUnlock(pRef->exclave_timers_lock);
867 OSSafeReleaseNULL(tes);
868
869 EXLOG("%s: timer %u register success!\n", __func__, *timer_id);
870 return true;
871 #else /* CONFIG_EXCLAVES */
872 return false;
873 #endif /* CONFIG_EXCLAVES */
874 }
875
876 bool
exclaveRemoveTimer(IOExclaveProxyState * pRef,uint32_t timer_id)877 IOService::exclaveRemoveTimer(IOExclaveProxyState * pRef, uint32_t timer_id)
878 {
879 #if CONFIG_EXCLAVES
880 IOTimerEventSource *tes;
881 IOWorkLoop *wl;
882 char timerKey[5];
883
884 assert(getWorkLoop());
885
886 if (!pRef) {
887 return false;
888 }
889
890 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
891
892 IOLockLock(pRef->exclave_timers_lock);
893 tes = copyExclaveTimerEventSource(pRef, timer_id);
894 if (!tes) {
895 IOLockUnlock(pRef->exclave_timers_lock);
896 OSSafeReleaseNULL(tes);
897 return false;
898 }
899 pRef->exclave_timers->removeObject(timerKey);
900 IOLockUnlock(pRef->exclave_timers_lock);
901
902 wl = getWorkLoop();
903 if (!wl) {
904 OSSafeReleaseNULL(tes);
905 return false;
906 }
907
908 wl->removeEventSource(tes);
909 OSSafeReleaseNULL(tes);
910
911 EXLOG("%s: timer %u removed successfully\n", __func__, timer_id);
912 return true;
913 #else /* CONFIG_EXCLAVES */
914 return false;
915 #endif /* CONFIG_EXCLAVES */
916 }
917
918 bool
exclaveEnableTimer(IOExclaveProxyState * pRef,uint32_t timer_id,bool enable)919 IOService::exclaveEnableTimer(IOExclaveProxyState * pRef, uint32_t timer_id, bool enable)
920 {
921 #if CONFIG_EXCLAVES
922 IOTimerEventSource *tes;
923
924 assert(getWorkLoop());
925
926 if (!pRef) {
927 return false;
928 }
929
930 IOLockLock(pRef->exclave_timers_lock);
931 tes = copyExclaveTimerEventSource(pRef, timer_id);
932 if (!tes) {
933 IOLockUnlock(pRef->exclave_timers_lock);
934 OSSafeReleaseNULL(tes);
935 return false;
936 }
937
938 if (enable) {
939 tes->enable();
940 } else {
941 tes->disable();
942 }
943 IOLockUnlock(pRef->exclave_timers_lock);
944 OSSafeReleaseNULL(tes);
945
946 EXLOG("%s: timer %u %s success\n", __func__, timer_id, enable ? "enable" : "disable");
947 return true;
948 #else /* CONFIG_EXCLAVES */
949 return false;
950 #endif /* CONFIG_EXCLAVES */
951 }
952
953 bool
exclaveTimerSetTimeout(IOExclaveProxyState * pRef,uint32_t timer_id,uint32_t options,AbsoluteTime interval,AbsoluteTime leeway,kern_return_t * kr)954 IOService::exclaveTimerSetTimeout(IOExclaveProxyState * pRef, uint32_t timer_id, uint32_t options, AbsoluteTime interval, AbsoluteTime leeway, kern_return_t *kr)
955 {
956 #if CONFIG_EXCLAVES
957 IOTimerEventSource *tes;
958
959 assert(getWorkLoop());
960
961 if (!pRef || !kr) {
962 return false;
963 }
964
965 IOLockLock(pRef->exclave_timers_lock);
966 tes = copyExclaveTimerEventSource(pRef, timer_id);
967 if (!tes) {
968 IOLockUnlock(pRef->exclave_timers_lock);
969 OSSafeReleaseNULL(tes);
970 return false;
971 }
972
973 *kr = tes->setTimeout(options, interval, leeway);
974 IOLockUnlock(pRef->exclave_timers_lock);
975 OSSafeReleaseNULL(tes);
976
977 EXLOG("%s: timer %u setTimeout completed (kr %d)\n", __func__, timer_id, *kr);
978 return true;
979 #else /* CONFIG_EXCLAVES */
980 return false;
981 #endif /* CONFIG_EXCLAVES */
982 }
983
984 bool
exclaveTimerCancelTimeout(IOExclaveProxyState * pRef,uint32_t timer_id)985 IOService::exclaveTimerCancelTimeout(IOExclaveProxyState * pRef, uint32_t timer_id)
986 {
987 #if CONFIG_EXCLAVES
988 IOTimerEventSource *tes;
989
990 assert(getWorkLoop());
991
992 if (!pRef) {
993 return false;
994 }
995
996 IOLockLock(pRef->exclave_timers_lock);
997 tes = copyExclaveTimerEventSource(pRef, timer_id);
998 if (!tes) {
999 IOLockUnlock(pRef->exclave_timers_lock);
1000 OSSafeReleaseNULL(tes);
1001 return false;
1002 }
1003
1004 tes->cancelTimeout();
1005 IOLockUnlock(pRef->exclave_timers_lock);
1006 OSSafeReleaseNULL(tes);
1007 EXLOG("%s: timer %u setTimeout success\n", __func__, timer_id);
1008 return true;
1009 #else /* CONFIG_EXCLAVES */
1010 return false;
1011 #endif /* CONFIG_EXCLAVES */
1012 }
1013
1014 void
exclaveTimerFired(IOTimerEventSource * eventSource)1015 IOService::exclaveTimerFired(IOTimerEventSource *eventSource)
1016 {
1017 #if CONFIG_EXCLAVES
1018 tb_error_t tberr;
1019 IOService::IOExclaveProxyState *ref;
1020 __block bool found = false;
1021 __block uint32_t timer_id;
1022
1023 if (!eventSource) {
1024 printf("%s ERROR: IOTimerEventSource is null\n", __func__);
1025 return;
1026 }
1027
1028 ref = getProxyStateFromRegistryID(getRegistryEntryID());
1029 if (!ref) {
1030 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
1031 return;
1032 }
1033
1034 // Find timer ID
1035 IOLockLock(ref->exclave_timers_lock);
1036 ref->exclave_timers->iterateObjects(^bool (const OSSymbol * id, OSObject * obj)
1037 {
1038 if (obj == eventSource) {
1039 found = true;
1040 const char *key = id->getCStringNoCopy();
1041 timer_id = (uint32_t) strtol(key, NULL, 0);
1042 return true;
1043 }
1044 return false;
1045 });
1046 IOLockUnlock(ref->exclave_timers_lock);
1047
1048 if (!found) {
1049 printf("%s ERROR: Could not find timer ID\n", __func__);
1050 return;
1051 }
1052
1053 EXLOG("%s id 0x%llx (timer_id %u)\n", __func__, getRegistryEntryID(), timer_id);
1054 assert(ref->edk_endpoint_exists);
1055 tberr = ioservice_ioserviceprivate_timerfiredprivate(&ref->edk_client, timer_id);
1056 assert(TB_ERROR_SUCCESS == tberr);
1057 if (TB_ERROR_SUCCESS != tberr) {
1058 printf("%s ERROR: tightbeam call failed\n", __func__);
1059 return;
1060 }
1061 #endif /* CONFIG_EXCLAVES */
1062 }
1063
1064
1065 kern_return_t
exclaveAsyncNotificationRegister(IOExclaveProxyState * pRef,IOInterruptEventSource * notification,uint32_t * notificationID)1066 IOService::exclaveAsyncNotificationRegister(IOExclaveProxyState * pRef, IOInterruptEventSource *notification, uint32_t *notificationID)
1067 {
1068 #if CONFIG_EXCLAVES
1069 kern_return_t ret;
1070 if (notification == NULL) {
1071 return kIOReturnBadArgument;
1072 }
1073
1074 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
1075 if (!pRef->exclaveAsyncNotificationEventSources) {
1076 pRef->exclaveAsyncNotificationEventSources = OSArray::withCapacity(1);
1077 }
1078 if (pRef->exclaveAsyncNotificationEventSources) {
1079 *notificationID = (uint32_t) pRef->exclaveAsyncNotificationEventSources->getCount();
1080 pRef->exclaveAsyncNotificationEventSources->setObject(notification);
1081 ret = kIOReturnSuccess;
1082 } else {
1083 ret = kIOReturnNoMemory;
1084 }
1085 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1086 return ret;
1087 #else
1088 #pragma unused(pRef, notification, notificationID)
1089 return kIOReturnUnsupported;
1090 #endif /* CONFIG_EXCLAVES*/
1091 }
1092
1093 kern_return_t
exclaveAsyncNotificationSignal(IOExclaveProxyState * pRef,uint32_t notificationID)1094 IOService::exclaveAsyncNotificationSignal(IOExclaveProxyState * pRef, uint32_t notificationID)
1095 {
1096 #if CONFIG_EXCLAVES
1097 kern_return_t ret;
1098 IOInterruptEventSource *event;
1099
1100 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
1101 if (pRef->exclaveAsyncNotificationEventSources && (event = OSDynamicCast(IOInterruptEventSource, pRef->exclaveAsyncNotificationEventSources->getObject((unsigned int)notificationID)))) {
1102 event->interruptOccurred(NULL, NULL, 0);
1103 ret = kIOReturnSuccess;
1104 } else {
1105 ret = kIOReturnError;
1106 }
1107 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1108 return ret;
1109 #else
1110 #pragma unused(pRef, notificationID)
1111 return kIOReturnUnsupported;
1112 #endif /* CONFIG_EXCLAVES */
1113 }
1114
1115 #if CONFIG_EXCLAVES
1116
1117 void
exclaves_wait_for_cpu_init()1118 exclaves_wait_for_cpu_init()
1119 {
1120 OSDictionary *match_dict = IOService::resourceMatching(gIOAllCPUInitializedKey);
1121 IOService *match = IOService::waitForMatchingService(match_dict);
1122 match_dict->release();
1123 match->release();
1124 }
1125
1126 /* ANE Upcalls */
1127
1128 static kern_return_t
exclaveRegisterANEUpcallHelper(IOService::IOExclaveProxyState * pRef,IOExclaveANEUpcallType type,IOWorkLoop * wl,void * block)1129 exclaveRegisterANEUpcallHelper(IOService::IOExclaveProxyState * pRef, IOExclaveANEUpcallType type, IOWorkLoop *wl, void *block)
1130 {
1131 void __block * _block = block;
1132
1133 if (!_block) {
1134 return kIOReturnBadArgument;
1135 }
1136
1137 if (!pRef) {
1138 Block_release(_block);
1139 return kIOReturnBadArgument;
1140 }
1141
1142 if (wl != NULL) {
1143 return wl->runActionBlock(^{
1144 switch (type) {
1145 case kIOExclaveANEUpcallTypeSetPowerState:
1146 if (pRef->aneSetPowerStateUpcallHandler) {
1147 Block_release(pRef->aneSetPowerStateUpcallHandler);
1148 }
1149 pRef->aneSetPowerStateUpcallHandler = (ANEUpcallSetPowerStateHandler) _block;
1150 break;
1151 case kIOExclaveANEUpcallTypeWorkSubmit:
1152 if (pRef->aneWorkSubmitUpcallHandler) {
1153 Block_release(pRef->aneWorkSubmitUpcallHandler);
1154 }
1155 pRef->aneWorkSubmitUpcallHandler = (ANEUpcallWorkHandler) _block;
1156 break;
1157 case kIOExclaveANEUpcallTypeWorkBegin:
1158 if (pRef->aneWorkBeginUpcallHandler) {
1159 Block_release(pRef->aneWorkBeginUpcallHandler);
1160 }
1161 pRef->aneWorkBeginUpcallHandler = (ANEUpcallWorkHandler) _block;
1162 break;
1163 case kIOExclaveANEUpcallTypeWorkEnd:
1164 if (pRef->aneWorkEndUpcallHandler) {
1165 Block_release(pRef->aneWorkEndUpcallHandler);
1166 }
1167 pRef->aneWorkEndUpcallHandler = (ANEUpcallWorkHandler) _block;
1168 break;
1169 default:
1170 Block_release(_block);
1171 return kIOReturnBadArgument;
1172 }
1173 return kIOReturnSuccess;
1174 });
1175 } else {
1176 Block_release(_block);
1177 return kIOReturnError;
1178 }
1179 }
1180
1181 #endif /* CONFIG_EXCLAVES */
1182
1183 kern_return_t
exclaveRegisterANEUpcallSetPowerState(IOExclaveProxyState * pRef,ANEUpcallSetPowerStateHandler handler)1184 IOService::exclaveRegisterANEUpcallSetPowerState(IOExclaveProxyState * pRef, ANEUpcallSetPowerStateHandler handler)
1185 {
1186 #if CONFIG_EXCLAVES
1187 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeSetPowerState, getWorkLoop(), Block_copy(handler));
1188 #else
1189 #pragma unused(pRef, handler)
1190 return kIOReturnUnsupported;
1191 #endif /* CONFIG_EXCLAVES*/
1192 }
1193
1194 kern_return_t
exclaveRegisterANEUpcallWorkSubmit(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1195 IOService::exclaveRegisterANEUpcallWorkSubmit(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1196 {
1197 #if CONFIG_EXCLAVES
1198 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkSubmit, getWorkLoop(), Block_copy(handler));
1199 #else
1200 #pragma unused(pRef, handler)
1201 return kIOReturnUnsupported;
1202 #endif /* CONFIG_EXCLAVES*/
1203 }
1204
1205 kern_return_t
exclaveRegisterANEUpcallWorkBegin(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1206 IOService::exclaveRegisterANEUpcallWorkBegin(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1207 {
1208 #if CONFIG_EXCLAVES
1209 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkBegin, getWorkLoop(), Block_copy(handler));
1210 #else
1211 #pragma unused(pRef, handler)
1212 return kIOReturnUnsupported;
1213 #endif /* CONFIG_EXCLAVES*/
1214 }
1215
1216 kern_return_t
exclaveRegisterANEUpcallWorkEnd(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1217 IOService::exclaveRegisterANEUpcallWorkEnd(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1218 {
1219 #if CONFIG_EXCLAVES
1220 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkEnd, getWorkLoop(), Block_copy(handler));
1221 #else
1222 #pragma unused(pRef, handler)
1223 return kIOReturnUnsupported;
1224 #endif /* CONFIG_EXCLAVES*/
1225 }
1226