1 /*
2 * Copyright (c) 1998-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <IOKit/system.h>
30 #include <IOKit/IOService.h>
31 #include <libkern/OSDebug.h>
32 #include <libkern/c++/OSAllocation.h>
33 #include <libkern/c++/OSContainers.h>
34 #include <libkern/c++/OSKext.h>
35 #include <libkern/c++/OSUnserialize.h>
36 #include <libkern/c++/OSKext.h>
37 #include <libkern/c++/OSSharedPtr.h>
38 #include <libkern/Block.h>
39 #include <IOKit/IOCatalogue.h>
40 #include <IOKit/IOCommand.h>
41 #include <IOKit/IODeviceTreeSupport.h>
42 #include <IOKit/IODeviceMemory.h>
43 #include <IOKit/IOInterrupts.h>
44 #include <IOKit/IOInterruptController.h>
45 #include <IOKit/IOPlatformExpert.h>
46 #include <IOKit/IOMessage.h>
47 #include <IOKit/IOLib.h>
48 #include <IOKit/IOKitKeysPrivate.h>
49 #include <IOKit/IOBSD.h>
50 #include <IOKit/IOUserClient.h>
51 #include <IOKit/IOUserServer.h>
52 #include <IOKit/IOWorkLoop.h>
53 #include <IOKit/IOTimeStamp.h>
54 #include <IOKit/IOHibernatePrivate.h>
55 #include <IOKit/IOInterruptAccountingPrivate.h>
56 #include <IOKit/IOKernelReporters.h>
57 #include <IOKit/AppleKeyStoreInterface.h>
58 #include <IOKit/pwr_mgt/RootDomain.h>
59 #include <IOKit/IOCPU.h>
60 #include <Exclaves/Exclaves.h>
61 #include <kern/cs_blobs.h>
62 #include <mach/sync_policy.h>
63 #include <mach/thread_info.h>
64 #include <IOKit/assert.h>
65 #include <sys/errno.h>
66 #include <sys/kdebug.h>
67 #include <string.h>
68
69 #include <machine/pal_routines.h>
70
71 #define LOG kprintf
72 //#define LOG IOLog
73 #define MATCH_DEBUG 0
74 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
75
76 // disabled since lockForArbitration() can be held externally
77 #define DEBUG_NOTIFIER_LOCKED 0
78
79 enum{
80 kIOUserServerCheckInTimeoutSecs = 120ULL
81 };
82
83 #include "IOServicePrivate.h"
84 #include "IOKitKernelInternal.h"
85
86 // take lockForArbitration before LOCKNOTIFY
87
88 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
89
90 #define super IORegistryEntry
91
92 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
93
94 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
95 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
96
97 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
98
99 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
100
101 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
102
103 OSDefineMetaClassAndStructors(IOResources, IOService)
104 OSDefineMetaClassAndStructors(IOUserResources, IOService)
105 OSDefineMetaClassAndStructors(IOExclaveProxy, IOService)
106
107 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
108
109 OSDefineMetaClassAndStructors(_IOServiceStateNotification, IOService)
110
111 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
112
113 OSDefineMetaClassAndStructors(IOServiceCompatibility, IOService)
114
115 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
116
117 static IOPlatformExpert * gIOPlatform;
118 static class IOPMrootDomain * gIOPMRootDomain;
119 const IORegistryPlane * gIOServicePlane;
120 const IORegistryPlane * gIOPowerPlane;
121 const OSSymbol * gIODeviceMemoryKey;
122 const OSSymbol * gIOInterruptControllersKey;
123 const OSSymbol * gIOInterruptSpecifiersKey;
124
125 const OSSymbol * gIOResourcesKey;
126 const OSSymbol * gIOUserResourcesKey;
127 const OSSymbol * gIOResourceMatchKey;
128 const OSSymbol * gIOResourceMatchedKey;
129 const OSSymbol * gIOResourceIOKitKey;
130
131 const OSSymbol * gIOProviderClassKey;
132 const OSSymbol * gIONameMatchKey;
133 const OSSymbol * gIONameMatchedKey;
134 const OSSymbol * gIOPropertyMatchKey;
135 const OSSymbol * gIOPropertyExistsMatchKey;
136 const OSSymbol * gIOLocationMatchKey;
137 const OSSymbol * gIOParentMatchKey;
138 const OSSymbol * gIOPathMatchKey;
139 const OSSymbol * gIOMatchCategoryKey;
140 const OSSymbol * gIODefaultMatchCategoryKey;
141 const OSSymbol * gIOMatchedAtBootKey;
142 const OSSymbol * gIOMatchedServiceCountKey;
143 const OSSymbol * gIOMatchedPersonalityKey;
144 const OSSymbol * gIORematchPersonalityKey;
145 const OSSymbol * gIORematchCountKey;
146 const OSSymbol * gIODEXTMatchCountKey;
147 const OSSymbol * gIOSupportedPropertiesKey;
148 const OSSymbol * gIOUserServicePropertiesKey;
149 #if defined(XNU_TARGET_OS_OSX)
150 const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey;
151 #endif /* defined(XNU_TARGET_OS_OSX) */
152
153 const OSSymbol * gIOCompatibilityMatchKey;
154 const OSSymbol * gIOCompatibilityPropertiesKey;
155 const OSSymbol * gIOPathKey;
156
157 const OSSymbol * gIOMapperIDKey;
158 const OSSymbol * gIOUserClientClassKey;
159
160 const OSSymbol * gIOUserClassKey;
161 const OSSymbol * gIOUserClassesKey;
162 const OSSymbol * gIOUserServerClassKey;
163 const OSSymbol * gIOUserServerNameKey;
164 const OSSymbol * gIOUserServerTagKey;
165 const OSSymbol * gIOUserUserClientKey;
166 const OSSymbol * gIOUserServerOneProcessKey;
167 const OSSymbol * gIOUserServerPreserveUserspaceRebootKey;
168
169 const OSSymbol * gIOKitDebugKey;
170
171 const OSSymbol * gIOCommandPoolSizeKey;
172
173 const OSSymbol * gIOConsoleLockedKey;
174 const OSSymbol * gIOConsoleUsersKey;
175 const OSSymbol * gIOConsoleSessionUIDKey;
176 const OSSymbol * gIOConsoleSessionAuditIDKey;
177 const OSSymbol * gIOConsoleUsersSeedKey;
178 const OSSymbol * gIOConsoleSessionOnConsoleKey;
179 const OSSymbol * gIOConsoleSessionLoginDoneKey;
180 const OSSymbol * gIOConsoleSessionSecureInputPIDKey;
181 const OSSymbol * gIOConsoleSessionScreenLockedTimeKey;
182 const OSSymbol * gIOConsoleSessionScreenIsLockedKey;
183 clock_sec_t gIOConsoleLockTime;
184 static bool gIOConsoleLoggedIn;
185 #if HIBERNATION
186 static OSBoolean * gIOConsoleBooterLockState;
187 static uint32_t gIOScreenLockState;
188 #endif
189 static IORegistryEntry * gIOChosenEntry;
190
191 static int gIOResourceGenerationCount;
192
193 const OSSymbol * gIOServiceKey;
194 const OSSymbol * gIOPublishNotification;
195 const OSSymbol * gIOFirstPublishNotification;
196 const OSSymbol * gIOMatchedNotification;
197 const OSSymbol * gIOFirstMatchNotification;
198 const OSSymbol * gIOTerminatedNotification;
199 const OSSymbol * gIOWillTerminateNotification;
200
201 const OSSymbol * gIOUserClientEntitlementsKey;
202 const OSSymbol * gIOServiceDEXTEntitlementsKey;
203 const OSSymbol * gIODriverKitEntitlementKey;
204 const OSSymbol * gIODriverKitUserClientEntitlementsKey;
205 const OSSymbol * gIODriverKitUserClientEntitlementAllowAnyKey;
206 const OSSymbol * gIODriverKitRequiredEntitlementsKey;
207 const OSSymbol * gIODriverKitTestDriverEntitlementKey;
208 const OSSymbol * gIODriverKitUserClientEntitlementCommunicatesWithDriversKey;
209 const OSSymbol * gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey;
210 const OSSymbol * gIOMatchDeferKey;
211 const OSSymbol * gIOServiceMatchDeferredKey;
212 const OSSymbol * gIOServiceNotificationUserKey;
213
214 const OSSymbol * gIOExclaveAssignedKey;
215 const OSSymbol * gIOExclaveProxyKey;
216
217 const OSSymbol * gIOPrimaryDriverTerminateOptionsKey;
218 const OSSymbol * gIOMediaKey;
219 const OSSymbol * gIOBlockStorageDriverKey;
220 static const OSSymbol * gPhysicalInterconnectKey;
221 static const OSSymbol * gVirtualInterfaceKey;
222
223 const OSSymbol * gIOAllCPUInitializedKey;
224
225 const OSSymbol * gIOGeneralInterest;
226 const OSSymbol * gIOBusyInterest;
227 const OSSymbol * gIOAppPowerStateInterest;
228 const OSSymbol * gIOPriorityPowerStateInterest;
229 const OSSymbol * gIOConsoleSecurityInterest;
230
231 const OSSymbol * gIOBSDKey;
232 const OSSymbol * gIOBSDNameKey;
233 const OSSymbol * gIOBSDMajorKey;
234 const OSSymbol * gIOBSDMinorKey;
235 const OSSymbol * gIOBSDUnitKey;
236
237 const OSSymbol * gAKSGetKey;
238 #if defined(__i386__) || defined(__x86_64__)
239 const OSSymbol * gIOCreateEFIDevicePathSymbol;
240 #endif
241
242 static OSDictionary * gNotifications;
243 static IORecursiveLock * gNotificationLock;
244
245 static IOService * gIOResources;
246 static IOService * gIOUserResources;
247 static IOService * gIOServiceRoot;
248
249 static OSOrderedSet * gJobs;
250 static semaphore_port_t gJobsSemaphore;
251 static IOLock * gJobsLock;
252 static int gOutstandingJobs;
253 static int gNumConfigThreads;
254 static int gHighNumConfigThreads;
255 static int gMaxConfigThreads = kMaxConfigThreads;
256 static int gNumWaitingThreads;
257 static IOLock * gIOServiceBusyLock;
258 bool gCPUsRunning;
259 bool gIOKitWillTerminate;
260 bool gInUserspaceReboot;
261
262 #define kIOServiceRootMediaParentInvalid ((IOService *) -1UL)
263 #if NO_KEXTD
264 static bool gIOServiceHideIOMedia = false;
265 static IOService * gIOServiceRootMediaParent = NULL;
266 #else /* NO_KEXTD */
267 static bool gIOServiceHideIOMedia = true;
268 static IOService * gIOServiceRootMediaParent = kIOServiceRootMediaParentInvalid;
269 #endif /* !NO_KEXTD */
270
271 static thread_t gIOTerminateThread;
272 static thread_t gIOTerminateWorkerThread;
273 static UInt32 gIOTerminateWork;
274 static OSArray * gIOTerminatePhase2List;
275 static OSArray * gIOStopList;
276 static OSArray * gIOStopProviderList;
277 static OSArray * gIOFinalizeList;
278
279 #if !NO_KEXTD
280 static OSArray * gIOMatchDeferList;
281 #endif
282
283 static SInt32 gIOConsoleUsersSeed;
284 static OSData * gIOConsoleUsersSeedValue;
285
286 extern const OSSymbol * gIODTPHandleKey;
287
288 const OSSymbol * gIOPlatformFunctionHandlerSet;
289
290
291 static IOLock * gIOConsoleUsersLock;
292 static thread_call_t gIOConsoleLockCallout;
293 static IONotifier * gIOServiceNullNotifier;
294
295 static SECURITY_READ_ONLY_LATE(uint32_t) gIODextRelaunchMax;
296
297 #if DEVELOPMENT || DEBUG
298 uint64_t driverkit_checkin_timed_out = 0;
299 #endif
300
301 IORecursiveLock * gDriverKitLaunchLock;
302 OSSet * gDriverKitLaunches;
303 const OSSymbol * gIOAssociatedServicesKey;
304
305 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
306
307 #define LOCKREADNOTIFY() \
308 IORecursiveLockLock( gNotificationLock )
309 #define LOCKWRITENOTIFY() \
310 IORecursiveLockLock( gNotificationLock )
311 #define LOCKWRITE2READNOTIFY()
312 #define UNLOCKNOTIFY() \
313 IORecursiveLockUnlock( gNotificationLock )
314 #define SLEEPNOTIFY(event) \
315 IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
316 #define SLEEPNOTIFYTO(event, deadline) \
317 IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
318 #define WAKEUPNOTIFY(event) \
319 IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
320
321 #define randomDelay() \
322 int del = read_processor_clock(); \
323 del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff; \
324 IOSleep( del );
325
326 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
327
328 #define queue_element(entry, element, type, field) do { \
329 vm_address_t __ele = (vm_address_t) (entry); \
330 __ele -= -4 + ((size_t)(&((type) 4)->field)); \
331 (element) = (type) __ele; \
332 } while(0)
333
334 #define iterqueue(que, elt) \
335 for (queue_entry_t elt = queue_first(que); \
336 !queue_end(que, elt); \
337 elt = queue_next(elt))
338
339 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
340
341 struct IOInterruptAccountingReporter {
342 IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
343 IOInterruptAccountingData * statistics; /* The live statistics values, if any */
344 };
345
346 struct ArbitrationLockQueueElement {
347 queue_chain_t link;
348 IOThread thread;
349 IOService * service;
350 unsigned count;
351 bool required;
352 bool aborted;
353 };
354
355 static queue_head_t gArbitrationLockQueueActive;
356 static queue_head_t gArbitrationLockQueueWaiting;
357 static queue_head_t gArbitrationLockQueueFree;
358 static IOLock * gArbitrationLockQueueLock;
359
360 bool
isInactive(void) const361 IOService::isInactive( void ) const
362 {
363 return 0 != (kIOServiceInactiveState & getState());
364 }
365
366 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
367
368 // Only used by the intel implementation of
369 // IOService::requireMaxBusStall(UInt32 ns)
370 // IOService::requireMaxInterruptDelay(uint32_t ns)
371 struct CpuDelayEntry {
372 IOService * fService;
373 UInt32 fMaxDelay;
374 UInt32 fDelayType;
375 };
376
377 enum {
378 kCpuDelayBusStall,
379 #if defined(__x86_64__)
380 kCpuDelayInterrupt,
381 #endif /* defined(__x86_64__) */
382 kCpuNumDelayTypes
383 };
384
385 static OSData *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
386 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
387 static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes];
388 const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes];
389 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
390 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
391 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
392
393 static void
394 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
395 static IOReturn
396 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
397
398 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
399
400 IOCoreAnalyticsSendEventProc gIOCoreAnalyticsSendEventProc;
401
402 kern_return_t
IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)403 IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)
404 {
405 if (gIOCoreAnalyticsSendEventProc) {
406 return kIOReturnNotPermitted;
407 }
408 gIOCoreAnalyticsSendEventProc = proc;
409
410 return kIOReturnSuccess;
411 }
412
413 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
414
415
416 static IOMessage sSystemPower;
417 extern "C" bool restore_boot;
418
419 namespace IOServicePH
420 {
421 IONotifier * fRootNotifier;
422 OSArray * fUserServers;
423 OSArray * fUserServersWait;
424 OSArray * fMatchingWork;
425 OSArray * fMatchingDelayed;
426 IOService * fSystemPowerAckTo;
427 uint32_t fSystemPowerAckRef;
428 uint8_t fSystemOff;
429 uint8_t fUserServerOff;
430 uint8_t fWaitingUserServers;
431 thread_call_t fUserServerAckTimer;
432
433 void lock();
434 void unlock();
435
436 void init(IOPMrootDomain * root);
437
438 IOReturn systemPowerChange(
439 void * target,
440 void * refCon,
441 UInt32 messageType, IOService * service,
442 void * messageArgument, vm_size_t argSize);
443
444 bool matchingStart(IOService * service);
445 void matchingEnd(IOService * service);
446 void userServerAckTimerExpired(void *, void *);
447 };
448
449 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
450
451 void
initialize(void)452 IOService::initialize( void )
453 {
454 kern_return_t err;
455
456 gIOServicePlane = IORegistryEntry::makePlane( kIOServicePlane );
457 gIOPowerPlane = IORegistryEntry::makePlane( kIOPowerPlane );
458
459 gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
460 gIONameMatchKey = OSSymbol::withCStringNoCopy( kIONameMatchKey );
461 gIONameMatchedKey = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
462 gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
463 gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
464 gIOPathMatchKey = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
465 gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
466 gIOParentMatchKey = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
467
468 gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
469 gIODefaultMatchCategoryKey = OSSymbol::withCStringNoCopy(
470 kIODefaultMatchCategoryKey );
471 gIOMatchedAtBootKey = OSSymbol::withCStringNoCopy(
472 kIOMatchedAtBootKey );
473
474 gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy(
475 kIOMatchedServiceCountKey );
476 gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
477 kIOMatchedPersonalityKey );
478 gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
479 kIORematchPersonalityKey );
480 gIORematchCountKey = OSSymbol::withCStringNoCopy(
481 kIORematchCountKey );
482 gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
483 kIODEXTMatchCountKey );
484
485 #if defined(XNU_TARGET_OS_OSX)
486 gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
487 kIOServiceLegacyMatchingRegistryIDKey );
488 #endif /* defined(XNU_TARGET_OS_OSX) */
489
490 if (!PE_parse_boot_argn("dextrelaunch", &gIODextRelaunchMax, sizeof(gIODextRelaunchMax))) {
491 if (restore_boot) {
492 // Limit dext relaunches in the restore environment
493 gIODextRelaunchMax = 10;
494 } else {
495 gIODextRelaunchMax = 1000;
496 }
497 }
498 PE_parse_boot_argn("iocthreads", &gMaxConfigThreads, sizeof(gMaxConfigThreads));
499
500 gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
501
502 gIOUserClassKey = OSSymbol::withCStringNoCopy(kIOUserClassKey);
503 gIOUserClassesKey = OSSymbol::withCStringNoCopy(kIOUserClassesKey);
504
505 gIOUserServerClassKey = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
506 gIOUserServerNameKey = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
507 gIOUserServerTagKey = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
508 gIOUserUserClientKey = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
509
510 gIOUserServerOneProcessKey = OSSymbol::withCStringNoCopy(kIOUserServerOneProcessKey);
511 gIOUserServerPreserveUserspaceRebootKey = OSSymbol::withCStringNoCopy(kIOUserServerPreserveUserspaceRebootKey);
512
513 gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass );
514 gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
515 gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
516 gIOResourceIOKitKey = OSSymbol::withCStringNoCopy("IOKit");
517
518 gIODeviceMemoryKey = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
519 gIOInterruptControllersKey
520 = OSSymbol::withCStringNoCopy("IOInterruptControllers");
521 gIOInterruptSpecifiersKey
522 = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
523
524 gIOCompatibilityMatchKey = OSSymbol::withCStringNoCopy(kIOCompatibilityMatchKey);
525 gIOCompatibilityPropertiesKey = OSSymbol::withCStringNoCopy(kIOCompatibilityPropertiesKey);
526 gIOPathKey = OSSymbol::withCStringNoCopy(kIOPathKey);
527 gIOSupportedPropertiesKey = OSSymbol::withCStringNoCopy(kIOSupportedPropertiesKey);
528 gIOUserServicePropertiesKey = OSSymbol::withCStringNoCopy(kIOUserServicePropertiesKey);
529
530 gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
531
532 gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
533
534 gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
535
536 gIOGeneralInterest = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
537 gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest );
538 gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
539 gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
540 gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
541
542 gIOBSDKey = OSSymbol::withCStringNoCopy(kIOBSDKey);
543 gIOBSDNameKey = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
544 gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
545 gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
546 gIOBSDUnitKey = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
547
548 gNotifications = OSDictionary::withCapacity( 1 );
549 gIOPublishNotification = OSSymbol::withCStringNoCopy(
550 kIOPublishNotification );
551 gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
552 kIOFirstPublishNotification );
553 gIOMatchedNotification = OSSymbol::withCStringNoCopy(
554 kIOMatchedNotification );
555 gIOFirstMatchNotification = OSSymbol::withCStringNoCopy(
556 kIOFirstMatchNotification );
557 gIOTerminatedNotification = OSSymbol::withCStringNoCopy(
558 kIOTerminatedNotification );
559 gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
560 kIOWillTerminateNotification );
561 gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass);
562
563
564 gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
565 gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
566 gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
567 gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
568
569 gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
570 gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
571 gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
572 gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
573 gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
574 gIOConsoleSessionScreenIsLockedKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
575
576 gIOConsoleUsersSeedValue = OSData::withValueNoCopy(gIOConsoleUsersSeed);
577
578 gIOUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIOUserClientEntitlementsKey );
579 gIOServiceDEXTEntitlementsKey = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
580 gIODriverKitEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
581 gIODriverKitUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
582 #if XNU_TARGET_OS_OSX
583 gIODriverKitUserClientEntitlementAllowAnyKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementAllowAnyKey );
584 #else
585 gIODriverKitUserClientEntitlementAllowAnyKey = NULL;
586 #endif
587 gIODriverKitRequiredEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitRequiredEntitlementsKey );
588 gIODriverKitTestDriverEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitTestDriverEntitlementKey );
589 gIODriverKitUserClientEntitlementCommunicatesWithDriversKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementCommunicatesWithDriversKey);
590 gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey);
591
592 gIOMatchDeferKey = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
593 gIOServiceMatchDeferredKey = OSSymbol::withCStringNoCopy( kIOServiceMatchDeferredKey );
594 gIOServiceNotificationUserKey = OSSymbol::withCStringNoCopy( kIOServiceNotificationUserKey );
595 gIOExclaveAssignedKey = OSSymbol::withCStringNoCopy(kIOExclaveAssignedKey);
596 gIOExclaveProxyKey = OSSymbol::withCStringNoCopy(kIOExclaveProxyKey);
597
598 gIOPrimaryDriverTerminateOptionsKey = OSSymbol::withCStringNoCopy(kIOPrimaryDriverTerminateOptionsKey);
599 gIOMediaKey = OSSymbol::withCStringNoCopy("IOMedia");
600 gIOBlockStorageDriverKey = OSSymbol::withCStringNoCopy("IOBlockStorageDriver");
601 gPhysicalInterconnectKey = OSSymbol::withCStringNoCopy("Physical Interconnect");
602 gVirtualInterfaceKey = OSSymbol::withCStringNoCopy("Virtual Interface");
603
604 gIOAllCPUInitializedKey = OSSymbol::withCStringNoCopy( kIOAllCPUInitializedKey );
605
606 gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
607 sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
608 #if defined(__x86_64__)
609 sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
610 #endif /* defined(__x86_64__) */
611 uint32_t idx;
612 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
613 sCPULatencySet[idx] = OSNumber::withNumber(UINT_MAX, 32);
614 sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
615 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
616 }
617
618 #if defined(__x86_64__)
619 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
620 #endif /* defined(__x86_64__) */
621
622 gNotificationLock = IORecursiveLockAlloc();
623
624 gAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
625
626 #if CONFIG_EXCLAVES
627 gExclaveProxyStates = OSDictionary::withCapacity( 1 );
628
629 gExclaveProxyStateLock = IORecursiveLockAlloc();
630 assert( gExclaveProxyStates && gExclaveProxyStateLock);
631 #endif /* CONFIG_EXCLAVES */
632
633 assert( gIOServicePlane && gIODeviceMemoryKey
634 && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
635 && gIOResourcesKey && gNotifications && gNotificationLock
636 && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
637 && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
638 && gIOPublishNotification && gIOMatchedNotification
639 && gIOTerminatedNotification && gIOServiceKey
640 && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
641 && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
642 && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
643
644 gJobsLock = IOLockAlloc();
645 gJobs = OSOrderedSet::withCapacity( 10 );
646
647 gIOServiceBusyLock = IOLockAlloc();
648
649 gIOConsoleUsersLock = IOLockAlloc();
650
651 err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
652
653 gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
654
655 IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
656
657 assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
658 && gIOConsoleLockCallout && (err == KERN_SUCCESS));
659
660 gIOResources = IOResources::resources();
661 gIOUserResources = IOUserResources::resources();
662 assert( gIOResources && gIOUserResources );
663
664 gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
665 assert(gIOServiceNullNotifier);
666
667 gArbitrationLockQueueLock = IOLockAlloc();
668 queue_init(&gArbitrationLockQueueActive);
669 queue_init(&gArbitrationLockQueueWaiting);
670 queue_init(&gArbitrationLockQueueFree);
671
672 assert( gArbitrationLockQueueLock );
673
674 allocPMInitLock();
675
676 gIOTerminatePhase2List = OSArray::withCapacity( 2 );
677 gIOStopList = OSArray::withCapacity( 16 );
678 gIOStopProviderList = OSArray::withCapacity( 16 );
679 gIOFinalizeList = OSArray::withCapacity( 16 );
680 #if !NO_KEXTD
681 if (OSKext::iokitDaemonAvailable()) {
682 gIOMatchDeferList = OSArray::withCapacity( 16 );
683 } else {
684 gIOMatchDeferList = NULL;
685 }
686 #endif
687 assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
688
689 gDriverKitLaunches = OSSet::withCapacity(0);
690 gDriverKitLaunchLock = IORecursiveLockAlloc();
691 gIOAssociatedServicesKey = OSSymbol::withCStringNoCopy( "IOAssociatedServices" );
692 #if CONFIG_EXCLAVES
693 gDARTMapperFunctionSetActive = OSSymbol::withCStringNoCopy("setActive");
694 #endif /* CONFIG_EXCLAVES */
695
696 // worker thread that is responsible for terminating / cleaning up threads
697 kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
698 assert(gIOTerminateWorkerThread);
699 thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
700 }
701
702 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
703
704 #if defined(__x86_64__)
705 extern "C" {
706 const char *getCpuDelayBusStallHolderName(void);
707 const char *
getCpuDelayBusStallHolderName(void)708 getCpuDelayBusStallHolderName(void)
709 {
710 return sCPULatencyHolderName[kCpuDelayBusStall];
711 }
712
713 const char *getCpuInterruptDelayHolderName(void);
714 const char *
getCpuInterruptDelayHolderName(void)715 getCpuInterruptDelayHolderName(void)
716 {
717 return sCPULatencyHolderName[kCpuDelayInterrupt];
718 }
719 }
720 #endif /* defined(__x86_64__) */
721
722
723
724 #if IOMATCHDEBUG
725 static UInt64
getDebugFlags(OSDictionary * props)726 getDebugFlags( OSDictionary * props )
727 {
728 OSNumber * debugProp;
729 UInt64 debugFlags;
730
731 debugProp = OSDynamicCast( OSNumber,
732 props->getObject( gIOKitDebugKey ));
733 if (debugProp) {
734 debugFlags = debugProp->unsigned64BitValue();
735 } else {
736 debugFlags = gIOKitDebug;
737 }
738
739 return debugFlags;
740 }
741
742 static UInt64
getDebugFlags(IOService * inst)743 getDebugFlags( IOService * inst )
744 {
745 OSObject * prop;
746 OSNumber * debugProp;
747 UInt64 debugFlags;
748
749 prop = inst->copyProperty(gIOKitDebugKey);
750 debugProp = OSDynamicCast(OSNumber, prop);
751 if (debugProp) {
752 debugFlags = debugProp->unsigned64BitValue();
753 } else {
754 debugFlags = gIOKitDebug;
755 }
756
757 OSSafeReleaseNULL(prop);
758
759 return debugFlags;
760 }
761 #endif
762
763 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
764
765 // Probe a matched service and return an instance to be started.
766 // The default score is from the property table, & may be altered
767 // during probe to change the start order.
768
769 IOService *
probe(IOService * provider,SInt32 * score)770 IOService::probe( IOService * provider,
771 SInt32 * score )
772 {
773 return this;
774 }
775
776 bool
start(IOService * provider)777 IOService::start( IOService * provider )
778 {
779 return true;
780 }
781
782 void
stop(IOService * provider)783 IOService::stop( IOService * provider )
784 {
785 if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
786 reserved->uvars->userServer->serviceStop(this, provider);
787 }
788 }
789
790 bool
init(OSDictionary * dictionary)791 IOService::init( OSDictionary * dictionary )
792 {
793 bool ret;
794
795 ret = super::init(dictionary);
796 if (!ret) {
797 return false;
798 }
799 if (reserved) {
800 return true;
801 }
802
803 reserved = IOMallocType(ExpansionData);
804 IOLockInlineInit(&reserved->interruptStatisticsLock);
805 return true;
806 }
807
808 bool
init(IORegistryEntry * from,const IORegistryPlane * inPlane)809 IOService::init( IORegistryEntry * from,
810 const IORegistryPlane * inPlane )
811 {
812 bool ret;
813
814 ret = super::init(from, inPlane);
815 if (!ret) {
816 return false;
817 }
818 if (reserved) {
819 return true;
820 }
821
822 reserved = IOMallocType(ExpansionData);
823 IOLockInlineInit(&reserved->interruptStatisticsLock);
824
825 return true;
826 }
827
828 void
free(void)829 IOService::free( void )
830 {
831 IOInterruptSourcePrivate *sourcesPrivate = NULL;
832 int i = 0;
833 requireMaxBusStall(0);
834 #if defined(__x86_64__)
835 requireMaxInterruptDelay(0);
836 #endif /* defined(__x86_64__) */
837 if (getPropertyTable()) {
838 unregisterAllInterest();
839 }
840 PMfree();
841
842 if (reserved) {
843 if (reserved->interruptStatisticsArray) {
844 for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
845 if (reserved->interruptStatisticsArray[i].reporter) {
846 reserved->interruptStatisticsArray[i].reporter->release();
847 }
848 }
849
850 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
851 }
852
853 if (reserved->uvars && reserved->uvars->userServer) {
854 reserved->uvars->userServer->serviceFree(this);
855 }
856 sourcesPrivate = reserved->interruptSourcesPrivate;
857 IOLockInlineDestroy(&reserved->interruptStatisticsLock);
858 IOFreeType(reserved, ExpansionData);
859 }
860
861 if (_numInterruptSources && _interruptSources) {
862 assert(sourcesPrivate);
863 for (i = 0; i < _numInterruptSources; i++) {
864 void * block = sourcesPrivate[i].vectorBlock;
865 if (block) {
866 Block_release(block);
867 }
868 }
869 IODelete(_interruptSources, IOInterruptSource, _numInterruptSources);
870 _interruptSources = NULL;
871 IODelete(sourcesPrivate, IOInterruptSourcePrivate, _numInterruptSources);
872 }
873
874 super::free();
875 }
876
877 /*
878 * Attach in service plane
879 */
880 bool
attach(IOService * provider)881 IOService::attach( IOService * provider )
882 {
883 bool ok;
884 uint32_t count;
885 AbsoluteTime deadline;
886 int waitResult = THREAD_AWAKENED;
887 bool wait, computeDeadline = true;
888
889 if (provider) {
890 if (gIOKitDebug & kIOLogAttach) {
891 LOG( "%s::attach(%s)\n", getName(),
892 provider->getName());
893 }
894
895 ok = false;
896 do{
897 wait = false;
898 provider->lockForArbitration();
899 if (provider->__state[0] & kIOServiceInactiveState) {
900 ok = false;
901 } else {
902 count = provider->getChildCount(gIOServicePlane);
903 wait = (count > (kIOServiceBusyMax - 4));
904 if (!wait) {
905 ok = attachToParent(provider, gIOServicePlane);
906 } else {
907 IOLog("stalling for detach from %s\n", provider->getName());
908 IOLockLock( gIOServiceBusyLock );
909 provider->__state[1] |= kIOServiceWaitDetachState;
910 }
911 }
912 provider->unlockForArbitration();
913 if (wait) {
914 if (computeDeadline) {
915 clock_interval_to_deadline(15, kSecondScale, &deadline);
916 computeDeadline = false;
917 }
918 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
919 IOLockUnlock( gIOServiceBusyLock );
920 waitResult = thread_block(THREAD_CONTINUE_NULL);
921 wait = (waitResult != THREAD_TIMED_OUT);
922 }
923 }while (wait);
924 } else {
925 gIOServiceRoot = this;
926 ok = attachToParent( getRegistryRoot(), gIOServicePlane);
927 }
928
929 if (ok && !__provider) {
930 (void) getProvider();
931 }
932
933 return ok;
934 }
935
936 IOService *
getServiceRoot(void)937 IOService::getServiceRoot( void )
938 {
939 return gIOServiceRoot;
940 }
941
942 void
detach(IOService * provider)943 IOService::detach( IOService * provider )
944 {
945 IOService * newProvider = NULL;
946 SInt32 busy;
947 bool adjParent;
948
949 if (gIOKitDebug & kIOLogAttach) {
950 LOG("%s::detach(%s)\n", getName(), provider->getName());
951 }
952
953 #if !NO_KEXTD
954 IOLockLock(gJobsLock);
955 if (gIOMatchDeferList) {
956 auto idx = gIOMatchDeferList->getNextIndexOfObject(this, 0);
957 if (-1U != idx) {
958 gIOMatchDeferList->removeObject(idx);
959 }
960 }
961 if (IOServicePH::fMatchingDelayed) {
962 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(this, 0);
963 if (-1U != idx) {
964 IOServicePH::fMatchingDelayed->removeObject(idx);
965 }
966 }
967 IOLockUnlock(gJobsLock);
968 #endif /* NO_KEXTD */
969
970 lockForArbitration();
971
972 uint64_t regID1 = provider->getRegistryEntryID();
973 uint64_t regID2 = getRegistryEntryID();
974 IOServiceTrace(
975 IOSERVICE_DETACH,
976 (uintptr_t) regID1,
977 (uintptr_t) (regID1 >> 32),
978 (uintptr_t) regID2,
979 (uintptr_t) (regID2 >> 32));
980
981 adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
982 && (provider == getProvider()));
983
984 detachFromParent( provider, gIOServicePlane );
985
986 if (busy) {
987 newProvider = getProvider();
988 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
989 _adjustBusy( -busy );
990 }
991 }
992
993 if (kIOServiceInactiveState & __state[0]) {
994 getMetaClass()->removeInstance(this);
995 IORemoveServicePlatformActions(this);
996 }
997
998 unlockForArbitration();
999
1000 if (newProvider && adjParent) {
1001 newProvider->lockForArbitration();
1002 newProvider->_adjustBusy(1);
1003 newProvider->unlockForArbitration();
1004 }
1005
1006 // check for last client detach from a terminated service
1007 if (provider->lockForArbitration( true )) {
1008 if (kIOServiceStartState & __state[1]) {
1009 provider->scheduleTerminatePhase2();
1010 }
1011 if (adjParent) {
1012 provider->_adjustBusy( -1 );
1013 }
1014 if ((provider->__state[1] & kIOServiceTermPhase3State)
1015 && (NULL == provider->getClient())) {
1016 provider->scheduleFinalize(false);
1017 }
1018
1019 IOLockLock( gIOServiceBusyLock );
1020 if (kIOServiceWaitDetachState & provider->__state[1]) {
1021 provider->__state[1] &= ~kIOServiceWaitDetachState;
1022 thread_wakeup(&provider->__provider);
1023 }
1024 IOLockUnlock( gIOServiceBusyLock );
1025
1026 provider->unlockForArbitration();
1027 }
1028
1029 if (kIOServiceRematchOnDetach & __state[1]) {
1030 provider->registerService();
1031 }
1032 }
1033
1034 /*
1035 * Register instance - publish it for matching
1036 */
1037
1038 void
registerService(IOOptionBits options)1039 IOService::registerService( IOOptionBits options )
1040 {
1041 OSDataAllocation<char> pathBuf;
1042 const char * path;
1043 const char * skip;
1044 int len;
1045 enum { kMaxPathLen = 256 };
1046 enum { kMaxChars = 63 };
1047
1048 IORegistryEntry * parent = this;
1049 IORegistryEntry * root = getRegistryRoot();
1050 while (parent && (parent != root)) {
1051 parent = parent->getParentEntry( gIOServicePlane);
1052 }
1053
1054 if (parent != root) {
1055 IOLog("%s: not registry member at registerService()\n", getName());
1056 return;
1057 }
1058
1059 // Allow the Platform Expert to adjust this node.
1060 if (gIOPlatform && (!gIOPlatform->platformAdjustService(this))) {
1061 return;
1062 }
1063
1064 IOInstallServicePlatformActions(this);
1065 IOInstallServiceSleepPlatformActions(this);
1066
1067 if ((this != gIOResources)
1068 && (kIOLogRegister & gIOKitDebug)) {
1069 pathBuf = OSDataAllocation<char>( kMaxPathLen, OSAllocateMemory );
1070
1071 IOLog( "Registering: " );
1072
1073 len = kMaxPathLen;
1074 if (pathBuf && getPath( pathBuf.data(), &len, gIOServicePlane)) {
1075 path = pathBuf.data();
1076 if (len > kMaxChars) {
1077 IOLog("..");
1078 len -= kMaxChars;
1079 path += len;
1080 if ((skip = strchr( path, '/'))) {
1081 path = skip;
1082 }
1083 }
1084 } else {
1085 path = getName();
1086 }
1087
1088 IOLog( "%s\n", path );
1089 }
1090
1091 startMatching( options );
1092 }
1093
1094 void
startMatching(IOOptionBits options)1095 IOService::startMatching( IOOptionBits options )
1096 {
1097 IOService * provider;
1098 UInt32 prevBusy = 0;
1099 bool needConfig;
1100 bool needWake = false;
1101 bool sync;
1102 bool waitAgain;
1103 bool releaseAssertion = false;
1104
1105 if (options & kIOServiceDextRequirePowerForMatching) {
1106 bool ok = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
1107 if (!ok) {
1108 panic("%s: Failed to acquire power assertion for matching", getName());
1109 }
1110 releaseAssertion = true;
1111 }
1112
1113 lockForArbitration();
1114
1115 sync = (options & kIOServiceSynchronous)
1116 || ((provider = getProvider())
1117 && (provider->__state[1] & kIOServiceSynchronousState));
1118
1119 if (options & kIOServiceAsynchronous) {
1120 sync = false;
1121 }
1122
1123 needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1124 && (0 == (__state[0] & kIOServiceInactiveState));
1125
1126 __state[1] |= kIOServiceNeedConfigState;
1127
1128 // __state[0] &= ~kIOServiceInactiveState;
1129
1130 // if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1131 // OSKernelStackRemaining(), getName());
1132
1133 if (needConfig) {
1134 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1135 }
1136
1137 if (sync) {
1138 __state[1] |= kIOServiceSynchronousState;
1139 } else {
1140 __state[1] &= ~kIOServiceSynchronousState;
1141 }
1142
1143 if (needConfig) {
1144 prevBusy = _adjustBusy( 1 );
1145 }
1146
1147 unlockForArbitration();
1148
1149 if (needConfig) {
1150 if (needWake) {
1151 IOLockLock( gIOServiceBusyLock );
1152 thread_wakeup((event_t) this /*&__state[1]*/ );
1153 IOLockUnlock( gIOServiceBusyLock );
1154 } else if (!sync || (kIOServiceAsynchronous & options)) {
1155 // assertion will be released when matching job is complete
1156 releaseAssertion = false;
1157 _IOServiceJob::startJob( this, kMatchNubJob, options );
1158 } else {
1159 do {
1160 if ((__state[1] & kIOServiceNeedConfigState)) {
1161 doServiceMatch( options );
1162 }
1163
1164 lockForArbitration();
1165 IOLockLock( gIOServiceBusyLock );
1166
1167 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1168 && (0 == (__state[0] & kIOServiceInactiveState)));
1169
1170 if (waitAgain) {
1171 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1172 } else {
1173 __state[1] &= ~kIOServiceSyncPubState;
1174 }
1175
1176 unlockForArbitration();
1177
1178 if (waitAgain) {
1179 assert_wait((event_t) this /*&__state[1]*/, THREAD_UNINT);
1180 }
1181
1182 IOLockUnlock( gIOServiceBusyLock );
1183 if (waitAgain) {
1184 thread_block(THREAD_CONTINUE_NULL);
1185 }
1186 } while (waitAgain);
1187 }
1188 }
1189
1190 if (releaseAssertion) {
1191 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
1192 }
1193 }
1194
1195
1196 void
startDeferredMatches(void)1197 IOService::startDeferredMatches(void)
1198 {
1199 #if !NO_KEXTD
1200 OSArray * array;
1201
1202 IOLockLock(gJobsLock);
1203 array = gIOMatchDeferList;
1204 gIOMatchDeferList = NULL;
1205 IOLockUnlock(gJobsLock);
1206
1207 if (array) {
1208 IOLog("deferred rematching count %d\n", array->getCount());
1209 array->iterateObjects(^bool (OSObject * obj)
1210 {
1211 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
1212 return false;
1213 });
1214 array->release();
1215 }
1216 #endif /* !NO_KEXTD */
1217 }
1218
1219 void
iokitDaemonLaunched(void)1220 IOService::iokitDaemonLaunched(void)
1221 {
1222 #if !NO_KEXTD
1223 if (!OSKext::iokitDaemonAvailable()) {
1224 panic(kIOKitDaemonName " is unavailable in this environment, but it was launched");
1225 }
1226 IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1227 startDeferredMatches();
1228 getServiceRoot()->adjustBusy(-1);
1229 IOService::publishUserResource(gIOResourceIOKitKey);
1230 #endif /* !NO_KEXTD */
1231 }
1232
1233 /*
1234 * Possibly called with IORWLock from IOCatalog held.
1235 * This means that no calls to OSKext that could take
1236 * sKextLock can be performed from this function.
1237 */
1238 IOReturn
catalogNewDrivers(OSOrderedSet * newTables)1239 IOService::catalogNewDrivers( OSOrderedSet * newTables )
1240 {
1241 OSDictionary * table;
1242 OSSet * set;
1243 OSSet * allSet = NULL;
1244 IOService * service;
1245 #if IOMATCHDEBUG
1246 SInt32 count = 0;
1247 #endif
1248
1249 newTables->retain();
1250
1251 while ((table = (OSDictionary *) newTables->getFirstObject())) {
1252 LOCKWRITENOTIFY();
1253 set = (OSSet *) copyExistingServices( table,
1254 kIOServiceRegisteredState,
1255 kIOServiceExistingSet);
1256 UNLOCKNOTIFY();
1257 if (set) {
1258 #if IOMATCHDEBUG
1259 count += set->getCount();
1260 #endif
1261 if (allSet) {
1262 allSet->merge((const OSSet *) set);
1263 set->release();
1264 } else {
1265 allSet = set;
1266 }
1267 }
1268
1269 #if IOMATCHDEBUG
1270 if (getDebugFlags( table ) & kIOLogMatch) {
1271 LOG("Matching service count = %ld\n", (long)count);
1272 }
1273 #endif
1274 newTables->removeObject(table);
1275 }
1276
1277 if (allSet) {
1278 while ((service = (IOService *) allSet->getAnyObject())) {
1279 service->startMatching(kIOServiceAsynchronous);
1280 allSet->removeObject(service);
1281 }
1282 allSet->release();
1283 }
1284
1285 newTables->release();
1286
1287 return kIOReturnSuccess;
1288 }
1289
1290 _IOServiceJob *
startJob(IOService * nub,int type,IOOptionBits options)1291 _IOServiceJob::startJob( IOService * nub, int type,
1292 IOOptionBits options )
1293 {
1294 _IOServiceJob * job;
1295
1296 job = new _IOServiceJob;
1297 if (job && !job->init()) {
1298 job->release();
1299 job = NULL;
1300 }
1301
1302 if (job) {
1303 job->type = type;
1304 job->nub = nub;
1305 job->options = options;
1306 nub->retain(); // thread will release()
1307 pingConfig( job );
1308 }
1309
1310 return job;
1311 }
1312
1313 /*
1314 * Called on a registered service to see if it matches
1315 * a property table.
1316 */
1317
1318 bool
matchPropertyTable(OSDictionary * table,SInt32 * score)1319 IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1320 {
1321 return matchPropertyTable(table);
1322 }
1323
1324 bool
matchPropertyTable(OSDictionary * table)1325 IOService::matchPropertyTable( OSDictionary * table )
1326 {
1327 return true;
1328 }
1329
1330 /*
1331 * Called on a matched service to allocate resources
1332 * before first driver is attached.
1333 */
1334
1335 IOReturn
getResources(void)1336 IOService::getResources( void )
1337 {
1338 return kIOReturnSuccess;
1339 }
1340
1341 /*
1342 * Client/provider accessors
1343 */
1344
1345 IOService *
getProvider(void) const1346 IOService::getProvider( void ) const
1347 {
1348 IOService * self = (IOService *) this;
1349 IOService * parent;
1350 SInt32 generation;
1351
1352 generation = getRegistryEntryParentGenerationCount();
1353 if (__providerGeneration == generation) {
1354 return __provider;
1355 }
1356
1357 parent = (IOService *) getParentEntry( gIOServicePlane);
1358 if (parent == IORegistryEntry::getRegistryRoot()) {
1359 /* root is not an IOService */
1360 parent = NULL;
1361 }
1362
1363 self->__provider = parent;
1364 OSMemoryBarrier();
1365 // save the count from before call to getParentEntry()
1366 self->__providerGeneration = generation;
1367
1368 return parent;
1369 }
1370
1371 IOWorkLoop *
getWorkLoop() const1372 IOService::getWorkLoop() const
1373 {
1374 IOService *provider = getProvider();
1375
1376 if (provider) {
1377 return provider->getWorkLoop();
1378 } else {
1379 return NULL;
1380 }
1381 }
1382
1383 OSIterator *
getProviderIterator(void) const1384 IOService::getProviderIterator( void ) const
1385 {
1386 return getParentIterator( gIOServicePlane);
1387 }
1388
1389 IOService *
getClient(void) const1390 IOService::getClient( void ) const
1391 {
1392 return (IOService *) getChildEntry( gIOServicePlane);
1393 }
1394
1395 OSIterator *
getClientIterator(void) const1396 IOService::getClientIterator( void ) const
1397 {
1398 return getChildIterator( gIOServicePlane);
1399 }
1400
1401 OSIterator *
iterator(OSIterator * _iter,const IOService * client,const IOService * provider)1402 _IOOpenServiceIterator::iterator( OSIterator * _iter,
1403 const IOService * client,
1404 const IOService * provider )
1405 {
1406 _IOOpenServiceIterator * inst;
1407
1408 if (!_iter) {
1409 return NULL;
1410 }
1411
1412 inst = new _IOOpenServiceIterator;
1413
1414 if (inst && !inst->init()) {
1415 inst->release();
1416 inst = NULL;
1417 }
1418 if (inst) {
1419 inst->iter = _iter;
1420 inst->client = client;
1421 inst->provider = provider;
1422 } else {
1423 OSSafeReleaseNULL(_iter);
1424 }
1425
1426 return inst;
1427 }
1428
1429 void
free()1430 _IOOpenServiceIterator::free()
1431 {
1432 iter->release();
1433 if (last) {
1434 last->unlockForArbitration();
1435 }
1436 OSIterator::free();
1437 }
1438
1439 OSObject *
getNextObject()1440 _IOOpenServiceIterator::getNextObject()
1441 {
1442 IOService * next;
1443
1444 if (last) {
1445 last->unlockForArbitration();
1446 }
1447
1448 while ((next = (IOService *) iter->getNextObject())) {
1449 next->lockForArbitration();
1450 if ((client && (next->isOpen( client )))
1451 || (provider && (provider->isOpen( next )))) {
1452 break;
1453 }
1454 next->unlockForArbitration();
1455 }
1456
1457 last = next;
1458
1459 return next;
1460 }
1461
1462 bool
isValid()1463 _IOOpenServiceIterator::isValid()
1464 {
1465 return iter->isValid();
1466 }
1467
1468 void
reset()1469 _IOOpenServiceIterator::reset()
1470 {
1471 if (last) {
1472 last->unlockForArbitration();
1473 last = NULL;
1474 }
1475 iter->reset();
1476 }
1477
1478 OSIterator *
getOpenProviderIterator(void) const1479 IOService::getOpenProviderIterator( void ) const
1480 {
1481 return _IOOpenServiceIterator::iterator( getProviderIterator(), this, NULL );
1482 }
1483
1484 OSIterator *
getOpenClientIterator(void) const1485 IOService::getOpenClientIterator( void ) const
1486 {
1487 return _IOOpenServiceIterator::iterator( getClientIterator(), NULL, this );
1488 }
1489
1490
1491 IOReturn
callPlatformFunction(const OSSymbol * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1492 IOService::callPlatformFunction( const OSSymbol * functionName,
1493 bool waitForFunction,
1494 void *param1, void *param2,
1495 void *param3, void *param4 )
1496 {
1497 IOReturn result = kIOReturnUnsupported;
1498 IOService *provider;
1499
1500 if (functionName == gIOPlatformQuiesceActionKey ||
1501 functionName == gIOPlatformActiveActionKey ||
1502 functionName == gIOPlatformPanicActionKey) {
1503 /*
1504 * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction / IOPlatformPanicAction
1505 * must consume that event themselves, without passing it up to super/IOService.
1506 */
1507 if (gEnforcePlatformActionSafety) {
1508 panic("Class %s passed the %s action to IOService",
1509 getMetaClass()->getClassName(), functionName->getCStringNoCopy());
1510 }
1511 }
1512
1513 if (gIOPlatformFunctionHandlerSet == functionName) {
1514 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1515 IOService * target = (IOService *) param2;
1516 bool enable = (param3 != NULL);
1517
1518 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1519 result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1520 }
1521 #if defined(__x86_64__)
1522 else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1523 result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1524 }
1525 #endif /* defined(__x86_64__) */
1526 }
1527
1528 if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1529 result = provider->callPlatformFunction(functionName, waitForFunction,
1530 param1, param2, param3, param4);
1531 }
1532
1533 return result;
1534 }
1535
1536 IOReturn
callPlatformFunction(const char * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1537 IOService::callPlatformFunction( const char * functionName,
1538 bool waitForFunction,
1539 void *param1, void *param2,
1540 void *param3, void *param4 )
1541 {
1542 IOReturn result = kIOReturnNoMemory;
1543 const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1544
1545 if (functionSymbol != NULL) {
1546 result = callPlatformFunction(functionSymbol, waitForFunction,
1547 param1, param2, param3, param4);
1548 functionSymbol->release();
1549 }
1550
1551 return result;
1552 }
1553
1554
1555 /*
1556 * Accessors for global services
1557 */
1558
1559 IOPlatformExpert *
getPlatform(void)1560 IOService::getPlatform( void )
1561 {
1562 return gIOPlatform;
1563 }
1564
1565 class IOPMrootDomain *
getPMRootDomain(void)1566 IOService::getPMRootDomain( void )
1567 {
1568 return gIOPMRootDomain;
1569 }
1570
1571 IOService *
getResourceService(void)1572 IOService::getResourceService( void )
1573 {
1574 return gIOResources;
1575 }
1576
1577 IOService * gIOSystemStateNotificationService;
1578
1579 IOService *
getSystemStateNotificationService(void)1580 IOService::getSystemStateNotificationService(void)
1581 {
1582 return gIOSystemStateNotificationService;
1583 }
1584
1585 void
setPlatform(IOPlatformExpert * platform)1586 IOService::setPlatform( IOPlatformExpert * platform)
1587 {
1588 gIOPlatform = platform;
1589 gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1590
1591 gIOUserResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1592
1593 #if DEVELOPMENT || DEBUG
1594 // Test object that will be terminated for dext to match
1595 {
1596 IOService * ios;
1597 ios = OSTypeAlloc(IOService);
1598 ios->init();
1599 ios->attach(gIOUserResources);
1600 ios->setProperty(gIOMatchCategoryKey->getCStringNoCopy(), "com.apple.iokit.test");
1601 ios->setProperty(gIOModuleIdentifierKey->getCStringNoCopy(), "com.apple.kpi.iokit");
1602 ios->setProperty(gIOMatchedAtBootKey, kOSBooleanTrue);
1603 ios->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
1604 ios->release();
1605 }
1606 #endif
1607
1608 gIOSystemStateNotificationService = IOSystemStateNotification::initialize();
1609 gIOSystemStateNotificationService->attachToParent(platform, gIOServicePlane);
1610 gIOSystemStateNotificationService->registerService();
1611
1612 static const char * keys[kCpuNumDelayTypes] = {
1613 kIOPlatformMaxBusDelay,
1614 #if defined(__x86_64__)
1615 kIOPlatformMaxInterruptDelay
1616 #endif /* defined(__x86_64__) */
1617 };
1618 const OSObject * objs[2];
1619 OSArray * array;
1620 uint32_t idx;
1621
1622 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1623 objs[0] = sCPULatencySet[idx];
1624 objs[1] = sCPULatencyHolder[idx];
1625 array = OSArray::withObjects(objs, 2);
1626 if (!array) {
1627 break;
1628 }
1629 platform->setProperty(keys[idx], array);
1630 array->release();
1631 }
1632 }
1633
1634 void
setPMRootDomain(class IOPMrootDomain * rootDomain)1635 IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1636 {
1637 gIOPMRootDomain = rootDomain;
1638 }
1639
1640 void
publishPMRootDomain(void)1641 IOService::publishPMRootDomain( void )
1642 {
1643 assert(getPMRootDomain() != NULL);
1644 publishResource(gIOResourceIOKitKey);
1645 #if NO_KEXTD
1646 // Publish IOUserResources now since there is no IOKit daemon.
1647 publishUserResource(gIOResourceIOKitKey);
1648 #endif
1649 IOServicePH::init(getPMRootDomain());
1650 }
1651
1652 /*
1653 * Stacking change
1654 */
1655
1656 bool
lockForArbitration(bool isSuccessRequired)1657 IOService::lockForArbitration( bool isSuccessRequired )
1658 {
1659 bool found;
1660 bool success;
1661 ArbitrationLockQueueElement * element;
1662 ArbitrationLockQueueElement * owner;
1663 ArbitrationLockQueueElement * active;
1664 ArbitrationLockQueueElement * waiting;
1665
1666 enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1667
1668 // lock global access
1669 IOTakeLock( gArbitrationLockQueueLock );
1670
1671 // obtain an unused queue element
1672 if (!queue_empty( &gArbitrationLockQueueFree )) {
1673 queue_remove_first( &gArbitrationLockQueueFree,
1674 element,
1675 ArbitrationLockQueueElement *,
1676 link );
1677 } else {
1678 element = IOMallocType(ArbitrationLockQueueElement);
1679 assert( element );
1680 }
1681
1682 // prepare the queue element
1683 element->thread = IOThreadSelf();
1684 element->service = this;
1685 element->count = 1;
1686 element->required = isSuccessRequired;
1687 element->aborted = false;
1688
1689 // determine whether this object is already locked (ie. on active queue)
1690 found = false;
1691 queue_iterate( &gArbitrationLockQueueActive,
1692 owner,
1693 ArbitrationLockQueueElement *,
1694 link )
1695 {
1696 if (owner->service == element->service) {
1697 found = true;
1698 break;
1699 }
1700 }
1701
1702 if (found) { // this object is already locked
1703 active = owner;
1704 // determine whether it is the same or a different thread trying to lock
1705 if (active->thread != element->thread) { // it is a different thread
1706 ArbitrationLockQueueElement * victim = NULL;
1707
1708 // before placing this new thread on the waiting queue, we look for
1709 // a deadlock cycle...
1710
1711 while (1) {
1712 // determine whether the active thread holding the object we
1713 // want is waiting for another object to be unlocked
1714 found = false;
1715 queue_iterate( &gArbitrationLockQueueWaiting,
1716 waiting,
1717 ArbitrationLockQueueElement *,
1718 link )
1719 {
1720 if (waiting->thread == active->thread) {
1721 assert( false == waiting->aborted );
1722 found = true;
1723 break;
1724 }
1725 }
1726
1727 if (found) { // yes, active thread waiting for another object
1728 // this may be a candidate for rejection if the required
1729 // flag is not set, should we detect a deadlock later on
1730 if (false == waiting->required) {
1731 victim = waiting;
1732 }
1733
1734 // find the thread that is holding this other object, that
1735 // is blocking the active thread from proceeding (fun :-)
1736 found = false;
1737 queue_iterate( &gArbitrationLockQueueActive,
1738 active, // (reuse active queue element)
1739 ArbitrationLockQueueElement *,
1740 link )
1741 {
1742 if (active->service == waiting->service) {
1743 found = true;
1744 break;
1745 }
1746 }
1747
1748 // someone must be holding it or it wouldn't be waiting
1749 assert( found );
1750
1751 if (active->thread == element->thread) {
1752 // doh, it's waiting for the thread that originated
1753 // this whole lock (ie. current thread) -> deadlock
1754 if (false == element->required) { // willing to fail?
1755 // the originating thread doesn't have the required
1756 // flag, so it can fail
1757 success = false; // (fail originating lock request)
1758 break; // (out of while)
1759 } else { // originating thread is not willing to fail
1760 // see if we came across a waiting thread that did
1761 // not have the 'required' flag set: we'll fail it
1762 if (victim) {
1763 // we do have a willing victim, fail it's lock
1764 victim->aborted = true;
1765
1766 // take the victim off the waiting queue
1767 queue_remove( &gArbitrationLockQueueWaiting,
1768 victim,
1769 ArbitrationLockQueueElement *,
1770 link );
1771
1772 // wake the victim
1773 wakeup_thread_with_inheritor(&victim->service->__machPortHoldDestroy, // event
1774 THREAD_AWAKENED, LCK_WAKE_DEFAULT, victim->thread);
1775
1776 // allow this thread to proceed (ie. wait)
1777 success = true; // (put request on wait queue)
1778 break; // (out of while)
1779 } else {
1780 // all the waiting threads we came across in
1781 // finding this loop had the 'required' flag
1782 // set, so we've got a deadlock we can't avoid
1783 panic("I/O Kit: Unrecoverable deadlock.");
1784 }
1785 }
1786 } else {
1787 // repeat while loop, redefining active thread to be the
1788 // thread holding "this other object" (see above), and
1789 // looking for threads waiting on it; note the active
1790 // variable points to "this other object" already... so
1791 // there nothing to do in this else clause.
1792 }
1793 } else { // no, active thread is not waiting for another object
1794 success = true; // (put request on wait queue)
1795 break; // (out of while)
1796 }
1797 } // while forever
1798
1799 if (success) { // put the request on the waiting queue?
1800 kern_return_t wait_result;
1801
1802 // place this thread on the waiting queue and put it to sleep;
1803 // we place it at the tail of the queue...
1804 queue_enter( &gArbitrationLockQueueWaiting,
1805 element,
1806 ArbitrationLockQueueElement *,
1807 link );
1808
1809 // declare that this thread will wait for a given event
1810 restart_sleep:
1811 // unlock global access
1812 // & put thread to sleep, waiting for our event to fire...
1813 wait_result = IOLockSleepWithInheritor(gArbitrationLockQueueLock,
1814 LCK_SLEEP_UNLOCK,
1815 &element->service->__machPortHoldDestroy, // event
1816 owner->thread,
1817 element->required ? THREAD_UNINT : THREAD_INTERRUPTIBLE, TIMEOUT_WAIT_FOREVER);
1818
1819 // ...and we've been woken up; we might be in one of two states:
1820 // (a) we've been aborted and our queue element is not on
1821 // any of the three queues, but is floating around
1822 // (b) we're allowed to proceed with the lock and we have
1823 // already been moved from the waiting queue to the
1824 // active queue.
1825 // ...plus a 3rd state, should the thread have been interrupted:
1826 // (c) we're still on the waiting queue
1827
1828 // determine whether we were interrupted out of our sleep
1829 if (THREAD_INTERRUPTED == wait_result) {
1830 // re-lock global access
1831 IOTakeLock( gArbitrationLockQueueLock );
1832
1833 // determine whether we're still on the waiting queue
1834 found = false;
1835 queue_iterate( &gArbitrationLockQueueWaiting,
1836 waiting, // (reuse waiting queue element)
1837 ArbitrationLockQueueElement *,
1838 link )
1839 {
1840 if (waiting == element) {
1841 found = true;
1842 break;
1843 }
1844 }
1845
1846 if (found) { // yes, we're still on the waiting queue
1847 // determine whether we're willing to fail
1848 if (false == element->required) {
1849 // mark us as aborted
1850 element->aborted = true;
1851
1852 // take us off the waiting queue
1853 queue_remove( &gArbitrationLockQueueWaiting,
1854 element,
1855 ArbitrationLockQueueElement *,
1856 link );
1857 } else { // we are not willing to fail
1858 // ignore interruption, go back to sleep
1859 goto restart_sleep;
1860 }
1861 }
1862
1863 // unlock global access
1864 IOUnlock( gArbitrationLockQueueLock );
1865
1866 // proceed as though this were a normal wake up
1867 wait_result = THREAD_AWAKENED;
1868 }
1869
1870 assert( THREAD_AWAKENED == wait_result );
1871
1872 // determine whether we've been aborted while we were asleep
1873 if (element->aborted) {
1874 assert( false == element->required );
1875 // re-lock global access
1876 IOTakeLock( gArbitrationLockQueueLock );
1877
1878 action = kPutOnFreeQueue;
1879 success = false;
1880 } else { // we weren't aborted, so we must be ready to go :-)
1881 // we've already been moved from waiting to active queue
1882 return true;
1883 }
1884 } else { // the lock request is to be failed
1885 // return unused queue element to queue
1886 action = kPutOnFreeQueue;
1887 }
1888 } else { // it is the same thread, recursive access is allowed
1889 // add one level of recursion
1890 active->count++;
1891
1892 // return unused queue element to queue
1893 action = kPutOnFreeQueue;
1894 success = true;
1895 }
1896 } else { // this object is not already locked, so let this thread through
1897 action = kPutOnActiveQueue;
1898 success = true;
1899 }
1900
1901 // put the new element on a queue
1902 if (kPutOnActiveQueue == action) {
1903 queue_enter( &gArbitrationLockQueueActive,
1904 element,
1905 ArbitrationLockQueueElement *,
1906 link );
1907 } else if (kPutOnFreeQueue == action) {
1908 queue_enter( &gArbitrationLockQueueFree,
1909 element,
1910 ArbitrationLockQueueElement *,
1911 link );
1912 } else {
1913 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1914 }
1915
1916 // unlock global access
1917 IOUnlock( gArbitrationLockQueueLock );
1918
1919 return success;
1920 }
1921
1922 void
unlockForArbitration(void)1923 IOService::unlockForArbitration( void )
1924 {
1925 bool found;
1926 ArbitrationLockQueueElement * element;
1927
1928 // lock global access
1929 IOTakeLock( gArbitrationLockQueueLock );
1930
1931 // find the lock element for this object (ie. on active queue)
1932 found = false;
1933 queue_iterate( &gArbitrationLockQueueActive,
1934 element,
1935 ArbitrationLockQueueElement *,
1936 link )
1937 {
1938 if (element->service == this) {
1939 found = true;
1940 break;
1941 }
1942 }
1943
1944 assert( found );
1945
1946 // determine whether the lock has been taken recursively
1947 if (element->count > 1) {
1948 // undo one level of recursion
1949 element->count--;
1950 } else {
1951 // remove it from the active queue
1952 queue_remove( &gArbitrationLockQueueActive,
1953 element,
1954 ArbitrationLockQueueElement *,
1955 link );
1956
1957 // put it on the free queue
1958 queue_enter( &gArbitrationLockQueueFree,
1959 element,
1960 ArbitrationLockQueueElement *,
1961 link );
1962
1963 // determine whether a thread is waiting for object
1964 thread_t woken;
1965 kern_return_t kr =
1966 wakeup_one_with_inheritor(&__machPortHoldDestroy, // event
1967 THREAD_AWAKENED, LCK_WAKE_DEFAULT, &woken);
1968
1969 if (KERN_SUCCESS == kr) {
1970 found = false;
1971 queue_iterate( &gArbitrationLockQueueWaiting,
1972 element,
1973 ArbitrationLockQueueElement *,
1974 link )
1975 {
1976 if (element->thread == woken) {
1977 found = true;
1978 break;
1979 }
1980 }
1981 assert(found); // we found an interested thread on waiting queue
1982 // remove it from the waiting queue
1983 queue_remove( &gArbitrationLockQueueWaiting,
1984 element,
1985 ArbitrationLockQueueElement *,
1986 link );
1987
1988 // put it on the active queue
1989 queue_enter( &gArbitrationLockQueueActive,
1990 element,
1991 ArbitrationLockQueueElement *,
1992 link );
1993
1994 thread_deallocate(woken);
1995 }
1996 }
1997
1998 // unlock global access
1999 IOUnlock( gArbitrationLockQueueLock );
2000 }
2001
2002 uint32_t
isLockedForArbitration(IOService * service)2003 IOService::isLockedForArbitration(IOService * service)
2004 {
2005 #if DEBUG_NOTIFIER_LOCKED
2006 uint32_t count;
2007 ArbitrationLockQueueElement * active;
2008
2009 // lock global access
2010 IOLockLock(gArbitrationLockQueueLock);
2011
2012 // determine whether this object is already locked (ie. on active queue)
2013 count = 0;
2014 queue_iterate(&gArbitrationLockQueueActive,
2015 active,
2016 ArbitrationLockQueueElement *,
2017 link)
2018 {
2019 if ((active->thread == IOThreadSelf())
2020 && (!service || (active->service == service))) {
2021 count += 0x10000;
2022 count += active->count;
2023 }
2024 }
2025
2026 IOLockUnlock(gArbitrationLockQueueLock);
2027
2028 return count;
2029
2030 #else /* DEBUG_NOTIFIER_LOCKED */
2031
2032 return 0;
2033
2034 #endif /* DEBUG_NOTIFIER_LOCKED */
2035 }
2036
2037 void
setMachPortHoldDestroy(bool holdDestroy)2038 IOService::setMachPortHoldDestroy(bool holdDestroy)
2039 {
2040 __machPortHoldDestroy = holdDestroy;
2041 }
2042
2043 bool
machPortHoldDestroy()2044 IOService::machPortHoldDestroy()
2045 {
2046 return __machPortHoldDestroy;
2047 }
2048
2049 void
applyToProviders(IOServiceApplierFunction applier,void * context)2050 IOService::applyToProviders( IOServiceApplierFunction applier,
2051 void * context )
2052 {
2053 applyToParents((IORegistryEntryApplierFunction) applier,
2054 context, gIOServicePlane );
2055 }
2056
2057 void
applyToClients(IOServiceApplierFunction applier,void * context)2058 IOService::applyToClients( IOServiceApplierFunction applier,
2059 void * context )
2060 {
2061 applyToChildren((IORegistryEntryApplierFunction) applier,
2062 context, gIOServicePlane );
2063 }
2064
2065
2066 static void
IOServiceApplierToBlock(IOService * next,void * context)2067 IOServiceApplierToBlock(IOService * next, void * context)
2068 {
2069 IOServiceApplierBlock block = (IOServiceApplierBlock) context;
2070 block(next);
2071 }
2072
2073 void
applyToProviders(IOServiceApplierBlock applier)2074 IOService::applyToProviders(IOServiceApplierBlock applier)
2075 {
2076 applyToProviders(&IOServiceApplierToBlock, applier);
2077 }
2078
2079 void
applyToClients(IOServiceApplierBlock applier)2080 IOService::applyToClients(IOServiceApplierBlock applier)
2081 {
2082 applyToClients(&IOServiceApplierToBlock, applier);
2083 }
2084
2085 /*
2086 * Client messages
2087 */
2088
2089
2090 // send a message to a client or interested party of this service
2091 IOReturn
messageClient(UInt32 type,OSObject * client,void * argument,vm_size_t argSize)2092 IOService::messageClient( UInt32 type, OSObject * client,
2093 void * argument, vm_size_t argSize )
2094 {
2095 IOReturn ret;
2096 IOService * service;
2097 _IOServiceInterestNotifier * notify;
2098
2099 if ((service = OSDynamicCast( IOService, client))) {
2100 ret = service->message( type, this, argument );
2101 } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
2102 _IOServiceNotifierInvocation invocation;
2103 bool willNotify;
2104
2105 invocation.thread = current_thread();
2106
2107 LOCKWRITENOTIFY();
2108 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2109
2110 if (willNotify) {
2111 queue_enter( ¬ify->handlerInvocations, &invocation,
2112 _IOServiceNotifierInvocation *, link );
2113 }
2114 UNLOCKNOTIFY();
2115
2116 if (willNotify) {
2117 ret = (*notify->handler)( notify->target, notify->ref,
2118 type, this, argument, argSize );
2119
2120 LOCKWRITENOTIFY();
2121 queue_remove( ¬ify->handlerInvocations, &invocation,
2122 _IOServiceNotifierInvocation *, link );
2123 if (kIOServiceNotifyWaiter & notify->state) {
2124 notify->state &= ~kIOServiceNotifyWaiter;
2125 WAKEUPNOTIFY( notify );
2126 }
2127 UNLOCKNOTIFY();
2128 } else {
2129 ret = kIOReturnSuccess;
2130 }
2131 } else {
2132 ret = kIOReturnBadArgument;
2133 }
2134
2135 return ret;
2136 }
2137
2138 static void
applyToInterestNotifiers(const IORegistryEntry * target,const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2139 applyToInterestNotifiers(const IORegistryEntry *target,
2140 const OSSymbol * typeOfInterest,
2141 OSObjectApplierFunction applier,
2142 void * context )
2143 {
2144 OSArray * copyArray = NULL;
2145 OSObject * prop;
2146
2147 LOCKREADNOTIFY();
2148
2149 prop = target->copyProperty(typeOfInterest);
2150 IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
2151
2152 if (notifyList) {
2153 copyArray = OSArray::withCapacity(1);
2154
2155 // iterate over queue, entry is set to each element in the list
2156 iterqueue(¬ifyList->fCommandChain, entry) {
2157 _IOServiceInterestNotifier * notify;
2158
2159 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2160 copyArray->setObject(notify);
2161 }
2162 }
2163 UNLOCKNOTIFY();
2164
2165 if (copyArray) {
2166 unsigned int index;
2167 OSObject * next;
2168
2169 for (index = 0; (next = copyArray->getObject( index )); index++) {
2170 (*applier)(next, context);
2171 }
2172 copyArray->release();
2173 }
2174
2175 OSSafeReleaseNULL(prop);
2176 }
2177
2178 void
applyToInterested(const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2179 IOService::applyToInterested( const OSSymbol * typeOfInterest,
2180 OSObjectApplierFunction applier,
2181 void * context )
2182 {
2183 if (gIOGeneralInterest == typeOfInterest) {
2184 applyToClients((IOServiceApplierFunction) applier, context );
2185 }
2186 applyToInterestNotifiers(this, typeOfInterest, applier, context);
2187 }
2188
2189 struct MessageClientsContext {
2190 IOService * service;
2191 UInt32 type;
2192 void * argument;
2193 vm_size_t argSize;
2194 IOReturn ret;
2195 };
2196
2197 static void
messageClientsApplier(OSObject * object,void * ctx)2198 messageClientsApplier( OSObject * object, void * ctx )
2199 {
2200 IOReturn ret;
2201 MessageClientsContext * context = (MessageClientsContext *) ctx;
2202
2203 ret = context->service->messageClient( context->type,
2204 object, context->argument, context->argSize );
2205 if (kIOReturnSuccess != ret) {
2206 context->ret = ret;
2207 }
2208 }
2209
2210 // send a message to all clients
2211 IOReturn
messageClients(UInt32 type,void * argument,vm_size_t argSize)2212 IOService::messageClients( UInt32 type,
2213 void * argument, vm_size_t argSize )
2214 {
2215 MessageClientsContext context;
2216
2217 context.service = this;
2218 context.type = type;
2219 context.argument = argument;
2220 context.argSize = argSize;
2221 context.ret = kIOReturnSuccess;
2222
2223 applyToInterested( gIOGeneralInterest,
2224 &messageClientsApplier, &context );
2225
2226 return context.ret;
2227 }
2228
2229 IOReturn
acknowledgeNotification(IONotificationRef notification,IOOptionBits response)2230 IOService::acknowledgeNotification( IONotificationRef notification,
2231 IOOptionBits response )
2232 {
2233 return kIOReturnUnsupported;
2234 }
2235
2236 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2237 IOService::registerInterest( const OSSymbol * typeOfInterest,
2238 IOServiceInterestHandler handler, void * target, void * ref )
2239 {
2240 _IOServiceInterestNotifier * notify = NULL;
2241 IOReturn rc = kIOReturnError;
2242
2243 notify = new _IOServiceInterestNotifier;
2244 if (!notify) {
2245 return NULL;
2246 }
2247
2248 if (notify->init()) {
2249 rc = registerInterestForNotifier(notify, typeOfInterest,
2250 handler, target, ref);
2251 }
2252
2253 if (rc != kIOReturnSuccess) {
2254 notify->release();
2255 notify = NULL;
2256 }
2257
2258 return notify;
2259 }
2260
2261
2262
2263 static IOReturn
IOServiceInterestHandlerToBlock(void * target __unused,void * refCon,UInt32 messageType,IOService * provider,void * messageArgument,vm_size_t argSize)2264 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2265 UInt32 messageType, IOService * provider,
2266 void * messageArgument, vm_size_t argSize )
2267 {
2268 return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2269 }
2270
2271 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandlerBlock handler)2272 IOService::registerInterest(const OSSymbol * typeOfInterest,
2273 IOServiceInterestHandlerBlock handler)
2274 {
2275 IONotifier * notify;
2276 void * block;
2277
2278 block = Block_copy(handler);
2279 if (!block) {
2280 return NULL;
2281 }
2282
2283 notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
2284
2285 if (!notify) {
2286 Block_release(block);
2287 }
2288
2289 return notify;
2290 }
2291
2292 IOReturn
registerInterestForNotifier(IONotifier * svcNotify,const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2293 IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2294 IOServiceInterestHandler handler, void * target, void * ref )
2295 {
2296 IOReturn rc = kIOReturnSuccess;
2297 _IOServiceInterestNotifier *notify = NULL;
2298
2299
2300 if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)) || !handler) {
2301 return kIOReturnBadArgument;
2302 }
2303
2304 notify->handler = handler;
2305 notify->target = target;
2306 notify->ref = ref;
2307
2308 if ((typeOfInterest != gIOGeneralInterest)
2309 && (typeOfInterest != gIOBusyInterest)
2310 && (typeOfInterest != gIOAppPowerStateInterest)
2311 && (typeOfInterest != gIOConsoleSecurityInterest)
2312 && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2313 return kIOReturnBadArgument;
2314 }
2315
2316 lockForArbitration();
2317 if (0 == (__state[0] & kIOServiceInactiveState)) {
2318 notify->state = kIOServiceNotifyEnable;
2319
2320 ////// queue
2321
2322 LOCKWRITENOTIFY();
2323
2324 // Get the head of the notifier linked list
2325 IOCommand * notifyList;
2326 OSObject * obj = copyProperty( typeOfInterest );
2327 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2328 notifyList = OSTypeAlloc(IOCommand);
2329 if (notifyList) {
2330 notifyList->init();
2331 bool ok = setProperty( typeOfInterest, notifyList);
2332 notifyList->release();
2333 if (!ok) {
2334 notifyList = NULL;
2335 }
2336 }
2337 }
2338 if (obj) {
2339 obj->release();
2340 }
2341
2342 if (notifyList) {
2343 enqueue(¬ifyList->fCommandChain, ¬ify->chain);
2344 notify->retain(); // ref'ed while in list
2345 }
2346
2347 UNLOCKNOTIFY();
2348 } else {
2349 rc = kIOReturnNotReady;
2350 }
2351 unlockForArbitration();
2352
2353 return rc;
2354 }
2355
2356 static void
cleanInterestList(OSObject * head)2357 cleanInterestList( OSObject * head )
2358 {
2359 IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2360 if (!notifyHead) {
2361 return;
2362 }
2363
2364 LOCKWRITENOTIFY();
2365 while (queue_entry_t entry = dequeue(¬ifyHead->fCommandChain)) {
2366 queue_next(entry) = queue_prev(entry) = NULL;
2367
2368 _IOServiceInterestNotifier * notify;
2369
2370 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2371 notify->release();
2372 }
2373 UNLOCKNOTIFY();
2374 }
2375
2376 void
unregisterAllInterest(void)2377 IOService::unregisterAllInterest( void )
2378 {
2379 OSObject * prop;
2380
2381 prop = copyProperty(gIOGeneralInterest);
2382 cleanInterestList(prop);
2383 OSSafeReleaseNULL(prop);
2384
2385 prop = copyProperty(gIOBusyInterest);
2386 cleanInterestList(prop);
2387 OSSafeReleaseNULL(prop);
2388
2389 prop = copyProperty(gIOAppPowerStateInterest);
2390 cleanInterestList(prop);
2391 OSSafeReleaseNULL(prop);
2392
2393 prop = copyProperty(gIOPriorityPowerStateInterest);
2394 cleanInterestList(prop);
2395 OSSafeReleaseNULL(prop);
2396
2397 prop = copyProperty(gIOConsoleSecurityInterest);
2398 cleanInterestList(prop);
2399 OSSafeReleaseNULL(prop);
2400 }
2401
2402 /*
2403 * _IOServiceInterestNotifier
2404 */
2405
2406 // wait for all threads, other than the current one,
2407 // to exit the handler
2408
2409 void
wait()2410 _IOServiceInterestNotifier::wait()
2411 {
2412 _IOServiceNotifierInvocation * next;
2413 bool doWait;
2414
2415 do {
2416 doWait = false;
2417 queue_iterate( &handlerInvocations, next,
2418 _IOServiceNotifierInvocation *, link) {
2419 if (next->thread != current_thread()) {
2420 doWait = true;
2421 break;
2422 }
2423 }
2424 if (doWait) {
2425 state |= kIOServiceNotifyWaiter;
2426 SLEEPNOTIFY(this);
2427 }
2428 } while (doWait);
2429 }
2430
2431 void
free()2432 _IOServiceInterestNotifier::free()
2433 {
2434 assert( queue_empty( &handlerInvocations ));
2435
2436 if (handler == &IOServiceInterestHandlerToBlock) {
2437 Block_release(ref);
2438 }
2439
2440 OSObject::free();
2441 }
2442
2443 void
remove()2444 _IOServiceInterestNotifier::remove()
2445 {
2446 LOCKWRITENOTIFY();
2447
2448 if (queue_next( &chain )) {
2449 remqueue(&chain);
2450 queue_next( &chain) = queue_prev( &chain) = NULL;
2451 release();
2452 }
2453
2454 state &= ~kIOServiceNotifyEnable;
2455
2456 wait();
2457
2458 UNLOCKNOTIFY();
2459
2460 release();
2461 }
2462
2463 bool
disable()2464 _IOServiceInterestNotifier::disable()
2465 {
2466 bool ret;
2467
2468 LOCKWRITENOTIFY();
2469
2470 ret = (0 != (kIOServiceNotifyEnable & state));
2471 state &= ~kIOServiceNotifyEnable;
2472 if (ret) {
2473 wait();
2474 }
2475
2476 UNLOCKNOTIFY();
2477
2478 return ret;
2479 }
2480
2481 void
enable(bool was)2482 _IOServiceInterestNotifier::enable( bool was )
2483 {
2484 LOCKWRITENOTIFY();
2485 if (was) {
2486 state |= kIOServiceNotifyEnable;
2487 } else {
2488 state &= ~kIOServiceNotifyEnable;
2489 }
2490 UNLOCKNOTIFY();
2491 }
2492
2493 bool
init()2494 _IOServiceInterestNotifier::init()
2495 {
2496 queue_init( &handlerInvocations );
2497 return OSObject::init();
2498 }
2499 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2500
2501 /*
2502 * Termination
2503 */
2504
2505 #define tailQ(o) setObject(o)
2506 #define headQ(o) setObject(0, o)
2507 #define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2508
2509 static void
_workLoopAction(IOWorkLoop::Action action,IOService * service,void * p0=NULL,void * p1=NULL,void * p2=NULL,void * p3=NULL)2510 _workLoopAction( IOWorkLoop::Action action,
2511 IOService * service,
2512 void * p0 = NULL, void * p1 = NULL,
2513 void * p2 = NULL, void * p3 = NULL )
2514 {
2515 IOWorkLoop * wl;
2516
2517 if ((wl = service->getWorkLoop())) {
2518 wl->retain();
2519 wl->runAction( action, service, p0, p1, p2, p3 );
2520 wl->release();
2521 } else {
2522 (*action)( service, p0, p1, p2, p3 );
2523 }
2524 }
2525
2526 bool
requestTerminate(IOService * provider,IOOptionBits options)2527 IOService::requestTerminate( IOService * provider, IOOptionBits options )
2528 {
2529 bool ok;
2530
2531 // if its our only provider
2532 ok = isParent( provider, gIOServicePlane, true);
2533
2534 // -- compat
2535 if (ok) {
2536 provider->terminateClient( this, options | kIOServiceRecursing );
2537 ok = (0 != (kIOServiceInactiveState & __state[0]));
2538 }
2539 // --
2540
2541 return ok;
2542 }
2543
2544 bool
terminatePhase1(IOOptionBits options)2545 IOService::terminatePhase1( IOOptionBits options )
2546 {
2547 IOService * victim;
2548 IOService * client;
2549 IOService * rematchProvider;
2550 OSIterator * iter;
2551 OSArray * makeInactive;
2552 OSArray * waitingInactive;
2553 IOOptionBits callerOptions;
2554 int waitResult = THREAD_AWAKENED;
2555 bool wait;
2556 bool ok;
2557 bool didInactive;
2558 bool startPhase2 = false;
2559
2560 TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2561
2562 callerOptions = options;
2563 rematchProvider = NULL;
2564 uint64_t regID = getRegistryEntryID();
2565 IOServiceTrace(
2566 IOSERVICE_TERMINATE_PHASE1,
2567 (uintptr_t) regID,
2568 (uintptr_t) (regID >> 32),
2569 (uintptr_t) this,
2570 (uintptr_t) options);
2571
2572 // -- compat
2573 if (options & kIOServiceRecursing) {
2574 lockForArbitration();
2575 if (0 == (kIOServiceInactiveState & __state[0])) {
2576 __state[0] |= kIOServiceInactiveState;
2577 __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2578 }
2579 unlockForArbitration();
2580
2581 return true;
2582 }
2583 // --
2584
2585 makeInactive = OSArray::withCapacity( 16 );
2586 waitingInactive = OSArray::withCapacity( 16 );
2587 if (!makeInactive || !waitingInactive) {
2588 OSSafeReleaseNULL(makeInactive);
2589 OSSafeReleaseNULL(waitingInactive);
2590 return false;
2591 }
2592
2593 victim = this;
2594 victim->retain();
2595
2596 while (victim) {
2597 didInactive = victim->lockForArbitration( true );
2598 if (didInactive) {
2599 uint64_t regID1 = victim->getRegistryEntryID();
2600 IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2601 (uintptr_t) regID1,
2602 (uintptr_t) (regID1 >> 32),
2603 (uintptr_t) victim->__state[1],
2604 (uintptr_t) 0);
2605
2606 enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2607 didInactive = (kRP1 == (victim->__state[1] & kRP1))
2608 || (0 == (victim->__state[0] & kIOServiceInactiveState));
2609
2610 if (!didInactive) {
2611 // a multiply attached IOService can be visited twice
2612 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) {
2613 do{
2614 IOLockLock(gIOServiceBusyLock);
2615 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2616 if (wait) {
2617 TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2618 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2619 victim->__state[1] |= kIOServiceTerm1WaiterState;
2620 victim->unlockForArbitration();
2621 assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2622 }
2623 IOLockUnlock(gIOServiceBusyLock);
2624 if (wait) {
2625 waitResult = thread_block(THREAD_CONTINUE_NULL);
2626 TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2627 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2628 victim->lockForArbitration();
2629 }
2630 }while (wait && (waitResult != THREAD_TIMED_OUT));
2631 }
2632 } else {
2633 victim->__state[0] |= kIOServiceInactiveState;
2634 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2635 | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2636 victim->__state[1] &= ~kIOServiceRecursing;
2637 victim->__state[1] |= kIOServiceTermPhase1State;
2638 waitingInactive->headQ(victim);
2639 if (victim == this) {
2640 if (kIOServiceTerminateNeedWillTerminate & options) {
2641 victim->__state[1] |= kIOServiceNeedWillTerminate;
2642 }
2643 }
2644 victim->_adjustBusy( 1 );
2645
2646 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2647 if ((options & kIOServiceTerminateWithRematchCurrentDext)) {
2648 OSObject * obj;
2649 OSObject * rematchProps;
2650 OSNumber * num;
2651 uint32_t count;
2652
2653 rematchProvider = getProvider();
2654 if (rematchProvider) {
2655 obj = rematchProvider->copyProperty(gIORematchCountKey);
2656 num = OSDynamicCast(OSNumber, obj);
2657 count = 0;
2658 if (num) {
2659 count = num->unsigned32BitValue();
2660 count++;
2661 }
2662 num = OSNumber::withNumber(count, 32);
2663 rematchProvider->setProperty(gIORematchCountKey, num);
2664 rematchProps = copyProperty(gIOMatchedPersonalityKey);
2665 rematchProvider->setProperty(gIORematchPersonalityKey, rematchProps);
2666 OSSafeReleaseNULL(num);
2667 OSSafeReleaseNULL(rematchProps);
2668 OSSafeReleaseNULL(obj);
2669 }
2670 }
2671 victim->__state[1] |= kIOServiceRematchOnDetach;
2672 }
2673 }
2674 victim->unlockForArbitration();
2675 }
2676 if (victim == this) {
2677 options &= ~(kIOServiceTerminateWithRematch | kIOServiceTerminateWithRematchCurrentDext);
2678 startPhase2 = didInactive;
2679 }
2680 if (didInactive) {
2681 OSArray * notifiers;
2682 notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2683 victim->invokeNotifiers(¬ifiers);
2684
2685 IOUserClient::destroyUserReferences( victim );
2686
2687 iter = victim->getClientIterator();
2688 if (iter) {
2689 while ((client = (IOService *) iter->getNextObject())) {
2690 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2691 client->getName(), client->getRegistryEntryID(),
2692 victim->getName(), victim->getRegistryEntryID(), (long long)options);
2693 ok = client->requestTerminate( victim, options );
2694 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2695 client->getName(), client->getRegistryEntryID(),
2696 victim->getName(), victim->getRegistryEntryID(), ok);
2697
2698 uint64_t regID1 = client->getRegistryEntryID();
2699 uint64_t regID2 = victim->getRegistryEntryID();
2700 IOServiceTrace(
2701 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2702 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2703 (uintptr_t) regID1,
2704 (uintptr_t) (regID1 >> 32),
2705 (uintptr_t) regID2,
2706 (uintptr_t) (regID2 >> 32));
2707
2708 if (ok) {
2709 makeInactive->setObject( client );
2710 }
2711 }
2712 iter->release();
2713 }
2714 }
2715 victim->release();
2716 victim = (IOService *) makeInactive->getObject(0);
2717 if (victim) {
2718 victim->retain();
2719 makeInactive->removeObject(0);
2720 }
2721 }
2722
2723 makeInactive->release();
2724
2725 while ((victim = (IOService *) waitingInactive->getObject(0))) {
2726 victim->retain();
2727 waitingInactive->removeObject(0);
2728
2729 victim->lockForArbitration();
2730 victim->__state[1] &= ~kIOServiceTermPhase1State;
2731 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2732 victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2733 TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2734 IOLockLock( gIOServiceBusyLock );
2735 thread_wakeup((event_t) &victim->__state[1]);
2736 IOLockUnlock( gIOServiceBusyLock );
2737 }
2738 victim->unlockForArbitration();
2739 victim->release();
2740 }
2741
2742 waitingInactive->release();
2743
2744 if (startPhase2) {
2745 retain();
2746 lockForArbitration();
2747 scheduleTerminatePhase2(options);
2748 unlockForArbitration();
2749 release();
2750 }
2751
2752 if (rematchProvider) {
2753 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2754 }
2755
2756 return true;
2757 }
2758
2759 void
setTerminateDefer(IOService * provider,bool defer)2760 IOService::setTerminateDefer(IOService * provider, bool defer)
2761 {
2762 lockForArbitration();
2763 if (defer) {
2764 __state[1] |= kIOServiceStartState;
2765 } else {
2766 __state[1] &= ~kIOServiceStartState;
2767 }
2768 unlockForArbitration();
2769
2770 if (provider && !defer) {
2771 provider->lockForArbitration();
2772 provider->scheduleTerminatePhase2();
2773 provider->unlockForArbitration();
2774 }
2775 }
2776
2777 // Must call this while holding gJobsLock
2778 void
waitToBecomeTerminateThread(void)2779 IOService::waitToBecomeTerminateThread(void)
2780 {
2781 IOLockAssert(gJobsLock, kIOLockAssertOwned);
2782 bool wait;
2783 do {
2784 wait = (gIOTerminateThread != THREAD_NULL);
2785 if (wait) {
2786 IOLockSleepWithInheritor(
2787 gJobsLock,
2788 LCK_SLEEP_DEFAULT,
2789 &gIOTerminateThread,
2790 gIOTerminateThread,
2791 THREAD_UNINT,
2792 TIMEOUT_WAIT_FOREVER);
2793 }
2794 } while (wait);
2795 gIOTerminateThread = current_thread();
2796 }
2797
2798 // call with lockForArbitration
2799 void
scheduleTerminatePhase2(IOOptionBits options)2800 IOService::scheduleTerminatePhase2( IOOptionBits options )
2801 {
2802 AbsoluteTime deadline;
2803 uint64_t regID1;
2804 int waitResult = THREAD_AWAKENED;
2805 bool wait = false, haveDeadline = false;
2806
2807 if (!(__state[0] & kIOServiceInactiveState)) {
2808 return;
2809 }
2810
2811 regID1 = getRegistryEntryID();
2812 IOServiceTrace(
2813 IOSERVICE_TERM_SCHED_PHASE2,
2814 (uintptr_t) regID1,
2815 (uintptr_t) (regID1 >> 32),
2816 (uintptr_t) __state[1],
2817 (uintptr_t) options);
2818
2819 if (__state[1] & kIOServiceTermPhase1State) {
2820 return;
2821 }
2822
2823 retain();
2824 unlockForArbitration();
2825 options |= kIOServiceRequired;
2826 IOLockLock( gJobsLock );
2827
2828 if ((options & kIOServiceSynchronous)
2829 && (current_thread() != gIOTerminateThread)) {
2830 waitToBecomeTerminateThread();
2831 gIOTerminatePhase2List->setObject( this );
2832 gIOTerminateWork++;
2833
2834 do {
2835 while (gIOTerminateWork) {
2836 terminateWorker( options );
2837 }
2838 wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2839 if (wait) {
2840 /* wait for the victim to go non-busy */
2841 if (!haveDeadline) {
2842 clock_interval_to_deadline( 15, kSecondScale, &deadline );
2843 haveDeadline = true;
2844 }
2845 /* let others do work while we wait */
2846 gIOTerminateThread = NULL;
2847 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2848 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2849 deadline, THREAD_UNINT );
2850 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2851 IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2852 "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2853 }
2854 waitToBecomeTerminateThread();
2855 }
2856 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2857
2858 gIOTerminateThread = NULL;
2859 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2860 } else {
2861 // ! kIOServiceSynchronous
2862
2863 gIOTerminatePhase2List->setObject( this );
2864 if (0 == gIOTerminateWork++) {
2865 assert(gIOTerminateWorkerThread);
2866 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2867 }
2868 }
2869
2870 IOLockUnlock( gJobsLock );
2871 lockForArbitration();
2872 release();
2873 }
2874
2875 __attribute__((__noreturn__))
2876 void
terminateThread(void * arg,wait_result_t waitResult)2877 IOService::terminateThread( void * arg, wait_result_t waitResult )
2878 {
2879 // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2880 IOLockLock(gJobsLock);
2881 while (true) {
2882 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2883 waitToBecomeTerminateThread();
2884 }
2885
2886 while (gIOTerminateWork) {
2887 terminateWorker((IOOptionBits)(uintptr_t)arg );
2888 }
2889
2890 gIOTerminateThread = NULL;
2891 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2892 IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2893 }
2894 }
2895
2896 void
scheduleStop(IOService * provider)2897 IOService::scheduleStop( IOService * provider )
2898 {
2899 uint64_t regID1 = getRegistryEntryID();
2900 uint64_t regID2 = provider->getRegistryEntryID();
2901
2902 TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2903 IOServiceTrace(
2904 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2905 (uintptr_t) regID1,
2906 (uintptr_t) (regID1 >> 32),
2907 (uintptr_t) regID2,
2908 (uintptr_t) (regID2 >> 32));
2909
2910 IOLockLock( gJobsLock );
2911 gIOStopList->tailQ( this );
2912 gIOStopProviderList->tailQ( provider );
2913
2914 if (0 == gIOTerminateWork++) {
2915 assert(gIOTerminateWorkerThread);
2916 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2917 }
2918
2919 IOLockUnlock( gJobsLock );
2920 }
2921
2922 void
scheduleFinalize(bool now)2923 IOService::scheduleFinalize(bool now)
2924 {
2925 uint64_t regID1 = getRegistryEntryID();
2926
2927 TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2928 IOServiceTrace(
2929 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2930 (uintptr_t) regID1,
2931 (uintptr_t) (regID1 >> 32),
2932 0, 0);
2933
2934 if (now || IOUserClient::finalizeUserReferences(this)) {
2935 IOLockLock( gJobsLock );
2936 gIOFinalizeList->tailQ(this);
2937 if (0 == gIOTerminateWork++) {
2938 assert(gIOTerminateWorkerThread);
2939 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2940 }
2941 IOLockUnlock( gJobsLock );
2942 }
2943 }
2944
2945 bool
willTerminate(IOService * provider,IOOptionBits options)2946 IOService::willTerminate( IOService * provider, IOOptionBits options )
2947 {
2948 if (reserved->uvars) {
2949 IOUserServer::serviceWillTerminate(this, provider, options);
2950 }
2951 return true;
2952 }
2953
2954 bool
didTerminate(IOService * provider,IOOptionBits options,bool * defer)2955 IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2956 {
2957 if (reserved->uvars) {
2958 IOUserServer::serviceDidTerminate(this, provider, options, defer);
2959 }
2960
2961 if (false == *defer) {
2962 if (lockForArbitration( true )) {
2963 if (false == provider->handleIsOpen( this )) {
2964 scheduleStop( provider );
2965 }
2966 // -- compat
2967 else {
2968 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2969 if (false == provider->handleIsOpen( this )) {
2970 scheduleStop( provider );
2971 }
2972 }
2973 // --
2974 unlockForArbitration();
2975 }
2976 }
2977
2978 return true;
2979 }
2980
2981 void
actionWillTerminate(IOService * victim,IOOptionBits options,OSArray * doPhase2List,bool user,void * unused3 __unused)2982 IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2983 OSArray * doPhase2List,
2984 bool user,
2985 void *unused3 __unused)
2986 {
2987 OSIterator * iter;
2988 IOService * client;
2989 bool ok;
2990 uint64_t regID1, regID2 = victim->getRegistryEntryID();
2991
2992 iter = victim->getClientIterator();
2993 if (iter) {
2994 while ((client = (IOService *) iter->getNextObject())) {
2995 if (user != (NULL != client->reserved->uvars)) {
2996 continue;
2997 }
2998 regID1 = client->getRegistryEntryID();
2999 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3000 client->getName(), regID1,
3001 victim->getName(), regID2, (long long)options);
3002 IOServiceTrace(
3003 IOSERVICE_TERMINATE_WILL,
3004 (uintptr_t) regID1,
3005 (uintptr_t) (regID1 >> 32),
3006 (uintptr_t) regID2,
3007 (uintptr_t) (regID2 >> 32));
3008
3009 ok = client->willTerminate( victim, options );
3010 doPhase2List->tailQ( client );
3011 }
3012 iter->release();
3013 }
3014 }
3015
3016 void
actionDidTerminate(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3017 IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
3018 void *unused1 __unused, void *unused2 __unused,
3019 void *unused3 __unused )
3020 {
3021 OSIterator * iter;
3022 IOService * client;
3023 bool defer;
3024 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3025
3026 victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
3027
3028 iter = victim->getClientIterator();
3029 if (iter) {
3030 while ((client = (IOService *) iter->getNextObject())) {
3031 regID1 = client->getRegistryEntryID();
3032 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3033 client->getName(), regID1,
3034 victim->getName(), regID2, (long long)options);
3035 defer = false;
3036 client->didTerminate( victim, options, &defer );
3037
3038 IOServiceTrace(
3039 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3040 : IOSERVICE_TERMINATE_DID),
3041 (uintptr_t) regID1,
3042 (uintptr_t) (regID1 >> 32),
3043 (uintptr_t) regID2,
3044 (uintptr_t) (regID2 >> 32));
3045
3046 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3047 client->getName(), regID1,
3048 victim->getName(), regID2, defer);
3049 }
3050 iter->release();
3051 }
3052 }
3053
3054
3055 void
actionWillStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3056 IOService::actionWillStop( IOService * victim, IOOptionBits options,
3057 void *unused1 __unused, void *unused2 __unused,
3058 void *unused3 __unused )
3059 {
3060 OSIterator * iter;
3061 IOService * provider;
3062 bool ok;
3063 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3064
3065 iter = victim->getProviderIterator();
3066 if (iter) {
3067 while ((provider = (IOService *) iter->getNextObject())) {
3068 regID1 = provider->getRegistryEntryID();
3069 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3070 victim->getName(), regID2,
3071 provider->getName(), regID1, (long long)options);
3072 IOServiceTrace(
3073 IOSERVICE_TERMINATE_WILL,
3074 (uintptr_t) regID2,
3075 (uintptr_t) (regID2 >> 32),
3076 (uintptr_t) regID1,
3077 (uintptr_t) (regID1 >> 32));
3078
3079 ok = victim->willTerminate( provider, options );
3080 }
3081 iter->release();
3082 }
3083 }
3084
3085 void
actionDidStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3086 IOService::actionDidStop( IOService * victim, IOOptionBits options,
3087 void *unused1 __unused, void *unused2 __unused,
3088 void *unused3 __unused )
3089 {
3090 OSIterator * iter;
3091 IOService * provider;
3092 bool defer = false;
3093 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3094
3095 iter = victim->getProviderIterator();
3096 if (iter) {
3097 while ((provider = (IOService *) iter->getNextObject())) {
3098 regID1 = provider->getRegistryEntryID();
3099 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3100 victim->getName(), regID2,
3101 provider->getName(), regID1, (long long)options);
3102 victim->didTerminate( provider, options, &defer );
3103
3104 IOServiceTrace(
3105 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3106 : IOSERVICE_TERMINATE_DID),
3107 (uintptr_t) regID2,
3108 (uintptr_t) (regID2 >> 32),
3109 (uintptr_t) regID1,
3110 (uintptr_t) (regID1 >> 32));
3111
3112 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3113 victim->getName(), regID2,
3114 provider->getName(), regID1, defer);
3115 }
3116 iter->release();
3117 }
3118 }
3119
3120
3121 void
actionFinalize(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3122 IOService::actionFinalize( IOService * victim, IOOptionBits options,
3123 void *unused1 __unused, void *unused2 __unused,
3124 void *unused3 __unused )
3125 {
3126 uint64_t regID1 = victim->getRegistryEntryID();
3127 TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
3128 IOServiceTrace(
3129 IOSERVICE_TERMINATE_FINALIZE,
3130 (uintptr_t) regID1,
3131 (uintptr_t) (regID1 >> 32),
3132 0, 0);
3133
3134 victim->finalize( options );
3135 }
3136
3137 void
actionStop(IOService * provider,IOService * client,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3138 IOService::actionStop( IOService * provider, IOService * client,
3139 void *unused1 __unused, void *unused2 __unused,
3140 void *unused3 __unused )
3141 {
3142 uint64_t regID1 = provider->getRegistryEntryID();
3143 uint64_t regID2 = client->getRegistryEntryID();
3144
3145 TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3146 IOServiceTrace(
3147 IOSERVICE_TERMINATE_STOP,
3148 (uintptr_t) regID1,
3149 (uintptr_t) (regID1 >> 32),
3150 (uintptr_t) regID2,
3151 (uintptr_t) (regID2 >> 32));
3152
3153 client->stop( provider );
3154 if (provider->isOpen( client )) {
3155 provider->close( client );
3156 }
3157
3158 TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3159 client->detach( provider );
3160 }
3161
3162 #pragma clang diagnostic push
3163 #pragma clang diagnostic ignored "-Wcast-function-type"
3164
3165 void
terminateWorker(IOOptionBits options)3166 IOService::terminateWorker( IOOptionBits options )
3167 {
3168 OSArray * doPhase2List;
3169 OSArray * didPhase2List;
3170 OSSet * freeList;
3171 OSIterator * iter;
3172 UInt32 workDone;
3173 IOService * victim;
3174 IOService * client;
3175 IOService * provider;
3176 unsigned int idx;
3177 bool moreToDo;
3178 bool doPhase2;
3179 bool doPhase3;
3180
3181 options |= kIOServiceRequired;
3182
3183 doPhase2List = OSArray::withCapacity( 16 );
3184 didPhase2List = OSArray::withCapacity( 16 );
3185 freeList = OSSet::withCapacity( 16 );
3186 if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
3187 OSSafeReleaseNULL(doPhase2List);
3188 OSSafeReleaseNULL(didPhase2List);
3189 OSSafeReleaseNULL(freeList);
3190 return;
3191 }
3192
3193 do {
3194 workDone = gIOTerminateWork;
3195
3196 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(0))) {
3197 victim->retain();
3198 gIOTerminatePhase2List->removeObject(0);
3199 IOLockUnlock( gJobsLock );
3200
3201 uint64_t regID1 = victim->getRegistryEntryID();
3202 IOServiceTrace(
3203 IOSERVICE_TERM_START_PHASE2,
3204 (uintptr_t) regID1,
3205 (uintptr_t) (regID1 >> 32),
3206 (uintptr_t) 0,
3207 (uintptr_t) 0);
3208
3209 while (victim) {
3210 doPhase2 = victim->lockForArbitration( true );
3211 if (doPhase2) {
3212 doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
3213 if (doPhase2) {
3214 uint64_t regID1 = victim->getRegistryEntryID();
3215 IOServiceTrace(
3216 IOSERVICE_TERM_TRY_PHASE2,
3217 (uintptr_t) regID1,
3218 (uintptr_t) (regID1 >> 32),
3219 (uintptr_t) victim->__state[1],
3220 (uintptr_t) 0);
3221
3222 doPhase2 = (0 == (victim->__state[1] &
3223 (kIOServiceTermPhase1State
3224 | kIOServiceTermPhase2State
3225 | kIOServiceConfigState)));
3226
3227 if (doPhase2 && (iter = victim->getClientIterator())) {
3228 while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3229 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3230 if (!doPhase2) {
3231 uint64_t regID1 = client->getRegistryEntryID();
3232 IOServiceTrace(
3233 IOSERVICE_TERM_UC_DEFER,
3234 (uintptr_t) regID1,
3235 (uintptr_t) (regID1 >> 32),
3236 (uintptr_t) client->__state[1],
3237 (uintptr_t) 0);
3238 TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3239 victim->getName(), victim->getRegistryEntryID(),
3240 client->getName(), client->getRegistryEntryID());
3241 }
3242 }
3243 iter->release();
3244 }
3245 if (doPhase2) {
3246 victim->__state[1] |= kIOServiceTermPhase2State;
3247 }
3248 }
3249 victim->unlockForArbitration();
3250 }
3251 if (doPhase2) {
3252 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3253 if (NULL == victim->reserved->uvars) {
3254 _workLoopAction((IOWorkLoop::Action) &actionWillStop,
3255 victim, (void *)(uintptr_t) options);
3256 } else {
3257 actionWillStop(victim, options, NULL, NULL, NULL);
3258 }
3259 }
3260
3261 OSArray * notifiers;
3262 notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
3263 victim->invokeNotifiers(¬ifiers);
3264
3265 _workLoopAction((IOWorkLoop::Action) &actionWillTerminate,
3266 victim,
3267 (void *)(uintptr_t) options,
3268 (void *)(uintptr_t) doPhase2List,
3269 (void *)(uintptr_t) false);
3270
3271 actionWillTerminate(
3272 victim, options, doPhase2List, true, NULL);
3273
3274 didPhase2List->headQ( victim );
3275 }
3276 victim->release();
3277 victim = (IOService *) doPhase2List->getObject(0);
3278 if (victim) {
3279 victim->retain();
3280 doPhase2List->removeObject(0);
3281 }
3282 }
3283
3284 while ((victim = (IOService *) didPhase2List->getObject(0))) {
3285 bool scheduleFinalize = false;
3286 if (victim->lockForArbitration( true )) {
3287 victim->__state[1] |= kIOServiceTermPhase3State;
3288 scheduleFinalize = (NULL == victim->getClient());
3289 victim->unlockForArbitration();
3290 }
3291 _workLoopAction((IOWorkLoop::Action) &actionDidTerminate,
3292 victim, (void *)(uintptr_t) options );
3293 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3294 _workLoopAction((IOWorkLoop::Action) &actionDidStop,
3295 victim, (void *)(uintptr_t) options, NULL );
3296 }
3297 // no clients - will go to finalize
3298 if (scheduleFinalize) {
3299 victim->scheduleFinalize(false);
3300 }
3301 didPhase2List->removeObject(0);
3302 }
3303 IOLockLock( gJobsLock );
3304 }
3305
3306 // phase 3
3307 do {
3308 doPhase3 = false;
3309 // finalize leaves
3310 while ((victim = (IOService *) gIOFinalizeList->getObject(0))) {
3311 bool sendFinal = false;
3312 IOLockUnlock( gJobsLock );
3313 if (victim->lockForArbitration(true)) {
3314 sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3315 if (sendFinal) {
3316 victim->__state[1] |= kIOServiceFinalized;
3317 }
3318 victim->unlockForArbitration();
3319 }
3320 if (sendFinal) {
3321 _workLoopAction((IOWorkLoop::Action) &actionFinalize,
3322 victim, (void *)(uintptr_t) options );
3323 }
3324 IOLockLock( gJobsLock );
3325 // hold off free
3326 freeList->setObject( victim );
3327 // safe if finalize list is append only
3328 gIOFinalizeList->removeObject(0);
3329 }
3330
3331 for (idx = 0;
3332 (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx));) {
3333 provider = (IOService *) gIOStopProviderList->getObject(idx);
3334 assert( provider );
3335
3336 uint64_t regID1 = provider->getRegistryEntryID();
3337 uint64_t regID2 = client->getRegistryEntryID();
3338
3339 if (!provider->isChild( client, gIOServicePlane )) {
3340 // may be multiply queued - nop it
3341 TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3342 IOServiceTrace(
3343 IOSERVICE_TERMINATE_STOP_NOP,
3344 (uintptr_t) regID1,
3345 (uintptr_t) (regID1 >> 32),
3346 (uintptr_t) regID2,
3347 (uintptr_t) (regID2 >> 32));
3348 } else {
3349 // a terminated client is not ready for stop if it has clients, skip it
3350 bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3351 IOLockUnlock( gJobsLock );
3352 if (deferStop && client->lockForArbitration(true)) {
3353 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3354 //deferStop = (!deferStop && (0 != client->getClient()));
3355 //deferStop = (0 != client->getClient());
3356 client->unlockForArbitration();
3357 if (deferStop) {
3358 TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3359 IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3360 (uintptr_t) regID1,
3361 (uintptr_t) (regID1 >> 32),
3362 (uintptr_t) regID2,
3363 (uintptr_t) (regID2 >> 32));
3364
3365 idx++;
3366 IOLockLock( gJobsLock );
3367 continue;
3368 }
3369 }
3370 _workLoopAction((IOWorkLoop::Action) &actionStop,
3371 provider, (void *) client );
3372 IOLockLock( gJobsLock );
3373 // check the finalize list now
3374 doPhase3 = true;
3375 }
3376 // hold off free
3377 freeList->setObject( client );
3378 freeList->setObject( provider );
3379
3380 // safe if stop list is append only
3381 gIOStopList->removeObject( idx );
3382 gIOStopProviderList->removeObject( idx );
3383 idx = 0;
3384 }
3385 } while (doPhase3);
3386
3387 gIOTerminateWork -= workDone;
3388 moreToDo = (gIOTerminateWork != 0);
3389
3390 if (!moreToDo) {
3391 TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3392 IOServiceTrace(
3393 IOSERVICE_TERMINATE_DONE,
3394 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3395 }
3396 } while (moreToDo);
3397
3398 IOLockUnlock( gJobsLock );
3399
3400 freeList->release();
3401 doPhase2List->release();
3402 didPhase2List->release();
3403
3404 IOLockLock( gJobsLock );
3405 }
3406
3407 #pragma clang diagnostic pop
3408
3409 bool
finalize(IOOptionBits options)3410 IOService::finalize( IOOptionBits options )
3411 {
3412 OSIterator * iter;
3413 IOService * provider;
3414 uint64_t regID1, regID2 = getRegistryEntryID();
3415
3416 iter = getProviderIterator();
3417 assert( iter );
3418
3419 if (iter) {
3420 while ((provider = (IOService *) iter->getNextObject())) {
3421 // -- compat
3422 if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3423 /* we come down here on programmatic terminate */
3424
3425 regID1 = provider->getRegistryEntryID();
3426 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3427 IOServiceTrace(
3428 IOSERVICE_TERMINATE_STOP,
3429 (uintptr_t) regID1,
3430 (uintptr_t) (regID1 >> 32),
3431 (uintptr_t) regID2,
3432 (uintptr_t) (regID2 >> 32));
3433
3434 stop( provider );
3435 if (provider->isOpen( this )) {
3436 provider->close( this );
3437 }
3438 detach( provider );
3439 } else {
3440 //--
3441 if (provider->lockForArbitration( true )) {
3442 if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3443 scheduleStop( provider );
3444 }
3445 provider->unlockForArbitration();
3446 }
3447 }
3448 }
3449 iter->release();
3450 }
3451
3452 return true;
3453 }
3454
3455 #undef tailQ
3456 #undef headQ
3457
3458 /*
3459 * Terminate
3460 */
3461
3462 void
doServiceTerminate(IOOptionBits options)3463 IOService::doServiceTerminate( IOOptionBits options )
3464 {
3465 }
3466
3467 // a method in case someone needs to override it
3468 bool
terminateClient(IOService * client,IOOptionBits options)3469 IOService::terminateClient( IOService * client, IOOptionBits options )
3470 {
3471 bool ok;
3472
3473 if (client->isParent( this, gIOServicePlane, true)) {
3474 // we are the clients only provider
3475 ok = client->terminate( options );
3476 } else {
3477 ok = true;
3478 }
3479
3480 return ok;
3481 }
3482
3483 bool
terminate(IOOptionBits options)3484 IOService::terminate( IOOptionBits options )
3485 {
3486 options |= kIOServiceTerminate;
3487
3488 return terminatePhase1( options );
3489 }
3490
3491 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3492
3493 /*
3494 * Open & close
3495 */
3496
3497 struct ServiceOpenMessageContext {
3498 IOService * service;
3499 UInt32 type;
3500 IOService * excludeClient;
3501 IOOptionBits options;
3502 };
3503
3504 static void
serviceOpenMessageApplier(OSObject * object,void * ctx)3505 serviceOpenMessageApplier( OSObject * object, void * ctx )
3506 {
3507 ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3508
3509 if (object != context->excludeClient) {
3510 context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
3511 }
3512 }
3513
3514 bool
open(IOService * forClient,IOOptionBits options,void * arg)3515 IOService::open( IOService * forClient,
3516 IOOptionBits options,
3517 void * arg )
3518 {
3519 bool ok;
3520 kern_return_t ret = kIOReturnSuccess;
3521 ServiceOpenMessageContext context;
3522
3523 context.service = this;
3524 context.type = kIOMessageServiceIsAttemptingOpen;
3525 context.excludeClient = forClient;
3526 context.options = options;
3527
3528 applyToInterested( gIOGeneralInterest,
3529 &serviceOpenMessageApplier, &context );
3530
3531 if (false == lockForArbitration(false)) {
3532 return false;
3533 }
3534
3535 ok = (0 == (__state[0] & kIOServiceInactiveState));
3536
3537 if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3538 ret = forClient->reserved->uvars->userServer->serviceOpen(this, forClient);
3539 if (ret != kIOReturnSuccess) {
3540 ok = false;
3541 }
3542 }
3543
3544 if (ok) {
3545 ok = handleOpen( forClient, options, arg );
3546
3547 if (!ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3548 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3549 }
3550 }
3551
3552 unlockForArbitration();
3553
3554 return ok;
3555 }
3556
3557 void
close(IOService * forClient,IOOptionBits options)3558 IOService::close( IOService * forClient,
3559 IOOptionBits options )
3560 {
3561 bool wasClosed;
3562 bool last = false;
3563
3564 lockForArbitration();
3565
3566 wasClosed = handleIsOpen( forClient );
3567 if (wasClosed) {
3568 handleClose( forClient, options );
3569 last = (__state[1] & kIOServiceTermPhase3State);
3570
3571 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3572 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3573 }
3574 }
3575
3576 unlockForArbitration();
3577
3578 if (last) {
3579 forClient->scheduleStop( this );
3580 } else if (wasClosed) {
3581 ServiceOpenMessageContext context;
3582
3583 context.service = this;
3584 context.type = kIOMessageServiceWasClosed;
3585 context.excludeClient = forClient;
3586 context.options = options;
3587
3588 applyToInterested( gIOGeneralInterest,
3589 &serviceOpenMessageApplier, &context );
3590 }
3591 }
3592
3593 bool
isOpen(const IOService * forClient) const3594 IOService::isOpen( const IOService * forClient ) const
3595 {
3596 IOService * self = (IOService *) this;
3597 bool ok;
3598
3599 self->lockForArbitration();
3600
3601 ok = handleIsOpen( forClient );
3602
3603 self->unlockForArbitration();
3604
3605 return ok;
3606 }
3607
3608 bool
handleOpen(IOService * forClient,IOOptionBits options,void * arg)3609 IOService::handleOpen( IOService * forClient,
3610 IOOptionBits options,
3611 void * arg )
3612 {
3613 bool ok;
3614
3615 ok = (NULL == __owner);
3616 if (ok) {
3617 __owner = forClient;
3618 } else if (options & kIOServiceSeize) {
3619 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3620 __owner, (void *)(uintptr_t) options ));
3621 if (ok && (NULL == __owner)) {
3622 __owner = forClient;
3623 } else {
3624 ok = false;
3625 }
3626 }
3627 return ok;
3628 }
3629
3630 void
handleClose(IOService * forClient,IOOptionBits options)3631 IOService::handleClose( IOService * forClient,
3632 IOOptionBits options )
3633 {
3634 if (__owner == forClient) {
3635 __owner = NULL;
3636 }
3637 }
3638
3639 bool
handleIsOpen(const IOService * forClient) const3640 IOService::handleIsOpen( const IOService * forClient ) const
3641 {
3642 if (forClient) {
3643 return __owner == forClient;
3644 } else {
3645 return __owner != forClient;
3646 }
3647 }
3648
3649 /*
3650 * Probing & starting
3651 */
3652 static SInt32
IONotifyOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3653 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3654 {
3655 const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3656 const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3657 SInt32 val1;
3658 SInt32 val2;
3659
3660 val1 = 0;
3661 val2 = 0;
3662 if (obj1) {
3663 val1 = obj1->priority;
3664 }
3665 if (obj2) {
3666 val2 = obj2->priority;
3667 }
3668 if (val1 > val2) {
3669 return 1;
3670 }
3671 if (val1 < val2) {
3672 return -1;
3673 }
3674 return 0;
3675 }
3676
3677 static SInt32
IOServiceObjectOrder(const OSObject * entry,void * ref)3678 IOServiceObjectOrder( const OSObject * entry, void * ref)
3679 {
3680 OSDictionary * dict;
3681 IOService * service;
3682 _IOServiceNotifier * notify;
3683 OSSymbol * key = (OSSymbol *) ref;
3684 OSNumber * offset;
3685 OSObject * prop;
3686 SInt32 result;
3687
3688 prop = NULL;
3689 result = kIODefaultProbeScore;
3690 if ((dict = OSDynamicCast( OSDictionary, entry))) {
3691 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3692 } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3693 return notify->priority;
3694 } else if ((service = OSDynamicCast( IOService, entry))) {
3695 prop = service->copyProperty(key);
3696 offset = OSDynamicCast(OSNumber, prop);
3697 } else {
3698 assert( false );
3699 offset = NULL;
3700 }
3701
3702 if (offset) {
3703 result = offset->unsigned32BitValue();
3704 }
3705
3706 OSSafeReleaseNULL(prop);
3707
3708 return result;
3709 }
3710
3711 __attribute__((no_sanitize("signed-integer-overflow"))) SInt32
IOServiceOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3712 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3713 {
3714 const OSObject * obj1 = (const OSObject *) inObj1;
3715 const OSObject * obj2 = (const OSObject *) inObj2;
3716 SInt32 val1;
3717 SInt32 val2;
3718
3719 val1 = 0;
3720 val2 = 0;
3721
3722 if (obj1) {
3723 val1 = IOServiceObjectOrder( obj1, ref );
3724 }
3725
3726 if (obj2) {
3727 val2 = IOServiceObjectOrder( obj2, ref );
3728 }
3729
3730 return val1 - val2;
3731 }
3732
3733 IOService *
copyClientWithCategory(const OSSymbol * category)3734 IOService::copyClientWithCategory( const OSSymbol * category )
3735 {
3736 IOService * service = NULL;
3737 OSIterator * iter;
3738 const OSSymbol * nextCat;
3739
3740 iter = getClientIterator();
3741 if (iter) {
3742 while ((service = (IOService *) iter->getNextObject())) {
3743 if (kIOServiceInactiveState & service->__state[0]) {
3744 if (!(kIOServiceRematchOnDetach & service->__state[1])) {
3745 continue;
3746 }
3747 }
3748 nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3749 service->getProperty( gIOMatchCategoryKey ));
3750 if (category == nextCat) {
3751 service->retain();
3752 break;
3753 }
3754 }
3755 iter->release();
3756 }
3757 return service;
3758 }
3759
3760 IOService *
getClientWithCategory(const OSSymbol * category)3761 IOService::getClientWithCategory( const OSSymbol * category )
3762 {
3763 IOService *
3764 service = copyClientWithCategory(category);
3765 if (service) {
3766 service->release();
3767 }
3768 return service;
3769 }
3770
3771 bool
invokeNotifier(_IOServiceNotifier * notify)3772 IOService::invokeNotifier( _IOServiceNotifier * notify )
3773 {
3774 _IOServiceNotifierInvocation invocation;
3775 bool willNotify;
3776 bool ret = true;
3777 invocation.thread = current_thread();
3778
3779 #if DEBUG_NOTIFIER_LOCKED
3780 uint32_t count;
3781 if ((count = isLockedForArbitration(0))) {
3782 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3783 panic("[%s, 0x%x]", notify->type->getCStringNoCopy(), count);
3784 }
3785 #endif /* DEBUG_NOTIFIER_LOCKED */
3786
3787 LOCKWRITENOTIFY();
3788 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3789
3790 if (willNotify) {
3791 queue_enter( ¬ify->handlerInvocations, &invocation,
3792 _IOServiceNotifierInvocation *, link );
3793 }
3794 UNLOCKNOTIFY();
3795
3796 if (willNotify) {
3797 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3798
3799 LOCKWRITENOTIFY();
3800 queue_remove( ¬ify->handlerInvocations, &invocation,
3801 _IOServiceNotifierInvocation *, link );
3802 if (kIOServiceNotifyWaiter & notify->state) {
3803 notify->state &= ~kIOServiceNotifyWaiter;
3804 WAKEUPNOTIFY( notify );
3805 }
3806 UNLOCKNOTIFY();
3807 }
3808
3809 return ret;
3810 }
3811
3812 bool
invokeNotifiers(OSArray * willSend[])3813 IOService::invokeNotifiers(OSArray * willSend[])
3814 {
3815 OSArray * array;
3816 _IOServiceNotifier * notify;
3817 bool ret = true;
3818
3819 array = *willSend;
3820 if (!array) {
3821 return true;
3822 }
3823 *willSend = NULL;
3824
3825 for (unsigned int idx = 0;
3826 (notify = (_IOServiceNotifier *) array->getObject(idx));
3827 idx++) {
3828 ret &= invokeNotifier(notify);
3829 }
3830 array->release();
3831
3832 return ret;
3833 }
3834
3835
3836 TUNABLE(bool, iokit_print_verbose_match_logs, "iokit_print_verbose_match_logs", false);
3837
3838 /*
3839 * Alloc and probe matching classes,
3840 * called on the provider instance
3841 */
3842
3843 void
probeCandidates(OSOrderedSet * matches)3844 IOService::probeCandidates( OSOrderedSet * matches )
3845 {
3846 OSDictionary * match = NULL;
3847 OSSymbol * symbol;
3848 IOService * inst;
3849 IOService * newInst;
3850 OSDictionary * props;
3851 SInt32 score;
3852 OSNumber * newPri;
3853 OSOrderedSet * familyMatches = NULL;
3854 OSOrderedSet * startList;
3855 OSSet * kexts = NULL;
3856 OSObject * kextRef;
3857
3858 OSDictionary * startDict = NULL;
3859 const OSSymbol * category;
3860 OSIterator * iter;
3861 _IOServiceNotifier * notify;
3862 OSObject * nextMatch = NULL;
3863 bool started;
3864 bool needReloc = false;
3865 bool matchDeferred = false;
3866 #if IOMATCHDEBUG
3867 SInt64 debugFlags;
3868 #endif
3869 IOService * client = NULL;
3870 OSObject * prop1;
3871 OSObject * rematchCountProp;
3872 OSDictionary * rematchPersonality;
3873 OSNumber * num;
3874 uint32_t count;
3875 uint32_t dextCount;
3876 bool isDext;
3877 bool categoryConsumed;
3878
3879 rematchCountProp = NULL;
3880 count = 0;
3881 prop1 = copyProperty(gIORematchPersonalityKey);
3882 rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3883 if (rematchPersonality) {
3884 rematchCountProp = copyProperty(gIORematchCountKey);
3885 num = OSDynamicCast(OSNumber, rematchCountProp);
3886 if (num) {
3887 count = num->unsigned32BitValue();
3888 }
3889 removeProperty(gIORematchPersonalityKey);
3890 }
3891 dextCount = 0;
3892
3893 assert( matches );
3894 while (!needReloc
3895 && (nextMatch = matches->getFirstObject())) {
3896 nextMatch->retain();
3897 matches->removeObject(nextMatch);
3898
3899 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3900 if (0 == (__state[0] & kIOServiceInactiveState)) {
3901 invokeNotifier( notify );
3902 }
3903 nextMatch->release();
3904 nextMatch = NULL;
3905 continue;
3906 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3907 nextMatch->release();
3908 nextMatch = NULL;
3909 continue;
3910 }
3911
3912 props = NULL;
3913 #if IOMATCHDEBUG
3914 debugFlags = getDebugFlags( match );
3915 #endif
3916
3917 bool newIsBoot = false;
3918 bool existingIsBoot = false;
3919 bool isReplacementCandidate = false;
3920
3921 do {
3922 client = NULL;
3923 isDext = (NULL != match->getObject(gIOUserServerNameKey));
3924 if (isDext && !(kIODKEnable & gIODKDebug)) {
3925 continue;
3926 }
3927 if (isDext && !OSKext::iokitDaemonAvailable()) {
3928 continue;
3929 }
3930 if (isDext && !gIODextRelaunchMax && rematchCountProp) {
3931 continue;
3932 }
3933 newIsBoot = gIOCatalogue->personalityIsBoot(match);
3934
3935 category = OSDynamicCast( OSSymbol,
3936 match->getObject( gIOMatchCategoryKey ));
3937 if (NULL == category) {
3938 category = gIODefaultMatchCategoryKey;
3939 }
3940 client = copyClientWithCategory(category);
3941
3942 categoryConsumed = (client != NULL);
3943 if (categoryConsumed) {
3944 #if IOMATCHDEBUG
3945 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3946 LOG("%s: match category %s exists\n", getName(),
3947 category->getCStringNoCopy());
3948 }
3949 #endif
3950 existingIsBoot = client->propertyExists(gIOMatchedAtBootKey);
3951 isReplacementCandidate = existingIsBoot && !newIsBoot;
3952 if (!isDext && !isReplacementCandidate) {
3953 break;
3954 }
3955 }
3956
3957 // create a copy now in case its modified during matching
3958 props = OSDictionary::withDictionary(match, match->getCount());
3959 if (NULL == props) {
3960 break;
3961 }
3962 props->setCapacityIncrement(1);
3963
3964 // check the nub matches
3965 if (false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) {
3966 break;
3967 }
3968 if (isReplacementCandidate) {
3969 if (canTerminateForReplacement(client)) {
3970 client->terminate(kIOServiceTerminateNeedWillTerminate | kIOServiceTerminateWithRematch);
3971 break;
3972 }
3973 }
3974
3975 if (isDext || isReplacementCandidate) {
3976 if (isDext) {
3977 dextCount++;
3978 }
3979 if (categoryConsumed) {
3980 break;
3981 }
3982 }
3983 if (rematchPersonality) {
3984 bool personalityMatch = match->isEqualTo(rematchPersonality);
3985 if (count > gIODextRelaunchMax) {
3986 personalityMatch = !personalityMatch;
3987 }
3988 if (!personalityMatch) {
3989 break;
3990 }
3991 }
3992
3993 // Check to see if driver reloc has been loaded.
3994 needReloc = (false == gIOCatalogue->isModuleLoaded( match, &kextRef ));
3995 if (needReloc) {
3996 #if IOMATCHDEBUG
3997 if (debugFlags & kIOLogCatalogue) {
3998 LOG("%s: stalling for module\n", getName());
3999 }
4000 #endif
4001 // If reloc hasn't been loaded, exit;
4002 // reprobing will occur after reloc has been loaded.
4003 break;
4004 }
4005 if (kextRef) {
4006 if (NULL == kexts) {
4007 kexts = OSSet::withCapacity(1);
4008 }
4009 if (kexts) {
4010 kexts->setObject(kextRef);
4011 kextRef->release();
4012 }
4013 }
4014 if (newIsBoot) {
4015 props->setObject(gIOMatchedAtBootKey, kOSBooleanTrue);
4016 }
4017 if (isDext) {
4018 // copy saved for rematchng
4019 props->setObject(gIOMatchedPersonalityKey, match);
4020 }
4021 // reorder on family matchPropertyTable score.
4022 if (NULL == familyMatches) {
4023 familyMatches = OSOrderedSet::withCapacity( 1,
4024 IOServiceOrdering, (void *) gIOProbeScoreKey );
4025 }
4026 if (familyMatches) {
4027 familyMatches->setObject( props );
4028 }
4029 } while (false);
4030
4031 OSSafeReleaseNULL(client);
4032 OSSafeReleaseNULL(nextMatch);
4033 OSSafeReleaseNULL(props);
4034 }
4035 OSSafeReleaseNULL(matches);
4036 OSSafeReleaseNULL(rematchCountProp);
4037
4038 if (familyMatches) {
4039 while (!needReloc
4040 && (props = (OSDictionary *) familyMatches->getFirstObject())) {
4041 props->retain();
4042 familyMatches->removeObject( props );
4043
4044 inst = NULL;
4045 newInst = NULL;
4046 #if IOMATCHDEBUG
4047 debugFlags = getDebugFlags( props );
4048 #endif
4049 do {
4050 symbol = OSDynamicCast( OSSymbol,
4051 props->getObject( gIOClassKey));
4052 if (!symbol) {
4053 continue;
4054 }
4055
4056 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
4057
4058 // alloc the driver instance
4059 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
4060
4061 if (!inst || !OSDynamicCast(IOService, inst)) {
4062 IOLog("Couldn't alloc class \"%s\"\n",
4063 symbol->getCStringNoCopy());
4064 continue;
4065 }
4066
4067 // init driver instance
4068 if (!(inst->init( props ))) {
4069 #if IOMATCHDEBUG
4070 if (debugFlags & kIOLogStart) {
4071 IOLog("%s::init fails\n", symbol->getCStringNoCopy());
4072 }
4073 #endif
4074 continue;
4075 }
4076 if (__state[1] & kIOServiceSynchronousState) {
4077 inst->__state[1] |= kIOServiceSynchronousState;
4078 }
4079
4080 // give the driver the default match category if not specified
4081 category = OSDynamicCast( OSSymbol,
4082 props->getObject( gIOMatchCategoryKey ));
4083 if (NULL == category) {
4084 category = gIODefaultMatchCategoryKey;
4085 }
4086 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
4087 // attach driver instance
4088 if (!(inst->attach( this ))) {
4089 continue;
4090 }
4091
4092 // pass in score from property table
4093 score = familyMatches->orderObject( props );
4094
4095 // & probe the new driver instance
4096 #if IOMATCHDEBUG
4097 if (debugFlags & kIOLogProbe) {
4098 LOG("%s::probe(%s)\n",
4099 inst->getMetaClass()->getClassName(), getName());
4100 }
4101 #endif
4102 newInst = inst->probe( this, &score );
4103 inst->detach( this );
4104 if (NULL == newInst) {
4105 #if IOMATCHDEBUG
4106 if (debugFlags & kIOLogProbe) {
4107 IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
4108 }
4109 #endif
4110 continue;
4111 }
4112
4113 // save the score
4114 newPri = OSNumber::withNumber( score, 32 );
4115 if (newPri) {
4116 newInst->setProperty( gIOProbeScoreKey, newPri );
4117 newPri->release();
4118 }
4119
4120 // add to start list for the match category
4121 if (NULL == startDict) {
4122 startDict = OSDictionary::withCapacity( 1 );
4123 }
4124 assert( startDict );
4125 startList = (OSOrderedSet *)
4126 startDict->getObject( category );
4127 if (NULL == startList) {
4128 startList = OSOrderedSet::withCapacity( 1,
4129 IOServiceOrdering, (void *) gIOProbeScoreKey );
4130 if (startDict && startList) {
4131 startDict->setObject( category, startList );
4132 startList->release();
4133 }
4134 }
4135 assert( startList );
4136 if (startList) {
4137 startList->setObject( newInst );
4138 }
4139 } while (false);
4140
4141 props->release();
4142 if (inst) {
4143 inst->release();
4144 }
4145 }
4146 familyMatches->release();
4147 familyMatches = NULL;
4148 }
4149
4150 if ((debugFlags & kIOLogMatch) && iokit_print_verbose_match_logs && startDict != NULL) {
4151 IOLog("%s(0x%qx): %u categories\n", getName(), getRegistryEntryID(), startList->getCount());
4152 startDict->iterateObjects(^(const OSSymbol *key, OSObject *value) {
4153 OSOrderedSet *startList = OSDynamicCast(OSOrderedSet, value);
4154 if (startList) {
4155 IOLog("%s(0x%qx): category %s, %u matches\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), startList->getCount());
4156 startList->iterateObjects(^(OSObject *obj) {
4157 IOService *match = OSDynamicCast(IOService, obj);
4158 OSNumber *probeScore = OSDynamicCast(OSNumber, match->getProperty(gIOProbeScoreKey));
4159
4160 if (match && probeScore) {
4161 IOLog("%s(0x%qx): category %s: matched %s, probe score %qd\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), match->getName(), probeScore->unsigned64BitValue());
4162 }
4163 return false;
4164 });
4165 }
4166 return false;
4167 });
4168 }
4169
4170 // start the best (until success) of each category
4171
4172 iter = OSCollectionIterator::withCollection( startDict );
4173 assert(startDict || !iter);
4174 if (iter) {
4175 while ((category = (const OSSymbol *) iter->getNextObject())) {
4176 startList = (OSOrderedSet *) startDict->getObject( category );
4177 assert( startList );
4178 if (!startList) {
4179 continue;
4180 }
4181 started = false;
4182 while (true // (!started)
4183 && !matchDeferred
4184 && (inst = (IOService *)startList->getFirstObject())) {
4185 inst->retain();
4186 startList->removeObject(inst);
4187 #if IOMATCHDEBUG
4188 debugFlags = getDebugFlags( inst );
4189
4190 if (debugFlags & kIOLogStart) {
4191 if (started) {
4192 LOG( "match category exists, skipping " );
4193 }
4194 LOG( "%s::start(%s) <%d>\n", inst->getName(),
4195 getName(), inst->getRetainCount());
4196 }
4197 #endif
4198 if (false == started) {
4199 #if !NO_KEXTD
4200 IOLockLock(gJobsLock);
4201 matchDeferred = (gIOMatchDeferList
4202 && kOSBooleanTrue == inst->getProperty(gIOMatchDeferKey));
4203 if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(this, 0))) {
4204 gIOMatchDeferList->setObject(this);
4205 }
4206 if (matchDeferred) {
4207 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
4208 IOLog("%s(0x%qx): matching deferred by %s%s\n",
4209 getName(), getRegistryEntryID(),
4210 symbol ? symbol->getCStringNoCopy() : "",
4211 gInUserspaceReboot ? " in userspace reboot" : "");
4212 // rematching will occur after the IOKit daemon loads all plists
4213 }
4214 IOLockUnlock(gJobsLock);
4215 #endif
4216 if (!matchDeferred) {
4217 /* TODO
4218 * If a dext fails to start because an upgrade happened
4219 * concurrently, then the matching process has to restart
4220 */
4221 started = startCandidate( inst );
4222 #if IOMATCHDEBUG
4223 if ((debugFlags & kIOLogStart) && (false == started)) {
4224 LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
4225 inst->getRetainCount());
4226 }
4227 #endif
4228 if (!started && inst->propertyExists(gIOServiceMatchDeferredKey)) {
4229 matchDeferred = true;
4230 }
4231 }
4232 }
4233 inst->release();
4234 }
4235 }
4236 iter->release();
4237 }
4238
4239 OSSafeReleaseNULL(prop1);
4240
4241 if (dextCount) {
4242 num = OSNumber::withNumber(dextCount, 32);
4243 setProperty(gIODEXTMatchCountKey, num);
4244 OSSafeReleaseNULL(num);
4245 } else if (rematchPersonality) {
4246 removeProperty(gIODEXTMatchCountKey);
4247 }
4248
4249 // now that instances are created, drop the refs on any kexts allowing unload
4250 if (kexts) {
4251 OSKext::dropMatchingReferences(kexts);
4252 OSSafeReleaseNULL(kexts);
4253 }
4254
4255 // adjust the busy count by +1 if matching is stalled for a module,
4256 // or -1 if a previously stalled matching is complete.
4257 lockForArbitration();
4258 SInt32 adjBusy = 0;
4259 uint64_t regID = getRegistryEntryID();
4260
4261 if (needReloc) {
4262 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
4263 if (adjBusy) {
4264 IOServiceTrace(
4265 IOSERVICE_MODULESTALL,
4266 (uintptr_t) regID,
4267 (uintptr_t) (regID >> 32),
4268 (uintptr_t) this,
4269 0);
4270
4271 __state[1] |= kIOServiceModuleStallState;
4272 }
4273 } else if (__state[1] & kIOServiceModuleStallState) {
4274 IOServiceTrace(
4275 IOSERVICE_MODULEUNSTALL,
4276 (uintptr_t) regID,
4277 (uintptr_t) (regID >> 32),
4278 (uintptr_t) this,
4279 0);
4280
4281 __state[1] &= ~kIOServiceModuleStallState;
4282 adjBusy = -1;
4283 }
4284 if (adjBusy) {
4285 _adjustBusy( adjBusy );
4286 }
4287 unlockForArbitration();
4288
4289 if (startDict) {
4290 startDict->release();
4291 }
4292 }
4293
4294 /*
4295 * Wait for a IOUserServer to check in
4296 */
4297
4298 static
4299 __attribute__((noinline, not_tail_called))
4300 IOUserServer *
__WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)4301 __WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)
4302 {
4303 IOUserServer * result = NULL;
4304 IOService * server = NULL;
4305 const OSSymbol * serverName = token->copyServerName();
4306 OSNumber * serverTag = token->copyServerTag();
4307 OSDictionary * matching = IOService::serviceMatching(gIOUserServerClassKey);
4308
4309 if (!matching || !serverName || !serverTag) {
4310 goto finish;
4311 }
4312 IOService::propertyMatching(gIOUserServerNameKey, serverName, matching);
4313 if (!(kIODKDisableDextTag & gIODKDebug)) {
4314 IOService::propertyMatching(gIOUserServerTagKey, serverTag, matching);
4315 }
4316
4317 server = IOService::waitForMatchingServiceWithToken(matching, kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC, token);
4318 result = OSDynamicCast(IOUserServer, server);
4319 if (!result) {
4320 OSSafeReleaseNULL(server);
4321 token->cancel();
4322 }
4323
4324 finish:
4325 OSSafeReleaseNULL(matching);
4326 OSSafeReleaseNULL(serverName);
4327 OSSafeReleaseNULL(serverTag);
4328
4329 return result;
4330 }
4331
4332 void
willShutdown()4333 IOService::willShutdown()
4334 {
4335 gIOKitWillTerminate = true;
4336 #if !NO_KEXTD
4337 IOUserServerCheckInToken::cancelAll();
4338 #endif
4339 OSKext::willShutdown();
4340 }
4341
4342 void
userSpaceWillReboot()4343 IOService::userSpaceWillReboot()
4344 {
4345 IOLockLock(gJobsLock);
4346 #if !NO_KEXTD
4347 IOService * provider;
4348 IOService * service;
4349 OSIterator * iter;
4350
4351 // Recreate the defer list if it does not exist
4352 if (!gIOMatchDeferList && OSKext::iokitDaemonAvailable()) {
4353 gIOMatchDeferList = OSArray::withCapacity( 16 );
4354 }
4355
4356 if (gIOMatchDeferList) {
4357 iter = IORegistryIterator::iterateOver(gIOServicePlane, kIORegistryIterateRecursively);
4358 if (iter) {
4359 do {
4360 iter->reset();
4361 while ((service = (IOService *)iter->getNextObject())) {
4362 /* Rematch providers of services that will be terminated on userspace reboot, after the userspace reboot
4363 * is complete. This normally happens automatically as the IOKit daemon sends personalities to the kernel
4364 * which triggers rematching. But if this doesn't happen (for example, if a feature flag is turned off),
4365 * then these services will never get rematched.
4366 */
4367 if (service->propertyHasValue(gIOMatchDeferKey, kOSBooleanTrue) || service->hasUserServer()) {
4368 provider = service->getProvider();
4369 IOLog("deferring %s-%llx (provider of %s-%llx) matching after userspace reboot\n",
4370 provider->getName(), provider->getRegistryEntryID(), service->getName(), service->getRegistryEntryID());
4371 gIOMatchDeferList->setObject(provider);
4372 }
4373 }
4374 } while (!service && !iter->isValid());
4375
4376 OSSafeReleaseNULL(iter);
4377 }
4378 }
4379 #endif
4380 gInUserspaceReboot = true;
4381 IOLockUnlock(gJobsLock);
4382 }
4383
4384 void
userSpaceDidReboot()4385 IOService::userSpaceDidReboot()
4386 {
4387 IOLockLock(gJobsLock);
4388 gInUserspaceReboot = false;
4389 IOLockUnlock(gJobsLock);
4390 }
4391
4392 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4393
4394 void
init(IOPMrootDomain * root)4395 IOServicePH::init(IOPMrootDomain * root)
4396 {
4397 fUserServers = OSArray::withCapacity(4);
4398 fMatchingWork = OSArray::withCapacity(4);
4399
4400 assert(fUserServers && fMatchingWork);
4401
4402 fRootNotifier = root->registerInterest(
4403 gIOPriorityPowerStateInterest, &IOServicePH::systemPowerChange, NULL, NULL);
4404
4405 assert(fRootNotifier);
4406
4407 fUserServerAckTimer = thread_call_allocate(&IOServicePH::userServerAckTimerExpired, (thread_call_param_t)NULL);
4408 }
4409
4410 void
lock()4411 IOServicePH::lock()
4412 {
4413 IOLockLock(gJobsLock);
4414 }
4415
4416 void
unlock()4417 IOServicePH::unlock()
4418 {
4419 IOLockUnlock(gJobsLock);
4420 }
4421
4422 void
serverAdd(IOUserServer * server)4423 IOServicePH::serverAdd(IOUserServer * server)
4424 {
4425 uint32_t idx;
4426
4427 lock();
4428 idx = fUserServers->getNextIndexOfObject(server, 0);
4429 if (idx == -1U) {
4430 fUserServers->setObject(server);
4431 }
4432 unlock();
4433 }
4434
4435 void
serverRemove(IOUserServer * server)4436 IOServicePH::serverRemove(IOUserServer * server)
4437 {
4438 uint32_t idx;
4439
4440 lock();
4441 idx = fUserServers->getNextIndexOfObject(server, 0);
4442 if (idx != -1U) {
4443 fUserServers->removeObject(idx);
4444 }
4445
4446 if (fWaitingUserServers) {
4447 fWaitingUserServers = false;
4448 IOLockWakeup(gJobsLock, &fWaitingUserServers, /* one-thread */ false);
4449 }
4450
4451 unlock();
4452 }
4453
4454 void
serverAck(IOUserServer * server)4455 IOServicePH::serverAck(IOUserServer * server)
4456 {
4457 uint32_t idx;
4458 IOService * ackTo;
4459 uint32_t ackToRef;
4460
4461 ackTo = NULL;
4462 lock();
4463 if (server && fUserServersWait) {
4464 idx = fUserServersWait->getNextIndexOfObject(server, 0);
4465 if (idx != -1U) {
4466 fUserServersWait->removeObject(idx);
4467 if (0 == fUserServersWait->getCount()) {
4468 OSSafeReleaseNULL(fUserServersWait);
4469 }
4470 }
4471 }
4472 if (!fUserServersWait && !fMatchingWork->getCount()) {
4473 ackTo = fSystemPowerAckTo;
4474 ackToRef = fSystemPowerAckRef;
4475 fSystemPowerAckTo = NULL;
4476 if (ackTo) {
4477 thread_call_cancel(fUserServerAckTimer);
4478 }
4479 }
4480 if (fUserServersWait && fUserServersWait->getCount() > 0 && fMatchingWork && fMatchingWork->getCount() > 0) {
4481 DKLOG("Waiting for %u user servers, %u matching work\n", fUserServersWait->getCount(), fMatchingWork->getCount());
4482 }
4483 unlock();
4484
4485 if (ackTo) {
4486 DKLOG("allowPowerChange\n");
4487 ackTo->allowPowerChange((uintptr_t) ackToRef);
4488 }
4489 }
4490
4491 bool
matchingStart(IOService * service)4492 IOServicePH::matchingStart(IOService * service)
4493 {
4494 uint32_t idx;
4495 bool assertionActive = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
4496
4497 lock();
4498 bool matchNow = !fSystemOff && assertionActive;
4499 if (matchNow) {
4500 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4501 if (idx == -1U) {
4502 fMatchingWork->setObject(service);
4503 }
4504 } else {
4505 // Delay matching if system is transitioning to sleep
4506 if (!fMatchingDelayed) {
4507 fMatchingDelayed = OSArray::withObjects((const OSObject **) &service, 1, 1);
4508 } else {
4509 idx = fMatchingDelayed->getNextIndexOfObject(service, 0);
4510 if (idx == -1U) {
4511 fMatchingDelayed->setObject(service);
4512 }
4513 }
4514 }
4515 unlock();
4516 if (!matchNow && assertionActive) {
4517 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4518 }
4519
4520 return matchNow;
4521 }
4522
4523 void
matchingEnd(IOService * service)4524 IOServicePH::matchingEnd(IOService * service)
4525 {
4526 uint32_t idx;
4527 OSArray * notifyServers;
4528 OSArray * deferredMatches;
4529
4530 notifyServers = NULL;
4531 deferredMatches = NULL;
4532
4533 if (service) {
4534 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4535 }
4536
4537 lock();
4538
4539 if (service) {
4540 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4541 if (idx != -1U) {
4542 fMatchingWork->removeObject(idx);
4543 }
4544 }
4545
4546
4547 if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4548 if (fSystemOff) {
4549 if (0 == fMatchingWork->getCount()) {
4550 fUserServersWait = OSArray::withArray(fUserServers);
4551 notifyServers = OSArray::withArray(fUserServers);
4552 fUserServerOff = fSystemOff;
4553 }
4554 } else {
4555 notifyServers = OSArray::withArray(fUserServers);
4556 fUserServerOff = fSystemOff;
4557 }
4558 }
4559
4560 if (!fSystemOff && fMatchingDelayed) {
4561 deferredMatches = fMatchingDelayed;
4562 fMatchingDelayed = NULL;
4563 }
4564
4565 unlock();
4566
4567 if (notifyServers) {
4568 uint32_t sleepType = 0;
4569 uint32_t standbyTimer = 0;
4570 bool hibernate = false;
4571 if (fSystemOff && IOService::getPMRootDomain()->getSystemSleepType(&sleepType, &standbyTimer) == kIOReturnSuccess) {
4572 hibernate = (sleepType == kIOPMSleepTypeHibernate);
4573 }
4574 notifyServers->iterateObjects(^bool (OSObject * obj) {
4575 IOUserServer * us;
4576 us = (typeof(us))obj;
4577 us->systemPower(fSystemOff, hibernate);
4578 return false;
4579 });
4580 OSSafeReleaseNULL(notifyServers);
4581 }
4582
4583 if (deferredMatches) {
4584 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4585 deferredMatches->iterateObjects(^bool (OSObject * obj)
4586 {
4587 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
4588 return false;
4589 });
4590 deferredMatches->release();
4591 }
4592
4593 serverAck(NULL);
4594 }
4595
4596 TUNABLE(uint32_t, dk_shutdown_timeout_ms, "dk_shutdown_timeout_ms", 5000);
4597 TUNABLE(bool, dk_panic_on_shutdown_hang, "dk_panic_on_shutdown_hang", false);
4598 TUNABLE(bool, dk_panic_on_setpowerstate_hang, "dk_panic_on_setpowerstate_hang", false);
4599
4600 void
userServerAckTimerExpired(void *,void *)4601 IOServicePH::userServerAckTimerExpired(void *, void *)
4602 {
4603 OSArray * userServers = NULL;
4604 lock();
4605 if (fSystemPowerAckTo) {
4606 DKLOG("ack timer expired\n");
4607 if (dk_panic_on_setpowerstate_hang) {
4608 panic("DK ack timer expired after %u ms", dk_shutdown_timeout_ms);
4609 }
4610 userServers = fUserServersWait;
4611 fUserServersWait = NULL;
4612 }
4613 unlock();
4614
4615 if (userServers != NULL) {
4616 userServers->iterateObjects(^bool (OSObject *obj) {
4617 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4618 if (us) {
4619 DKLOG(DKS " power state transition failed\n", DKN(us));
4620 us->kill("Power Management Failed");
4621 }
4622 return false;
4623 });
4624 OSSafeReleaseNULL(userServers);
4625 }
4626
4627 serverAck(NULL);
4628 }
4629
4630 void
systemHalt(int howto)4631 IOServicePH::systemHalt(int howto)
4632 {
4633 OSArray * notifyServers;
4634 uint64_t deadline;
4635
4636 lock();
4637 notifyServers = OSArray::withArray(fUserServers);
4638 unlock();
4639
4640 if (notifyServers) {
4641 notifyServers->iterateObjects(^bool (OSObject * obj) {
4642 IOUserServer * us;
4643 us = (typeof(us))obj;
4644 us->systemHalt(howto);
4645 return false;
4646 });
4647 OSSafeReleaseNULL(notifyServers);
4648 }
4649
4650 lock();
4651 clock_interval_to_deadline(dk_shutdown_timeout_ms, kMillisecondScale, &deadline);
4652 while (0 < fUserServers->getCount()) {
4653 fWaitingUserServers = true;
4654 __assert_only int waitResult =
4655 IOLockSleepDeadline(gJobsLock, &fWaitingUserServers, deadline, THREAD_UNINT);
4656 assert((THREAD_AWAKENED == waitResult) || (THREAD_TIMED_OUT == waitResult));
4657 if (THREAD_TIMED_OUT == waitResult) {
4658 IOUserServer::beginLeakingObjects();
4659 #if DEVELOPMENT || DEBUG
4660 if (dk_panic_on_shutdown_hang) {
4661 panic("Shutdown timed out waiting for DK drivers to stop");
4662 }
4663 #endif /* DEVELOPMENT || DEBUG */
4664 break;
4665 }
4666 }
4667 unlock();
4668 }
4669
4670 bool
serverSlept(void)4671 IOServicePH::serverSlept(void)
4672 {
4673 bool ret;
4674
4675 lock();
4676 ret = (kIOMessageSystemWillSleep == sSystemPower)
4677 || (kIOMessageSystemWillPowerOff == sSystemPower)
4678 || (kIOMessageSystemWillRestart == sSystemPower);
4679 unlock();
4680
4681 return ret;
4682 }
4683
4684 TUNABLE(uint32_t, dk_power_state_timeout_ms, "dk_power_state_timeout_ms", 30000);
4685
4686 IOReturn
systemPowerChange(void * target,void * refCon,UInt32 messageType,IOService * service,void * messageArgument,vm_size_t argSize)4687 IOServicePH::systemPowerChange(
4688 void * target,
4689 void * refCon,
4690 UInt32 messageType, IOService * service,
4691 void * messageArgument, vm_size_t argSize)
4692 {
4693 IOReturn ret;
4694 IOUserServer * us;
4695 IOPMSystemCapabilityChangeParameters * params;
4696 AbsoluteTime deadline;
4697
4698 us = NULL;
4699
4700 switch (messageType) {
4701 case kIOMessageSystemCapabilityChange:
4702
4703 params = (typeof params)messageArgument;
4704
4705 if (kIODKLogPM & gIODKDebug) {
4706 IOLog("IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4707 params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4708 params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4709 params->fromCapabilities,
4710 params->toCapabilities);
4711 }
4712
4713 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4714 (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4715 ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4716 lock();
4717 DKLOG("arming ack timer, %u ms\n", dk_power_state_timeout_ms);
4718 clock_interval_to_deadline(dk_power_state_timeout_ms, kMillisecondScale, &deadline);
4719 fSystemOff = true;
4720 fSystemPowerAckRef = params->notifyRef;
4721 fSystemPowerAckTo = service;
4722 thread_call_enter_delayed(fUserServerAckTimer, deadline);
4723 unlock();
4724
4725 matchingEnd(NULL);
4726
4727 params->maxWaitForReply = dk_power_state_timeout_ms * 2 * 1000;
4728 ret = kIOReturnSuccess;
4729 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4730 ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4731 (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4732 lock();
4733 fSystemOff = false;
4734 unlock();
4735
4736 matchingEnd(NULL);
4737
4738 params->maxWaitForReply = 0;
4739 ret = kIOReturnSuccess;
4740 } else {
4741 params->maxWaitForReply = 0;
4742 ret = kIOReturnSuccess;
4743 }
4744 break;
4745
4746 default:
4747 ret = kIOReturnUnsupported;
4748 break;
4749 }
4750
4751 return ret;
4752 }
4753
4754 bool
checkPMReady(void)4755 IOServicePH::checkPMReady(void)
4756 {
4757 bool __block ready = true;
4758
4759 lock();
4760 fUserServers->iterateObjects(^bool (OSObject *obj) {
4761 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4762 if (us) {
4763 if (!us->checkPMReady()) {
4764 ready = false;
4765 return true;
4766 }
4767 }
4768 return false;
4769 });
4770 unlock();
4771
4772 return ready;
4773 }
4774
4775 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4776
4777 /*
4778 * Start a previously attached & probed instance,
4779 * called on exporting object instance
4780 */
4781
4782 bool
startCandidate(IOService * service)4783 IOService::startCandidate( IOService * service )
4784 {
4785 bool ok;
4786 OSObject * obj;
4787 OSObject * prop;
4788 IOUserServer * userServer;
4789 bool ph;
4790
4791 userServer = NULL;
4792 obj = service->copyProperty(gIOUserServerNameKey);
4793
4794 if (obj && (this == gIOResources)) {
4795 ok = false;
4796 } else {
4797 ok = service->attach( this );
4798 }
4799 if (!ok) {
4800 OSSafeReleaseNULL(obj);
4801 return false;
4802 }
4803
4804 if ((this != gIOResources) && (this != gIOUserResources)) {
4805 // stall for any nub resources
4806 checkResources();
4807 // stall for any driver resources
4808 service->checkResources();
4809 }
4810 ph = false;
4811 {
4812 OSString * bundleID;
4813 OSString * serverName;
4814 OSString * str;
4815 const OSSymbol * sym;
4816 OSNumber * serverTag;
4817 uint64_t entryID;
4818 IOUserServerCheckInToken * token;
4819 OSData * serverDUI;
4820
4821 if ((serverName = OSDynamicCast(OSString, obj))) {
4822 obj = service->copyProperty(gIOModuleIdentifierKey);
4823 bundleID = OSDynamicCast(OSString, obj);
4824 entryID = service->getRegistryEntryID();
4825 serverTag = OSNumber::withNumber(entryID, 64);
4826 token = NULL;
4827
4828 if (kIODKDisableDextLaunch & gIODKDebug) {
4829 DKLOG(DKS " dext launches are disabled \n", DKN(service));
4830 service->detach(this);
4831 OSSafeReleaseNULL(serverName);
4832 OSSafeReleaseNULL(obj);
4833 OSSafeReleaseNULL(serverTag);
4834 return false;
4835 }
4836
4837 if (gIOKitWillTerminate) {
4838 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4839 service->detach(this);
4840 OSSafeReleaseNULL(serverName);
4841 OSSafeReleaseNULL(obj);
4842 OSSafeReleaseNULL(serverTag);
4843 return false;
4844 }
4845
4846 ph = IOServicePH::matchingStart(this);
4847 if (!ph) {
4848 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4849 service->setProperty(gIOServiceMatchDeferredKey, kOSBooleanTrue);
4850 service->detach(this);
4851 OSSafeReleaseNULL(serverName);
4852 OSSafeReleaseNULL(obj);
4853 OSSafeReleaseNULL(serverTag);
4854 return false;
4855 }
4856
4857 prop = service->copyProperty(gIOUserClassKey);
4858 str = OSDynamicCast(OSString, prop);
4859 if (str) {
4860 service->setName(str);
4861 }
4862 OSSafeReleaseNULL(prop);
4863
4864 sym = OSSymbol::withString(serverName);
4865 bool reuse = service->propertyExists(gIOUserServerOneProcessKey);
4866 serverDUI = OSDynamicCast(OSData, service->getProperty(kOSBundleDextUniqueIdentifierKey));
4867 userServer = IOUserServer::launchUserServer(bundleID, sym, serverTag, reuse, &token, serverDUI);
4868 OSSafeReleaseNULL(sym);
4869 OSSafeReleaseNULL(serverTag);
4870 OSSafeReleaseNULL(serverName);
4871 if (userServer) {
4872 DKLOG(DKS " using existing server " DKS "\n", DKN(service), DKN(userServer));
4873 } else if (token != NULL) {
4874 const OSSymbol * tokenServerName = token->copyServerName();
4875 OSNumber * tokenServerTag = token->copyServerTag();
4876 assert(tokenServerName && tokenServerTag);
4877 DKLOG(DKS " waiting for server %s-%llx\n", DKN(service), tokenServerName->getCStringNoCopy(), tokenServerTag->unsigned64BitValue());
4878 userServer = __WAITING_FOR_USER_SERVER__(token);
4879 OSSafeReleaseNULL(tokenServerName);
4880 OSSafeReleaseNULL(tokenServerTag);
4881 } else {
4882 DKLOG(DKS " failed to launch server\n", DKN(service));
4883 }
4884
4885
4886 if (!userServer) {
4887 service->detach(this);
4888 IOServicePH::matchingEnd(this);
4889 OSSafeReleaseNULL(obj);
4890
4891 if (token != NULL) {
4892 DKLOG(DKS " user server timeout\n", DKN(service));
4893 #if DEVELOPMENT || DEBUG
4894 driverkit_checkin_timed_out = mach_absolute_time();
4895 #endif
4896 }
4897
4898 OSSafeReleaseNULL(token);
4899 return false;
4900 }
4901
4902 if (token && !(kIODKDisableCheckInTokenVerification & gIODKDebug)) {
4903 if (!userServer->serviceMatchesCheckInToken(token)) {
4904 OSSafeReleaseNULL(token);
4905 service->detach(this);
4906 IOServicePH::matchingEnd(this);
4907 OSSafeReleaseNULL(obj);
4908 userServer->exit("Check In Token verification failed");
4909 userServer->release();
4910 return false;
4911 }
4912 }
4913 OSSafeReleaseNULL(token);
4914 OSSafeReleaseNULL(obj);
4915
4916 if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4917 if (!userServer->checkEntitlements(this, service)) {
4918 service->detach(this);
4919 IOServicePH::matchingEnd(this);
4920 userServer->exit("Entitlements check failed");
4921 userServer->release();
4922 return false;
4923 }
4924 }
4925 #if !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG)
4926 // Prevent third party drivers from matching IOUserResources/IOResources, except when signed for development
4927 if (!userServer->isPlatformDriver() &&
4928 userServer->getCSValidationCategory() != CS_VALIDATION_CATEGORY_DEVELOPMENT
4929 && (this == gIOUserResources || this == gIOResources)) {
4930 service->detach(this);
4931 IOServicePH::matchingEnd(this);
4932 userServer->exit("Third party driver may only match real hardware");
4933 userServer->release();
4934 return false;
4935 }
4936 #endif /* !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG) */
4937
4938 userServer->serviceAttach(service, this);
4939 } else {
4940 OSSafeReleaseNULL(obj);
4941 }
4942 }
4943
4944 AbsoluteTime startTime;
4945 AbsoluteTime endTime;
4946 UInt64 nano;
4947 bool recordTime = (kIOLogStart & gIOKitDebug) != 0;
4948
4949 if (recordTime) {
4950 clock_get_uptime(&startTime);
4951 }
4952
4953 ok = service->start(this);
4954
4955 if (recordTime) {
4956 clock_get_uptime(&endTime);
4957
4958 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4959 SUB_ABSOLUTETIME(&endTime, &startTime);
4960 absolutetime_to_nanoseconds(endTime, &nano);
4961 if (nano > 500000000ULL) {
4962 IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
4963 }
4964 }
4965 }
4966 if (userServer) {
4967 userServer->serviceStarted(service, this, ok);
4968 userServer->release();
4969 }
4970
4971 if (ok) {
4972 IOInstallServiceSleepPlatformActions(service);
4973 #if 00
4974 if (!strcmp("XHC1", getName())) {
4975 service->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
4976 }
4977 #endif
4978 }
4979
4980 if (!ok) {
4981 service->detach( this );
4982 }
4983
4984 if (ph) {
4985 IOServicePH::matchingEnd(this);
4986 }
4987
4988 return ok;
4989 }
4990
4991 void
publishResource(const char * key,OSObject * value)4992 IOService::publishResource( const char * key, OSObject * value )
4993 {
4994 const OSSymbol * sym;
4995
4996 if ((sym = OSSymbol::withCString( key))) {
4997 publishResource( sym, value);
4998 sym->release();
4999 }
5000 }
5001
5002 void
publishResource(const OSSymbol * key,OSObject * value)5003 IOService::publishResource( const OSSymbol * key, OSObject * value )
5004 {
5005 if (NULL == value) {
5006 value = (OSObject *) gIOServiceKey;
5007 }
5008
5009 gIOResources->setProperty( key, value);
5010
5011 if (IORecursiveLockHaveLock( gNotificationLock)) {
5012 return;
5013 }
5014
5015 gIOResourceGenerationCount++;
5016 gIOResources->registerService();
5017 }
5018
5019 void
publishUserResource(const OSSymbol * key,OSObject * value)5020 IOService::publishUserResource( const OSSymbol * key, OSObject * value )
5021 {
5022 if (NULL == value) {
5023 value = (OSObject *) gIOServiceKey;
5024 }
5025
5026 gIOUserResources->setProperty( key, value);
5027
5028 if (IORecursiveLockHaveLock( gNotificationLock)) {
5029 return;
5030 }
5031
5032 gIOResourceGenerationCount++;
5033 gIOUserResources->registerService();
5034 }
5035
5036 bool
addNeededResource(const char * key)5037 IOService::addNeededResource( const char * key )
5038 {
5039 OSObject * resourcesProp;
5040 OSSet * set;
5041 OSString * newKey;
5042 bool ret;
5043
5044 resourcesProp = copyProperty( gIOResourceMatchKey );
5045 if (!resourcesProp) {
5046 return false;
5047 }
5048
5049 newKey = OSString::withCString( key );
5050 if (!newKey) {
5051 resourcesProp->release();
5052 return false;
5053 }
5054
5055 set = OSDynamicCast( OSSet, resourcesProp );
5056 if (!set) {
5057 set = OSSet::withCapacity( 1 );
5058 set->setObject( resourcesProp );
5059 } else {
5060 set->retain();
5061 }
5062
5063 set->setObject( newKey );
5064 newKey->release();
5065 ret = setProperty( gIOResourceMatchKey, set );
5066 set->release();
5067 resourcesProp->release();
5068
5069 return ret;
5070 }
5071
5072 bool
checkResource(OSObject * matching)5073 IOService::checkResource( OSObject * matching )
5074 {
5075 OSString * str;
5076 OSDictionary * table;
5077
5078 if ((str = OSDynamicCast( OSString, matching ))) {
5079 if (gIOResources->getProperty( str )) {
5080 return true;
5081 }
5082 }
5083
5084 if (str) {
5085 table = resourceMatching( str );
5086 } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
5087 table->retain();
5088 } else {
5089 IOLog("%s: Can't match using: %s\n", getName(),
5090 matching->getMetaClass()->getClassName());
5091 /* false would stall forever */
5092 return true;
5093 }
5094
5095 if (gIOKitDebug & kIOLogConfig) {
5096 LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
5097 }
5098
5099 waitForService( table );
5100
5101 if (gIOKitDebug & kIOLogConfig) {
5102 LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5103 }
5104
5105 return true;
5106 }
5107
5108 bool
checkResources(void)5109 IOService::checkResources( void )
5110 {
5111 OSObject * resourcesProp;
5112 OSSet * set;
5113 OSObject * obj;
5114 OSIterator * iter;
5115 bool ok;
5116
5117 resourcesProp = copyProperty( gIOResourceMatchKey );
5118 if (NULL == resourcesProp) {
5119 return true;
5120 }
5121
5122 if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
5123 iter = OSCollectionIterator::withCollection( set );
5124 ok = (NULL != iter);
5125 while (ok && (obj = iter->getNextObject())) {
5126 ok = checkResource( obj );
5127 }
5128 if (iter) {
5129 iter->release();
5130 }
5131 } else {
5132 ok = checkResource( resourcesProp );
5133 }
5134
5135 OSSafeReleaseNULL(resourcesProp);
5136
5137 return ok;
5138 }
5139
5140
5141 void
configThread(const char * name)5142 _IOConfigThread::configThread( const char * name )
5143 {
5144 _IOConfigThread * inst;
5145
5146 do {
5147 if (!(inst = new _IOConfigThread)) {
5148 continue;
5149 }
5150 if (!inst->init()) {
5151 continue;
5152 }
5153 thread_t thread;
5154 if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &thread)) {
5155 continue;
5156 }
5157
5158 char threadName[MAXTHREADNAMESIZE];
5159 snprintf(threadName, sizeof(threadName), "IOConfigThread_'%s'", name);
5160 thread_set_thread_name(thread, threadName);
5161 thread_deallocate(thread);
5162
5163 return;
5164 } while (false);
5165
5166 if (inst) {
5167 inst->release();
5168 }
5169
5170 return;
5171 }
5172
5173 /*
5174 * To support driver replacement of boot matched drivers later in boot, drivers can
5175 * opt-in to be being terminated if a non-boot driver matches their provider, by
5176 * setting the gIOPrimaryDriverTerminateOptionsKey property. The driver providing the
5177 * root disk media may not be terminated.
5178 * IOMedia objects are hidden from user space until all drivers are available, but any
5179 * associated with the root disk must be published immediately.
5180 */
5181
5182 struct FindRootMediaContext {
5183 OSArray * services;
5184 IOService * parent;
5185 };
5186
5187 bool
hasParent(IOService * parent)5188 IOService::hasParent(IOService * parent)
5189 {
5190 IOService * service;
5191
5192 for (service = this;
5193 service && (service != parent);
5194 service = service->getProvider()) {
5195 }
5196
5197 return service != NULL;
5198 }
5199
5200 bool
publishHiddenMediaApplier(const OSObject * entry,void * context)5201 IOService::publishHiddenMediaApplier(const OSObject * entry, void * context)
5202 {
5203 FindRootMediaContext * ctx = (typeof(ctx))context;
5204 IOService * service = (typeof(service))entry;
5205
5206 do {
5207 if (ctx->parent && !service->hasParent(ctx->parent)) {
5208 break;
5209 }
5210 if (ctx->services) {
5211 ctx->services->setObject(service);
5212 } else {
5213 ctx->services = OSArray::withObjects((const OSObject **) &service, 1);
5214 assert(ctx->services);
5215 }
5216 } while (false);
5217
5218 return false;
5219 }
5220
5221 // publish to user space any hidden IOMedia under the 'parent' object, or all
5222 // if 'parent' is NULL
5223
5224 void
publishHiddenMedia(IOService * parent)5225 IOService::publishHiddenMedia(IOService * parent)
5226 {
5227 const OSMetaClass * iomediaClass;
5228 bool wasHiding;
5229
5230 iomediaClass = OSMetaClass::getMetaClassWithName(gIOMediaKey);
5231 assert(iomediaClass);
5232
5233 LOCKWRITENOTIFY();
5234 wasHiding = gIOServiceHideIOMedia;
5235 if (wasHiding && !parent) {
5236 gIOServiceHideIOMedia = false;
5237 }
5238 UNLOCKNOTIFY();
5239
5240 FindRootMediaContext ctx = { .services = NULL, .parent = parent };
5241
5242 if (wasHiding) {
5243 iomediaClass->applyToInstances(publishHiddenMediaApplier, &ctx);
5244 }
5245 if (ctx.services) {
5246 unsigned int idx, notiIdx;
5247 IOService * service;
5248 OSArray * notifiers[3] = {};
5249
5250 for (idx = 0; (service = (IOService *) ctx.services->getObject(idx)); idx++) {
5251 service->lockForArbitration(true);
5252 if (!(kIOServiceUserInvisibleMatchState & service->__state[0])) {
5253 service->unlockForArbitration();
5254 continue;
5255 }
5256 service->__state[0] &= ~kIOServiceUserInvisibleMatchState;
5257 service->__state[1] |= kIOServiceUserUnhidden;
5258 notifiers[0] = service->copyNotifiers(gIOFirstPublishNotification, 0, 0xffffffff);
5259 if (kIOServiceMatchedState & service->__state[0]) {
5260 notifiers[1] = service->copyNotifiers(gIOMatchedNotification, 0, 0xffffffff);
5261 }
5262 if (kIOServiceFirstMatchState & service->__state[0]) {
5263 notifiers[2] = service->copyNotifiers(gIOFirstMatchNotification, 0, 0xffffffff);
5264 }
5265 service->unlockForArbitration();
5266 for (notiIdx = 0; notiIdx < 3; notiIdx++) {
5267 service->invokeNotifiers(¬ifiers[notiIdx]);
5268 }
5269 }
5270 OSSafeReleaseNULL(ctx.services);
5271 }
5272 }
5273
5274 // Find the block storage driver providing the root disk, or NULL if not booting from
5275 // a block device
5276
5277 void
setRootMedia(IOService * root)5278 IOService::setRootMedia(IOService * root)
5279 {
5280 const OSMetaClass * ioblockstoragedriverClass;
5281 bool unhide;
5282
5283 ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey);
5284 assert(ioblockstoragedriverClass);
5285
5286 while (root) {
5287 if (root->metaCast(ioblockstoragedriverClass)) {
5288 break;
5289 }
5290 root = root->getProvider();
5291 }
5292
5293 LOCKWRITENOTIFY();
5294 unhide = (kIOServiceRootMediaParentInvalid == gIOServiceRootMediaParent);
5295 if (unhide) {
5296 gIOServiceRootMediaParent = root;
5297 }
5298 UNLOCKNOTIFY();
5299
5300 if (unhide) {
5301 publishHiddenMedia(root);
5302 }
5303 }
5304
5305 // Check if the driver may be terminated when a later driver could be used instead
5306
5307 bool
canTerminateForReplacement(IOService * client)5308 IOService::canTerminateForReplacement(IOService * client)
5309 {
5310 IOService * parent;
5311
5312 assert(kIOServiceRootMediaParentInvalid != gIOServiceRootMediaParent);
5313
5314 if (!client->propertyExists(gIOPrimaryDriverTerminateOptionsKey)) {
5315 return false;
5316 }
5317 if (!gIOServiceRootMediaParent) {
5318 return false;
5319 }
5320 parent = gIOServiceRootMediaParent;
5321 while (parent && (parent != client)) {
5322 parent = parent->getProvider();
5323 }
5324 if (parent) {
5325 IOLog("Can't replace primary matched driver on root media %s-0x%qx\n",
5326 client->getName(), client->getRegistryEntryID());
5327 return false;
5328 }
5329 return true;
5330 }
5331
5332 void
doServiceMatch(IOOptionBits options)5333 IOService::doServiceMatch( IOOptionBits options )
5334 {
5335 _IOServiceNotifier * notify;
5336 OSIterator * iter;
5337 OSOrderedSet * matches;
5338 OSArray * resourceKeys = NULL;
5339 SInt32 catalogGeneration;
5340 bool keepGuessing = true;
5341 bool reRegistered = true;
5342 bool didRegister;
5343 OSArray * notifiers[2] = {NULL};
5344
5345 // job->nub->deliverNotification( gIOPublishNotification,
5346 // kIOServiceRegisteredState, 0xffffffff );
5347
5348 while (keepGuessing) {
5349 matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
5350 // the matches list should always be created by findDrivers()
5351 if (matches) {
5352 lockForArbitration();
5353 if (0 == (__state[0] & kIOServiceFirstPublishState)) {
5354 getMetaClass()->addInstance(this);
5355 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
5356 kIOServiceFirstPublishState, 0xffffffff );
5357 }
5358 LOCKREADNOTIFY();
5359 __state[1] &= ~kIOServiceNeedConfigState;
5360 __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
5361 didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
5362 __state[0] |= kIOServiceRegisteredState;
5363
5364 if (gIOServiceHideIOMedia
5365 && metaCast(gIOMediaKey)
5366 && !(kIOServiceUserUnhidden & __state[1])
5367 && gIOServiceRootMediaParent
5368 && !hasParent(gIOServiceRootMediaParent)
5369 && propertyExists(gIOPrimaryDriverTerminateOptionsKey, gIOServicePlane)) {
5370 __state[0] |= kIOServiceUserInvisibleMatchState;
5371 }
5372
5373 keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
5374 if (reRegistered && keepGuessing) {
5375 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
5376 gNotifications->getObject( gIOPublishNotification ));
5377 if (iter) {
5378 while ((notify = (_IOServiceNotifier *)
5379 iter->getNextObject())) {
5380 if (matchPassive(notify->matching, 0)
5381 && (kIOServiceNotifyEnable & notify->state)) {
5382 matches->setObject( notify );
5383 }
5384 }
5385 iter->release();
5386 }
5387 }
5388
5389 UNLOCKNOTIFY();
5390 unlockForArbitration();
5391 invokeNotifiers(¬ifiers[0]);
5392 if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
5393 if ((this == gIOResources) || (this == gIOUserResources)) {
5394 if (resourceKeys) {
5395 resourceKeys->release();
5396 }
5397 resourceKeys = copyPropertyKeys();
5398 }
5399 probeCandidates( matches );
5400 } else {
5401 matches->release();
5402 }
5403 }
5404
5405 lockForArbitration();
5406 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
5407 keepGuessing =
5408 (reRegistered || (catalogGeneration !=
5409 gIOCatalogue->getGenerationCount()))
5410 && (0 == (__state[0] & kIOServiceInactiveState));
5411
5412 if (keepGuessing) {
5413 unlockForArbitration();
5414 }
5415 }
5416
5417 if ((0 == (__state[0] & kIOServiceInactiveState))
5418 && (0 == (__state[1] & kIOServiceModuleStallState))) {
5419 if (resourceKeys) {
5420 setProperty(gIOResourceMatchedKey, resourceKeys);
5421 }
5422
5423 notifiers[0] = copyNotifiers(gIOMatchedNotification,
5424 kIOServiceMatchedState, 0xffffffff);
5425 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
5426 notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
5427 kIOServiceFirstMatchState, 0xffffffff);
5428 }
5429 }
5430
5431 __state[1] &= ~kIOServiceConfigRunning;
5432 unlockForArbitration();
5433
5434 if (resourceKeys) {
5435 resourceKeys->release();
5436 }
5437
5438 invokeNotifiers(¬ifiers[0]);
5439 invokeNotifiers(¬ifiers[1]);
5440
5441 lockForArbitration();
5442 __state[1] &= ~kIOServiceConfigState;
5443 scheduleTerminatePhase2();
5444
5445 _adjustBusy(-1, /*unlock*/ true);
5446 // does unlockForArbitration();
5447 }
5448
5449 UInt32
_adjustBusy(SInt32 delta)5450 IOService::_adjustBusy(SInt32 delta)
5451 {
5452 return _adjustBusy(delta, false);
5453 }
5454
5455 UInt32
_adjustBusy(SInt32 delta,bool unlock)5456 IOService::_adjustBusy(SInt32 delta, bool unlock)
5457 {
5458 IOService * next;
5459 IOService * nextProvider;
5460 UInt32 count;
5461 UInt32 result;
5462 bool wasQuiet, nowQuiet, needWake;
5463
5464 next = this;
5465 result = __state[1] & kIOServiceBusyStateMask;
5466
5467 if (delta) {
5468 do {
5469 if (next != this) {
5470 next->lockForArbitration();
5471 }
5472 count = next->__state[1] & kIOServiceBusyStateMask;
5473 wasQuiet = (0 == count);
5474 if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
5475 OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), (uint32_t)count, (int)delta);
5476 } else {
5477 count += delta;
5478 }
5479 next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
5480 nowQuiet = (0 == count);
5481 needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
5482
5483 if (needWake) {
5484 next->__state[1] &= ~kIOServiceBusyWaiterState;
5485 IOLockLock( gIOServiceBusyLock );
5486 thread_wakeup((event_t) next);
5487 IOLockUnlock( gIOServiceBusyLock );
5488 }
5489 if (wasQuiet || nowQuiet) {
5490 nextProvider = next->getProvider();
5491 if (nextProvider) {
5492 nextProvider->retain();
5493 }
5494 }
5495 if ((next != this) || unlock) {
5496 next->unlockForArbitration();
5497 }
5498
5499 if ((wasQuiet || nowQuiet)) {
5500 uint64_t regID = next->getRegistryEntryID();
5501 IOServiceTrace(
5502 ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
5503 (uintptr_t) regID,
5504 (uintptr_t) (regID >> 32),
5505 (uintptr_t) next,
5506 0);
5507
5508 if (wasQuiet) {
5509 next->__timeBusy = mach_absolute_time();
5510 } else {
5511 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
5512 next->__timeBusy = 0;
5513 }
5514
5515 MessageClientsContext context;
5516
5517 context.service = next;
5518 context.type = kIOMessageServiceBusyStateChange;
5519 context.argument = (void *) wasQuiet; /*nowBusy*/
5520 context.argSize = 0;
5521
5522 applyToInterestNotifiers( next, gIOBusyInterest,
5523 &messageClientsApplier, &context );
5524
5525 #if !NO_KEXTD
5526 if (nowQuiet && (next == gIOServiceRoot)) {
5527 if (gIOServiceHideIOMedia) {
5528 publishHiddenMedia(NULL);
5529 }
5530
5531 OSKext::considerUnloads();
5532 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
5533 }
5534 #endif
5535 }
5536
5537 delta = nowQuiet ? -1 : +1;
5538 if (next != this) {
5539 next->release();
5540 }
5541 next = nextProvider;
5542 nextProvider = NULL;
5543 } while ((wasQuiet || nowQuiet) && next);
5544 }
5545
5546 return result;
5547 }
5548
5549 void
adjustBusy(SInt32 delta)5550 IOService::adjustBusy( SInt32 delta )
5551 {
5552 lockForArbitration();
5553 _adjustBusy( delta );
5554 unlockForArbitration();
5555 }
5556
5557 uint64_t
getAccumulatedBusyTime(void)5558 IOService::getAccumulatedBusyTime( void )
5559 {
5560 uint64_t accumBusy = __accumBusy;
5561 uint64_t timeBusy = __timeBusy;
5562 uint64_t nano;
5563
5564 do{
5565 accumBusy = __accumBusy;
5566 timeBusy = __timeBusy;
5567 if (timeBusy) {
5568 accumBusy += mach_absolute_time() - timeBusy;
5569 }
5570 }while (timeBusy != __timeBusy);
5571
5572 absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
5573
5574 return nano;
5575 }
5576
5577 UInt32
getBusyState(void)5578 IOService::getBusyState( void )
5579 {
5580 return __state[1] & kIOServiceBusyStateMask;
5581 }
5582
5583 IOReturn
waitForState(UInt32 mask,UInt32 value,mach_timespec_t * timeout)5584 IOService::waitForState( UInt32 mask, UInt32 value,
5585 mach_timespec_t * timeout )
5586 {
5587 panic("waitForState");
5588 return kIOReturnUnsupported;
5589 }
5590
5591 IOReturn
waitForState(UInt32 mask,UInt32 value,uint64_t timeout)5592 IOService::waitForState( UInt32 mask, UInt32 value,
5593 uint64_t timeout )
5594 {
5595 bool wait;
5596 int waitResult = THREAD_AWAKENED;
5597 bool computeDeadline = true;
5598 AbsoluteTime abstime;
5599
5600 do {
5601 lockForArbitration();
5602 IOLockLock( gIOServiceBusyLock );
5603 wait = (value != (__state[1] & mask));
5604 if (wait) {
5605 __state[1] |= kIOServiceBusyWaiterState;
5606 unlockForArbitration();
5607 if (timeout != UINT64_MAX) {
5608 if (computeDeadline) {
5609 AbsoluteTime nsinterval;
5610 nanoseconds_to_absolutetime(timeout, &nsinterval );
5611 clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
5612 computeDeadline = false;
5613 }
5614 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
5615 } else {
5616 assert_wait((event_t)this, THREAD_UNINT );
5617 }
5618 } else {
5619 unlockForArbitration();
5620 }
5621 IOLockUnlock( gIOServiceBusyLock );
5622 if (wait) {
5623 waitResult = thread_block(THREAD_CONTINUE_NULL);
5624 }
5625 } while (wait && (waitResult != THREAD_TIMED_OUT));
5626
5627 if (waitResult == THREAD_TIMED_OUT) {
5628 return kIOReturnTimeout;
5629 } else {
5630 return kIOReturnSuccess;
5631 }
5632 }
5633
5634 IOReturn
waitQuietWithOptions(uint64_t timeout,IOOptionBits options)5635 IOService::waitQuietWithOptions( uint64_t timeout, IOOptionBits options )
5636 {
5637 IOReturn ret;
5638 uint32_t loops, timeoutExtensions;
5639 char * busyEntriesString = NULL;
5640 char * panicString = NULL;
5641 size_t busyEntriesStringLen;
5642 size_t panicStringLen;
5643 uint64_t time;
5644 uint64_t nano;
5645 bool pendingRequests;
5646 bool registryRootBusy;
5647 bool multipleEntries;
5648 bool dopanic = false;
5649
5650 enum { kIOServiceBusyTimeoutExtensionsMax = 8 };
5651 #if KASAN
5652 /*
5653 * On kasan kernels, everything takes longer, so double the number of
5654 * timeout extensions. This should help with issues like 41259215
5655 * where WindowServer was timing out waiting for kextd to get all the
5656 * kasan kexts loaded and started.
5657 *
5658 * On legacy/x86 systems give a bit more time since we may be
5659 * booting from a HDD.
5660 */
5661 enum { kTimeoutExtensions = 8 };
5662 #define WITH_IOWAITQUIET_EXTENSIONS 1
5663 #elif defined(__x86_64__)
5664 enum { kTimeoutExtensions = 4 };
5665 #define WITH_IOWAITQUIET_EXTENSIONS 1
5666 #elif defined(XNU_TARGET_OS_OSX)
5667 enum { kTimeoutExtensions = 1 };
5668 #define WITH_IOWAITQUIET_EXTENSIONS 1
5669 #else
5670 enum { kTimeoutExtensions = 1 };
5671 #define WITH_IOWAITQUIET_EXTENSIONS 0
5672 #endif
5673
5674 timeoutExtensions = kTimeoutExtensions;
5675 time = mach_absolute_time();
5676 pendingRequests = false;
5677 for (loops = 0; loops < timeoutExtensions; loops++) {
5678 ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
5679
5680 if (loops && (kIOReturnSuccess == ret)) {
5681 time = mach_absolute_time() - time;
5682 absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
5683 IOLog("busy extended ok[%d], (%llds, %llds)\n",
5684 loops, timeout / 1000000000ULL, nano / 1000000000ULL);
5685 break;
5686 } else if (kIOReturnTimeout != ret) {
5687 break;
5688 } else if (timeout < (41ull * NSEC_PER_SEC)) {
5689 break;
5690 }
5691
5692 {
5693 IORegistryIterator * iter;
5694 OSOrderedSet * set;
5695 OSOrderedSet * leaves;
5696 IOService * next;
5697 IOService * nextParent;
5698 char * s;
5699 size_t l;
5700 size_t busyEntriesStringRemaining;
5701
5702 busyEntriesStringLen = 256;
5703 panicStringLen = 256;
5704 if (!busyEntriesString) {
5705 busyEntriesString = IONewZeroData(char, busyEntriesStringLen);
5706 assert(busyEntriesString != NULL);
5707 }
5708 if (!panicString) {
5709 panicString = IONewZeroData(char, panicStringLen);
5710 assert(panicString != NULL);
5711 }
5712
5713 set = NULL;
5714 pendingRequests = OSKext::pendingIOKitDaemonRequests();
5715 iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
5716 leaves = OSOrderedSet::withCapacity(4);
5717 if (iter) {
5718 set = iter->iterateAll();
5719 }
5720 if (leaves && set) {
5721 busyEntriesString[0] = panicString[0] = 0;
5722 set->setObject(this);
5723 while ((next = (IOService *) set->getLastObject())) {
5724 if (next->getBusyState()) {
5725 if (kIOServiceModuleStallState & next->__state[1]) {
5726 pendingRequests = true;
5727 }
5728 #if defined(XNU_TARGET_OS_OSX)
5729 OSObject * prop;
5730 if ((prop = next->copyProperty(kIOServiceBusyTimeoutExtensionsKey, gIOServicePlane))) {
5731 OSNumber * num;
5732 uint32_t value;
5733 if ((num = OSDynamicCast(OSNumber, prop))) {
5734 value = num->unsigned32BitValue();
5735 if (value
5736 && (value <= kIOServiceBusyTimeoutExtensionsMax)
5737 && (value > timeoutExtensions)) {
5738 timeoutExtensions = value;
5739 }
5740 }
5741 OSSafeReleaseNULL(prop);
5742 }
5743 #endif /* defined(XNU_TARGET_OS_OSX) */
5744 leaves->setObject(next);
5745 nextParent = next;
5746 while ((nextParent = nextParent->getProvider())) {
5747 set->removeObject(nextParent);
5748 leaves->removeObject(nextParent);
5749 }
5750 }
5751 set->removeObject(next);
5752 }
5753 registryRootBusy = leaves->getCount() == 1 && leaves->getObject(0) == getServiceRoot();
5754 multipleEntries = leaves->getCount() > 1;
5755 s = busyEntriesString;
5756 busyEntriesStringRemaining = busyEntriesStringLen;
5757
5758 if (registryRootBusy) {
5759 snprintf(s, busyEntriesStringRemaining, "registry root held busy, " kIOKitDaemonName " %s checked in", OSKext::iokitDaemonActive() ? "has" : "has not");
5760 } else {
5761 while ((next = (IOService *) leaves->getLastObject())) {
5762 l = snprintf(s, busyEntriesStringRemaining, "%s'%s' (%x,%x)", ((s == busyEntriesString) ? "" : ", "), next->getName(), (uint32_t)next->__state[0], (uint32_t)next->__state[1]);
5763 if (l >= busyEntriesStringRemaining) {
5764 break;
5765 }
5766 s += l;
5767 busyEntriesStringRemaining -= l;
5768 leaves->removeObject(next);
5769 }
5770 }
5771 }
5772 OSSafeReleaseNULL(leaves);
5773 OSSafeReleaseNULL(set);
5774 OSSafeReleaseNULL(iter);
5775 }
5776
5777 dopanic = (kIOWaitQuietPanics & gIOKitDebug) && (options & kIOWaitQuietPanicOnFailure) && !gIOKitWillTerminate;
5778 #if WITH_IOWAITQUIET_EXTENSIONS
5779 dopanic = (dopanic && (loops >= (timeoutExtensions - 1)));
5780 #endif
5781 assert(panicString != NULL);
5782 if (multipleEntries) {
5783 snprintf(panicString, panicStringLen,
5784 "%s[%d], (%llds): multiple entries holding the registry busy, IOKit termination queue depth %u: %s",
5785 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5786 loops, timeout / 1000000000ULL,
5787 (uint32_t)gIOTerminateWork,
5788 busyEntriesString ? busyEntriesString : "");
5789 } else {
5790 snprintf(panicString, panicStringLen,
5791 "%s[%d], (%llds): %s",
5792 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5793 loops, timeout / 1000000000ULL,
5794 busyEntriesString ? busyEntriesString : "");
5795 }
5796
5797 IOLog("%s\n", panicString);
5798 if (dopanic) {
5799 panic("%s", panicString);
5800 } else if (!loops) {
5801 getPMRootDomain()->startSpinDump(1);
5802 }
5803 }
5804
5805 if (busyEntriesString) {
5806 IODeleteData(busyEntriesString, char, busyEntriesStringLen);
5807 }
5808 if (panicString) {
5809 IODeleteData(panicString, char, panicStringLen);
5810 }
5811
5812 return ret;
5813 }
5814
5815 IOReturn
waitQuiet(uint64_t timeout)5816 IOService::waitQuiet( uint64_t timeout )
5817 {
5818 return waitQuietWithOptions(timeout);
5819 }
5820
5821 IOReturn
waitQuiet(mach_timespec_t * timeout)5822 IOService::waitQuiet( mach_timespec_t * timeout )
5823 {
5824 uint64_t timeoutNS;
5825
5826 if (timeout) {
5827 timeoutNS = timeout->tv_sec;
5828 timeoutNS *= kSecondScale;
5829 timeoutNS += timeout->tv_nsec;
5830 } else {
5831 timeoutNS = UINT64_MAX;
5832 }
5833
5834 return waitQuiet(timeoutNS);
5835 }
5836
5837 bool
serializeProperties(OSSerialize * s) const5838 IOService::serializeProperties( OSSerialize * s ) const
5839 {
5840 #if 0
5841 ((IOService *)this)->setProperty(((IOService *)this)->__state,
5842 sizeof(__state), "__state");
5843 #endif
5844 return super::serializeProperties(s);
5845 }
5846
5847 void
resetRematchProperties()5848 IOService::resetRematchProperties()
5849 {
5850 removeProperty(gIORematchCountKey);
5851 removeProperty(gIORematchPersonalityKey);
5852 }
5853
5854
5855 void
main(void * arg,wait_result_t result)5856 _IOConfigThread::main(void * arg, wait_result_t result)
5857 {
5858 _IOConfigThread * self = (_IOConfigThread *) arg;
5859 _IOServiceJob * job;
5860 IOService * nub;
5861 bool alive = true;
5862 kern_return_t kr;
5863 thread_precedence_policy_data_t precedence = { -1 };
5864
5865 kr = thread_policy_set(current_thread(),
5866 THREAD_PRECEDENCE_POLICY,
5867 (thread_policy_t) &precedence,
5868 THREAD_PRECEDENCE_POLICY_COUNT);
5869 if (KERN_SUCCESS != kr) {
5870 IOLog("thread_policy_set(%d)\n", kr);
5871 }
5872
5873 do {
5874 // randomDelay();
5875
5876 semaphore_wait( gJobsSemaphore );
5877
5878 IOTakeLock( gJobsLock );
5879 job = (_IOServiceJob *) gJobs->getFirstObject();
5880 job->retain();
5881 gJobs->removeObject(job);
5882 if (job) {
5883 gOutstandingJobs--;
5884 // gNumConfigThreads--; // we're out of service
5885 gNumWaitingThreads--; // we're out of service
5886 }
5887 IOUnlock( gJobsLock );
5888
5889 if (job) {
5890 nub = job->nub;
5891
5892 if (gIOKitDebug & kIOLogConfig) {
5893 LOG("config(%p): starting on %s, %d\n",
5894 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5895 }
5896
5897 switch (job->type) {
5898 case kMatchNubJob:
5899 nub->doServiceMatch( job->options );
5900 break;
5901
5902 default:
5903 LOG("config(%p): strange type (%d)\n",
5904 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5905 break;
5906 }
5907
5908 if (job->options & kIOServiceDextRequirePowerForMatching) {
5909 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
5910 }
5911
5912 OSSafeReleaseNULL(nub);
5913 OSSafeReleaseNULL(job);
5914
5915 IOTakeLock( gJobsLock );
5916 alive = (gOutstandingJobs > gNumWaitingThreads);
5917 if (alive) {
5918 gNumWaitingThreads++; // back in service
5919 }
5920 // gNumConfigThreads++;
5921 else {
5922 if (0 == --gNumConfigThreads) {
5923 // IOLog("MATCH IDLE\n");
5924 IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
5925 }
5926 }
5927 IOUnlock( gJobsLock );
5928 }
5929 } while (alive);
5930
5931 if (gIOKitDebug & kIOLogConfig) {
5932 LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5933 }
5934
5935 self->release();
5936 }
5937
5938 IOReturn
waitMatchIdle(UInt32 msToWait)5939 IOService::waitMatchIdle( UInt32 msToWait )
5940 {
5941 bool wait;
5942 int waitResult = THREAD_AWAKENED;
5943 bool computeDeadline = true;
5944 AbsoluteTime deadline;
5945
5946 IOLockLock( gJobsLock );
5947 do {
5948 wait = (0 != gNumConfigThreads);
5949 if (wait) {
5950 if (msToWait) {
5951 if (computeDeadline) {
5952 clock_interval_to_deadline(
5953 msToWait, kMillisecondScale, &deadline );
5954 computeDeadline = false;
5955 }
5956 waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
5957 deadline, THREAD_UNINT );
5958 } else {
5959 waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
5960 THREAD_UNINT );
5961 }
5962 }
5963 } while (wait && (waitResult != THREAD_TIMED_OUT));
5964 IOLockUnlock( gJobsLock );
5965
5966 if (waitResult == THREAD_TIMED_OUT) {
5967 return kIOReturnTimeout;
5968 } else {
5969 return kIOReturnSuccess;
5970 }
5971 }
5972
5973 void
cpusRunning(void)5974 IOService::cpusRunning(void)
5975 {
5976 gCPUsRunning = true;
5977 }
5978
5979 void
pingConfig(_IOServiceJob * job)5980 _IOServiceJob::pingConfig( _IOServiceJob * job )
5981 {
5982 int count;
5983 bool create;
5984 IOService * nub;
5985
5986 assert( job );
5987 nub = job->nub;
5988
5989 IOTakeLock( gJobsLock );
5990
5991 gOutstandingJobs++;
5992 if (nub == gIOResources) {
5993 gJobs->setFirstObject( job );
5994 } else {
5995 gJobs->setLastObject( job );
5996 }
5997
5998 count = gNumWaitingThreads;
5999 // if( gNumConfigThreads) count++;// assume we're called from a config thread
6000
6001 create = ((gOutstandingJobs > count)
6002 && ((gNumConfigThreads < gMaxConfigThreads)
6003 || (nub == gIOResources)
6004 || !gCPUsRunning));
6005 if (create) {
6006 gNumConfigThreads++;
6007 gNumWaitingThreads++;
6008 if (gNumConfigThreads > gHighNumConfigThreads) {
6009 gHighNumConfigThreads = gNumConfigThreads;
6010 }
6011 }
6012
6013 IOUnlock( gJobsLock );
6014
6015 job->release();
6016
6017 if (create) {
6018 if (gIOKitDebug & kIOLogConfig) {
6019 LOG("config(%d): creating\n", gNumConfigThreads - 1);
6020 }
6021 _IOConfigThread::configThread(nub->getName());
6022 }
6023
6024 semaphore_signal( gJobsSemaphore );
6025 }
6026
6027 struct IOServiceMatchContext {
6028 OSDictionary * table;
6029 OSObject * result;
6030 uint32_t options;
6031 uint32_t state;
6032 uint32_t count;
6033 uint32_t done;
6034 };
6035
6036 bool
instanceMatch(const OSObject * entry,void * context)6037 IOService::instanceMatch(const OSObject * entry, void * context)
6038 {
6039 IOServiceMatchContext * ctx = (typeof(ctx))context;
6040 IOService * service = (typeof(service))entry;
6041 OSDictionary * table = ctx->table;
6042 uint32_t options = ctx->options;
6043 uint32_t state = ctx->state;
6044 uint32_t done;
6045 bool match;
6046
6047 done = 0;
6048 do{
6049 match = ((state == (state & service->__state[0]))
6050 && (0 == (service->__state[0] & kIOServiceInactiveState)));
6051 if (!match) {
6052 break;
6053 }
6054
6055 match = service->matchInternal(table, options, &done);
6056 if (match) {
6057 ctx->count += table->getCount();
6058 ctx->done += done;
6059 }
6060 }while (false);
6061 if (!match) {
6062 return false;
6063 }
6064
6065 if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
6066 service->retain();
6067 ctx->result = service;
6068 return true;
6069 } else if (!ctx->result) {
6070 ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
6071 } else {
6072 ((OSSet *)ctx->result)->setObject(service);
6073 }
6074 return false;
6075 }
6076
6077 // internal - call with gNotificationLock
6078 OSObject *
copyExistingServices(OSDictionary * matching,IOOptionBits inState,IOOptionBits options)6079 IOService::copyExistingServices( OSDictionary * matching,
6080 IOOptionBits inState, IOOptionBits options )
6081 {
6082 OSObject * current = NULL;
6083 OSIterator * iter;
6084 IOService * service;
6085 OSObject * obj;
6086 OSString * str;
6087
6088 if (!matching) {
6089 return NULL;
6090 }
6091
6092 #if MATCH_DEBUG
6093 OSSerialize * s = OSSerialize::withCapacity(128);
6094 matching->serialize(s);
6095 #endif
6096
6097 if ((obj = matching->getObject(gIOProviderClassKey))
6098 && gIOResourcesKey
6099 && gIOResourcesKey->isEqualTo(obj)
6100 && (service = gIOResources)) {
6101 if ((inState == (service->__state[0] & inState))
6102 && (0 == (service->__state[0] & kIOServiceInactiveState))
6103 && service->matchPassive(matching, options)) {
6104 if (options & kIONotifyOnce) {
6105 service->retain();
6106 current = service;
6107 } else {
6108 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
6109 }
6110 }
6111 } else {
6112 IOServiceMatchContext ctx;
6113
6114 options |= kIOServiceClassDone;
6115 ctx.table = matching;
6116 ctx.state = inState;
6117 ctx.count = 0;
6118 ctx.done = 0;
6119 ctx.options = options;
6120 ctx.result = NULL;
6121
6122 if ((str = OSDynamicCast(OSString, obj))) {
6123 const OSSymbol * sym = OSSymbol::withString(str);
6124 OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
6125 sym->release();
6126 } else {
6127 IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
6128 }
6129
6130 if (((!(options & kIONotifyOnce) || !ctx.result))
6131 && matching->getObject(gIOCompatibilityMatchKey)) {
6132 IOServiceCompatibility::gMetaClass.applyToInstances(instanceMatch, &ctx);
6133 }
6134
6135 current = ctx.result;
6136 options |= kIOServiceInternalDone;
6137 if (current && (ctx.done != ctx.count)) {
6138 OSSet * source = OSDynamicCast(OSSet, current);
6139 current = NULL;
6140 while ((service = (IOService *) source->getAnyObject())) {
6141 if (service->matchPassive(matching, options)) {
6142 if (options & kIONotifyOnce) {
6143 service->retain();
6144 current = service;
6145 break;
6146 }
6147 if (current) {
6148 ((OSSet *)current)->setObject( service );
6149 } else {
6150 current = OSSet::withObjects(
6151 (const OSObject **) &service, 1, 1 );
6152 }
6153 }
6154 source->removeObject(service);
6155 }
6156 source->release();
6157 }
6158 }
6159
6160 #if MATCH_DEBUG
6161 {
6162 OSObject * _current = 0;
6163
6164 iter = IORegistryIterator::iterateOver( gIOServicePlane,
6165 kIORegistryIterateRecursively );
6166 if (iter) {
6167 do {
6168 iter->reset();
6169 while ((service = (IOService *) iter->getNextObject())) {
6170 if ((inState == (service->__state[0] & inState))
6171 && (0 == (service->__state[0] & kIOServiceInactiveState))
6172 && service->matchPassive(matching, 0)) {
6173 if (options & kIONotifyOnce) {
6174 service->retain();
6175 _current = service;
6176 break;
6177 }
6178 if (_current) {
6179 ((OSSet *)_current)->setObject( service );
6180 } else {
6181 _current = OSSet::withObjects(
6182 (const OSObject **) &service, 1, 1 );
6183 }
6184 }
6185 }
6186 } while (!service && !iter->isValid());
6187 iter->release();
6188 }
6189
6190 if (((current != 0) != (_current != 0))
6191 || (current && _current && !current->isEqualTo(_current))) {
6192 OSSerialize * s1 = OSSerialize::withCapacity(128);
6193 OSSerialize * s2 = OSSerialize::withCapacity(128);
6194 current->serialize(s1);
6195 _current->serialize(s2);
6196 kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
6197 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
6198 s1->release();
6199 s2->release();
6200 }
6201
6202 if (_current) {
6203 _current->release();
6204 }
6205 }
6206
6207 s->release();
6208 #endif
6209
6210 if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
6211 iter = OSCollectionIterator::withCollection((OSSet *)current );
6212 current->release();
6213 current = iter;
6214 }
6215
6216 return current;
6217 }
6218
6219 // public version
6220 OSIterator *
getMatchingServices(OSDictionary * matching)6221 IOService::getMatchingServices( OSDictionary * matching )
6222 {
6223 OSIterator * iter;
6224
6225 // is a lock even needed?
6226 LOCKWRITENOTIFY();
6227
6228 iter = (OSIterator *) copyExistingServices( matching,
6229 kIOServiceMatchedState );
6230
6231 UNLOCKNOTIFY();
6232
6233 return iter;
6234 }
6235
6236 IOService *
copyMatchingService(OSDictionary * matching)6237 IOService::copyMatchingService( OSDictionary * matching )
6238 {
6239 IOService * service;
6240
6241 // is a lock even needed?
6242 LOCKWRITENOTIFY();
6243
6244 service = (IOService *) copyExistingServices( matching,
6245 kIOServiceMatchedState, kIONotifyOnce );
6246
6247 UNLOCKNOTIFY();
6248
6249 return service;
6250 }
6251
6252 struct _IOServiceMatchingNotificationHandlerRef {
6253 IOServiceNotificationHandler handler;
6254 void * ref;
6255 };
6256
6257 static bool
_IOServiceMatchingNotificationHandler(void * target,void * refCon,IOService * newService,IONotifier * notifier)6258 _IOServiceMatchingNotificationHandler( void * target, void * refCon,
6259 IOService * newService,
6260 IONotifier * notifier )
6261 {
6262 return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
6263 }
6264
6265 // internal - call with gNotificationLock
6266 IONotifier *
setNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6267 IOService::setNotification(
6268 const OSSymbol * type, OSDictionary * matching,
6269 IOServiceMatchingNotificationHandler handler, void * target, void * ref,
6270 SInt32 priority )
6271 {
6272 _IOServiceNotifier * notify = NULL;
6273 OSOrderedSet * set;
6274
6275 if (!matching) {
6276 return NULL;
6277 }
6278
6279 notify = new _IOServiceNotifier;
6280 if (notify && !notify->init()) {
6281 notify->release();
6282 notify = NULL;
6283 }
6284
6285 if (notify) {
6286 notify->handler = handler;
6287 notify->target = target;
6288 notify->type = type;
6289 notify->matching = matching;
6290 matching->retain();
6291 if (handler == &_IOServiceMatchingNotificationHandler) {
6292 notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
6293 notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
6294 } else {
6295 notify->ref = ref;
6296 }
6297 notify->priority = priority;
6298 notify->state = kIOServiceNotifyEnable;
6299 queue_init( ¬ify->handlerInvocations );
6300
6301 ////// queue
6302
6303 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
6304 set = OSOrderedSet::withCapacity( 1,
6305 IONotifyOrdering, NULL );
6306 if (set) {
6307 gNotifications->setObject( type, set );
6308 set->release();
6309 }
6310 }
6311 notify->whence = set;
6312 if (set) {
6313 set->setObject( notify );
6314 }
6315 }
6316
6317 return notify;
6318 }
6319
6320 // internal - call with gNotificationLock
6321 IONotifier *
doInstallNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6322 IOService::doInstallNotification(
6323 const OSSymbol * type, OSDictionary * matching,
6324 IOServiceMatchingNotificationHandler handler,
6325 void * target, void * ref,
6326 SInt32 priority, OSIterator ** existing )
6327 {
6328 OSIterator * exist;
6329 IONotifier * notify;
6330 IOOptionBits inState;
6331
6332 if (!matching) {
6333 return NULL;
6334 }
6335
6336 if (type == gIOPublishNotification) {
6337 inState = kIOServiceRegisteredState;
6338 } else if (type == gIOFirstPublishNotification) {
6339 inState = kIOServiceFirstPublishState;
6340 } else if (type == gIOMatchedNotification) {
6341 inState = kIOServiceMatchedState;
6342 } else if (type == gIOFirstMatchNotification) {
6343 inState = kIOServiceFirstMatchState;
6344 } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
6345 inState = 0;
6346 } else {
6347 return NULL;
6348 }
6349
6350 notify = setNotification( type, matching, handler, target, ref, priority );
6351
6352 if (inState) {
6353 // get the current set
6354 exist = (OSIterator *) copyExistingServices( matching, inState );
6355 } else {
6356 exist = NULL;
6357 }
6358
6359 *existing = exist;
6360
6361 return notify;
6362 }
6363
6364 #if !defined(__LP64__)
6365 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority,OSIterator ** existing)6366 IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
6367 IOServiceNotificationHandler handler,
6368 void * target, void * refCon,
6369 SInt32 priority, OSIterator ** existing )
6370 {
6371 IONotifier * result;
6372 _IOServiceMatchingNotificationHandlerRef ref;
6373 ref.handler = handler;
6374 ref.ref = refCon;
6375
6376 result = (_IOServiceNotifier *) installNotification( type, matching,
6377 &_IOServiceMatchingNotificationHandler,
6378 target, &ref, priority, existing );
6379 if (result) {
6380 matching->release();
6381 }
6382
6383 return result;
6384 }
6385
6386 #endif /* !defined(__LP64__) */
6387
6388
6389 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6390 IOService::installNotification(
6391 const OSSymbol * type, OSDictionary * matching,
6392 IOServiceMatchingNotificationHandler handler,
6393 void * target, void * ref,
6394 SInt32 priority, OSIterator ** existing )
6395 {
6396 IONotifier * notify;
6397
6398 LOCKWRITENOTIFY();
6399
6400 notify = doInstallNotification( type, matching, handler, target, ref,
6401 priority, existing );
6402
6403 // in case handler remove()s
6404 if (notify) {
6405 notify->retain();
6406 }
6407
6408 UNLOCKNOTIFY();
6409
6410 return notify;
6411 }
6412
6413 IONotifier *
addNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority)6414 IOService::addNotification(
6415 const OSSymbol * type, OSDictionary * matching,
6416 IOServiceNotificationHandler handler,
6417 void * target, void * refCon,
6418 SInt32 priority )
6419 {
6420 IONotifier * result;
6421 _IOServiceMatchingNotificationHandlerRef ref;
6422
6423 ref.handler = handler;
6424 ref.ref = refCon;
6425
6426 result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
6427 target, &ref, priority);
6428
6429 if (result) {
6430 matching->release();
6431 }
6432
6433 return result;
6434 }
6435
6436 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6437 IOService::addMatchingNotification(
6438 const OSSymbol * type, OSDictionary * matching,
6439 IOServiceMatchingNotificationHandler handler,
6440 void * target, void * ref,
6441 SInt32 priority )
6442 {
6443 OSIterator * existing = NULL;
6444 IONotifier * ret;
6445 _IOServiceNotifier * notify;
6446 IOService * next;
6447
6448 ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
6449 handler, target, ref, priority, &existing );
6450 if (!ret) {
6451 OSSafeReleaseNULL(existing);
6452 return NULL;
6453 }
6454
6455 // send notifications for existing set
6456 if (existing) {
6457 while ((next = (IOService *) existing->getNextObject())) {
6458 if (0 == (next->__state[0] & kIOServiceInactiveState)) {
6459 next->invokeNotifier( notify );
6460 }
6461 }
6462 existing->release();
6463 }
6464
6465 LOCKWRITENOTIFY();
6466 bool removed = (NULL == notify->whence);
6467 notify->release();
6468 if (removed) {
6469 ret = gIOServiceNullNotifier;
6470 }
6471 UNLOCKNOTIFY();
6472
6473 return ret;
6474 }
6475
6476 static bool
IOServiceMatchingNotificationHandlerToBlock(void * target __unused,void * refCon,IOService * newService,IONotifier * notifier)6477 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
6478 IOService * newService,
6479 IONotifier * notifier )
6480 {
6481 return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
6482 }
6483
6484 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,SInt32 priority,IOServiceMatchingNotificationHandlerBlock handler)6485 IOService::addMatchingNotification(
6486 const OSSymbol * type, OSDictionary * matching,
6487 SInt32 priority,
6488 IOServiceMatchingNotificationHandlerBlock handler)
6489 {
6490 IONotifier * notify;
6491 void * block;
6492
6493 block = Block_copy(handler);
6494 if (!block) {
6495 return NULL;
6496 }
6497
6498 notify = addMatchingNotification(type, matching,
6499 &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
6500
6501 if (!notify) {
6502 Block_release(block);
6503 }
6504
6505 return notify;
6506 }
6507
6508 struct IOUserServerCancellationHandlerArgs {
6509 IOService ** ref;
6510 bool canceled;
6511 };
6512
6513 void
userServerCheckInTokenCancellationHandler(__unused IOUserServerCheckInToken * token,void * ref)6514 IOService::userServerCheckInTokenCancellationHandler(
6515 __unused IOUserServerCheckInToken *token,
6516 void *ref)
6517 {
6518 IOUserServerCancellationHandlerArgs * args = (typeof(args))ref;
6519 LOCKWRITENOTIFY();
6520 WAKEUPNOTIFY(args->ref);
6521 args->canceled = true;
6522 UNLOCKNOTIFY();
6523 }
6524
6525 bool
syncNotificationHandler(void *,void * ref,IOService * newService,IONotifier * notifier)6526 IOService::syncNotificationHandler(
6527 void * /* target */, void * ref,
6528 IOService * newService,
6529 IONotifier * notifier )
6530 {
6531 LOCKWRITENOTIFY();
6532 if (!*((IOService **) ref)) {
6533 newService->retain();
6534 (*(IOService **) ref) = newService;
6535 WAKEUPNOTIFY(ref);
6536 }
6537 UNLOCKNOTIFY();
6538
6539 return false;
6540 }
6541
6542 IOService *
waitForMatchingServiceWithToken(OSDictionary * matching,uint64_t timeout,IOUserServerCheckInToken * checkInToken)6543 IOService::waitForMatchingServiceWithToken( OSDictionary * matching,
6544 uint64_t timeout,
6545 IOUserServerCheckInToken * checkInToken)
6546 {
6547 IONotifier * notify = NULL;
6548 // priority doesn't help us much since we need a thread wakeup
6549 SInt32 priority = 0;
6550 IOService * result;
6551 IOUserServerCancellationHandlerArgs cancelArgs;
6552 _IOUserServerCheckInCancellationHandler * cancellationHandler = NULL;
6553
6554 if (!matching) {
6555 return NULL;
6556 }
6557
6558 result = NULL;
6559 cancelArgs.ref = &result;
6560 cancelArgs.canceled = false;
6561
6562 #if DEBUG || DEVELOPMENT
6563 char currentName[MAXTHREADNAMESIZE];
6564 char newName[MAXTHREADNAMESIZE];
6565 OSObject * obj;
6566 OSString * str;
6567 OSDictionary * dict;
6568
6569 currentName[0] = '\0';
6570 if (thread_has_thread_name(current_thread())) {
6571 dict = matching;
6572 obj = matching->getObject(gIOPropertyMatchKey);
6573 if ((dict = OSDynamicCast(OSDictionary, obj))) {
6574 OSObject * result __block = NULL;
6575 dict->iterateObjects(^bool (const OSSymbol * sym, OSObject * value) {
6576 result = __DECONST(OSObject *, sym);
6577 return true;
6578 });
6579 obj = result;
6580 }
6581 if (!obj) {
6582 obj = matching->getObject(gIOResourceMatchKey);
6583 }
6584 if (!obj) {
6585 obj = matching->getObject(gIONameMatchKey);
6586 }
6587 if (!obj) {
6588 obj = matching->getObject(gIOProviderClassKey);
6589 }
6590 if ((str = OSDynamicCast(OSString, obj))) {
6591 thread_get_thread_name(current_thread(), currentName);
6592 snprintf(newName, sizeof(newName), "Waiting_'%s'", str->getCStringNoCopy());
6593 thread_set_thread_name(current_thread(), newName);
6594 }
6595 }
6596 #endif /* DEBUG || DEVELOPMENT */
6597
6598 if (checkInToken) {
6599 cancellationHandler = checkInToken->setCancellationHandler(&IOService::userServerCheckInTokenCancellationHandler,
6600 &cancelArgs);
6601 }
6602
6603 LOCKWRITENOTIFY();
6604 do{
6605 if (cancelArgs.canceled) {
6606 // token was already canceled, no need to wait or find services
6607 break;
6608 }
6609 result = (IOService *) copyExistingServices( matching,
6610 kIOServiceMatchedState, kIONotifyOnce );
6611 if (result) {
6612 break;
6613 }
6614 notify = IOService::setNotification( gIOMatchedNotification, matching,
6615 &IOService::syncNotificationHandler, (void *) NULL,
6616 &result, priority );
6617 if (!notify) {
6618 break;
6619 }
6620 if (UINT64_MAX != timeout) {
6621 AbsoluteTime deadline;
6622 nanoseconds_to_absolutetime(timeout, &deadline);
6623 clock_absolutetime_interval_to_deadline(deadline, &deadline);
6624 SLEEPNOTIFYTO(&result, deadline);
6625 } else {
6626 SLEEPNOTIFY(&result);
6627 }
6628 }while (false);
6629
6630 UNLOCKNOTIFY();
6631
6632 if (checkInToken && cancellationHandler) {
6633 checkInToken->removeCancellationHandler(cancellationHandler);
6634 }
6635
6636 #if DEBUG || DEVELOPMENT
6637 if (currentName[0]) {
6638 thread_set_thread_name(current_thread(), currentName);
6639 }
6640 #endif /* DEBUG || DEVELOPMENT */
6641
6642 if (notify) {
6643 notify->remove(); // dequeues
6644 }
6645
6646 OSSafeReleaseNULL(cancellationHandler);
6647
6648 return result;
6649 }
6650
6651 IOService *
waitForMatchingService(OSDictionary * matching,uint64_t timeout)6652 IOService::waitForMatchingService( OSDictionary * matching,
6653 uint64_t timeout)
6654 {
6655 return IOService::waitForMatchingServiceWithToken(matching, timeout, NULL);
6656 }
6657
6658 IOService *
waitForService(OSDictionary * matching,mach_timespec_t * timeout)6659 IOService::waitForService( OSDictionary * matching,
6660 mach_timespec_t * timeout )
6661 {
6662 IOService * result;
6663 uint64_t timeoutNS;
6664
6665 if (timeout) {
6666 timeoutNS = timeout->tv_sec;
6667 timeoutNS *= kSecondScale;
6668 timeoutNS += timeout->tv_nsec;
6669 } else {
6670 timeoutNS = UINT64_MAX;
6671 }
6672
6673 result = waitForMatchingService(matching, timeoutNS);
6674
6675 matching->release();
6676 if (result) {
6677 result->release();
6678 }
6679
6680 return result;
6681 }
6682
6683 __dead2
6684 void
deliverNotification(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6685 IOService::deliverNotification( const OSSymbol * type,
6686 IOOptionBits orNewState, IOOptionBits andNewState )
6687 {
6688 panic("deliverNotification");
6689 }
6690
6691 OSArray *
copyNotifiers(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6692 IOService::copyNotifiers(const OSSymbol * type,
6693 IOOptionBits orNewState, IOOptionBits andNewState )
6694 {
6695 _IOServiceNotifier * notify;
6696 OSIterator * iter;
6697 OSArray * willSend = NULL;
6698
6699 lockForArbitration();
6700
6701 if ((0 == (__state[0] & kIOServiceInactiveState))
6702 || (type == gIOTerminatedNotification)
6703 || (type == gIOWillTerminateNotification)) {
6704 LOCKREADNOTIFY();
6705
6706 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
6707 gNotifications->getObject( type ));
6708
6709 if (iter) {
6710 while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
6711 if (matchPassive(notify->matching, 0)
6712 && (kIOServiceNotifyEnable & notify->state)) {
6713 if (NULL == willSend) {
6714 willSend = OSArray::withCapacity(8);
6715 }
6716 if (willSend) {
6717 willSend->setObject( notify );
6718 }
6719 }
6720 }
6721 iter->release();
6722 }
6723 __state[0] = (__state[0] | orNewState) & andNewState;
6724 UNLOCKNOTIFY();
6725 }
6726
6727 unlockForArbitration();
6728
6729 return willSend;
6730 }
6731
6732 IOOptionBits
getState(void) const6733 IOService::getState( void ) const
6734 {
6735 return __state[0];
6736 }
6737
6738 /*
6739 * Helpers to make matching objects for simple cases
6740 */
6741
6742 OSDictionary *
serviceMatching(const OSString * name,OSDictionary * table)6743 IOService::serviceMatching( const OSString * name,
6744 OSDictionary * table )
6745 {
6746 const OSString * str;
6747
6748 str = OSSymbol::withString(name);
6749 if (!str) {
6750 return NULL;
6751 }
6752
6753 if (!table) {
6754 table = OSDictionary::withCapacity( 2 );
6755 }
6756 if (table) {
6757 table->setObject(gIOProviderClassKey, (OSObject *)str );
6758 }
6759 str->release();
6760
6761 return table;
6762 }
6763
6764
6765 OSSharedPtr<OSDictionary>
serviceMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6766 IOService::serviceMatching( const OSString * name,
6767 OSSharedPtr<OSDictionary> table)
6768 {
6769 OSDictionary * result = serviceMatching(name, table.get());
6770 if (table) {
6771 return OSSharedPtr<OSDictionary>(result, OSRetain);
6772 } else {
6773 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6774 }
6775 }
6776
6777
6778 OSDictionary *
serviceMatching(const char * name,OSDictionary * table)6779 IOService::serviceMatching( const char * name,
6780 OSDictionary * table )
6781 {
6782 const OSString * str;
6783
6784 str = OSSymbol::withCString( name );
6785 if (!str) {
6786 return NULL;
6787 }
6788
6789 table = serviceMatching( str, table );
6790 str->release();
6791 return table;
6792 }
6793
6794
6795 OSSharedPtr<OSDictionary>
serviceMatching(const char * className,OSSharedPtr<OSDictionary> table)6796 IOService::serviceMatching( const char * className,
6797 OSSharedPtr<OSDictionary> table)
6798 {
6799 OSDictionary * result = serviceMatching(className, table.get());
6800 if (table) {
6801 return OSSharedPtr<OSDictionary>(result, OSRetain);
6802 } else {
6803 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6804 }
6805 }
6806
6807
6808 OSDictionary *
nameMatching(const OSString * name,OSDictionary * table)6809 IOService::nameMatching( const OSString * name,
6810 OSDictionary * table )
6811 {
6812 if (!table) {
6813 table = OSDictionary::withCapacity( 2 );
6814 }
6815 if (table) {
6816 table->setObject( gIONameMatchKey, (OSObject *)name );
6817 }
6818
6819 return table;
6820 }
6821
6822
6823 OSSharedPtr<OSDictionary>
nameMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6824 IOService::nameMatching( const OSString * name,
6825 OSSharedPtr<OSDictionary> table)
6826 {
6827 OSDictionary * result = nameMatching(name, table.get());
6828 if (table) {
6829 return OSSharedPtr<OSDictionary>(result, OSRetain);
6830 } else {
6831 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6832 }
6833 }
6834
6835
6836 OSDictionary *
nameMatching(const char * name,OSDictionary * table)6837 IOService::nameMatching( const char * name,
6838 OSDictionary * table )
6839 {
6840 const OSString * str;
6841
6842 str = OSSymbol::withCString( name );
6843 if (!str) {
6844 return NULL;
6845 }
6846
6847 table = nameMatching( str, table );
6848 str->release();
6849 return table;
6850 }
6851
6852
6853 OSSharedPtr<OSDictionary>
nameMatching(const char * name,OSSharedPtr<OSDictionary> table)6854 IOService::nameMatching( const char * name,
6855 OSSharedPtr<OSDictionary> table)
6856 {
6857 OSDictionary * result = nameMatching(name, table.get());
6858 if (table) {
6859 return OSSharedPtr<OSDictionary>(result, OSRetain);
6860 } else {
6861 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6862 }
6863 }
6864
6865
6866 OSDictionary *
resourceMatching(const OSString * str,OSDictionary * table)6867 IOService::resourceMatching( const OSString * str,
6868 OSDictionary * table )
6869 {
6870 table = serviceMatching( gIOResourcesKey, table );
6871 if (table) {
6872 table->setObject( gIOResourceMatchKey, (OSObject *) str );
6873 }
6874
6875 return table;
6876 }
6877
6878
6879 OSSharedPtr<OSDictionary>
resourceMatching(const OSString * str,OSSharedPtr<OSDictionary> table)6880 IOService::resourceMatching( const OSString * str,
6881 OSSharedPtr<OSDictionary> table)
6882 {
6883 OSDictionary * result = resourceMatching(str, table.get());
6884 if (table) {
6885 return OSSharedPtr<OSDictionary>(result, OSRetain);
6886 } else {
6887 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6888 }
6889 }
6890
6891
6892 OSDictionary *
resourceMatching(const char * name,OSDictionary * table)6893 IOService::resourceMatching( const char * name,
6894 OSDictionary * table )
6895 {
6896 const OSSymbol * str;
6897
6898 str = OSSymbol::withCString( name );
6899 if (!str) {
6900 return NULL;
6901 }
6902
6903 table = resourceMatching( str, table );
6904 str->release();
6905
6906 return table;
6907 }
6908
6909
6910 OSSharedPtr<OSDictionary>
resourceMatching(const char * name,OSSharedPtr<OSDictionary> table)6911 IOService::resourceMatching( const char * name,
6912 OSSharedPtr<OSDictionary> table)
6913 {
6914 OSDictionary * result = resourceMatching(name, table.get());
6915 if (table) {
6916 return OSSharedPtr<OSDictionary>(result, OSRetain);
6917 } else {
6918 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6919 }
6920 }
6921
6922
6923 OSDictionary *
propertyMatching(const OSSymbol * key,const OSObject * value,OSDictionary * table)6924 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6925 OSDictionary * table )
6926 {
6927 OSDictionary * properties;
6928
6929 properties = OSDictionary::withCapacity( 2 );
6930 if (!properties) {
6931 return NULL;
6932 }
6933 properties->setObject( key, value );
6934
6935 if (!table) {
6936 table = OSDictionary::withCapacity( 2 );
6937 }
6938 if (table) {
6939 table->setObject( gIOPropertyMatchKey, properties );
6940 }
6941
6942 properties->release();
6943
6944 return table;
6945 }
6946
6947
6948 OSSharedPtr<OSDictionary>
propertyMatching(const OSSymbol * key,const OSObject * value,OSSharedPtr<OSDictionary> table)6949 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6950 OSSharedPtr<OSDictionary> table)
6951 {
6952 OSDictionary * result = propertyMatching(key, value, table.get());
6953 if (table) {
6954 return OSSharedPtr<OSDictionary>(result, OSRetain);
6955 } else {
6956 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6957 }
6958 }
6959
6960
6961 OSDictionary *
registryEntryIDMatching(uint64_t entryID,OSDictionary * table)6962 IOService::registryEntryIDMatching( uint64_t entryID,
6963 OSDictionary * table )
6964 {
6965 OSNumber * num;
6966
6967 num = OSNumber::withNumber( entryID, 64 );
6968 if (!num) {
6969 return NULL;
6970 }
6971
6972 if (!table) {
6973 table = OSDictionary::withCapacity( 2 );
6974 }
6975 if (table) {
6976 table->setObject( gIORegistryEntryIDKey, num );
6977 }
6978
6979 if (num) {
6980 num->release();
6981 }
6982
6983 return table;
6984 }
6985
6986
6987 OSSharedPtr<OSDictionary>
registryEntryIDMatching(uint64_t entryID,OSSharedPtr<OSDictionary> table)6988 IOService::registryEntryIDMatching( uint64_t entryID,
6989 OSSharedPtr<OSDictionary> table)
6990 {
6991 OSDictionary * result = registryEntryIDMatching(entryID, table.get());
6992 if (table) {
6993 return OSSharedPtr<OSDictionary>(result, OSRetain);
6994 } else {
6995 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6996 }
6997 }
6998
6999
7000
7001 /*
7002 * _IOServiceNotifier
7003 */
7004
7005 // wait for all threads, other than the current one,
7006 // to exit the handler
7007
7008 void
wait()7009 _IOServiceNotifier::wait()
7010 {
7011 _IOServiceNotifierInvocation * next;
7012 bool doWait;
7013
7014 do {
7015 doWait = false;
7016 queue_iterate( &handlerInvocations, next,
7017 _IOServiceNotifierInvocation *, link) {
7018 if (next->thread != current_thread()) {
7019 doWait = true;
7020 break;
7021 }
7022 }
7023 if (doWait) {
7024 state |= kIOServiceNotifyWaiter;
7025 SLEEPNOTIFY(this);
7026 }
7027 } while (doWait);
7028 }
7029
7030 void
free()7031 _IOServiceNotifier::free()
7032 {
7033 assert( queue_empty( &handlerInvocations ));
7034
7035 if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
7036 Block_release(ref);
7037 }
7038
7039 OSObject::free();
7040 }
7041
7042 void
remove()7043 _IOServiceNotifier::remove()
7044 {
7045 LOCKWRITENOTIFY();
7046
7047 if (whence) {
7048 whence->removeObject((OSObject *) this );
7049 whence = NULL;
7050 }
7051 if (matching) {
7052 matching->release();
7053 matching = NULL;
7054 }
7055
7056 state &= ~kIOServiceNotifyEnable;
7057
7058 wait();
7059
7060 UNLOCKNOTIFY();
7061
7062 release();
7063 }
7064
7065 bool
disable()7066 _IOServiceNotifier::disable()
7067 {
7068 bool ret;
7069
7070 LOCKWRITENOTIFY();
7071
7072 ret = (0 != (kIOServiceNotifyEnable & state));
7073 state &= ~kIOServiceNotifyEnable;
7074 if (ret) {
7075 wait();
7076 }
7077
7078 UNLOCKNOTIFY();
7079
7080 return ret;
7081 }
7082
7083 void
enable(bool was)7084 _IOServiceNotifier::enable( bool was )
7085 {
7086 LOCKWRITENOTIFY();
7087 if (was) {
7088 state |= kIOServiceNotifyEnable;
7089 } else {
7090 state &= ~kIOServiceNotifyEnable;
7091 }
7092 UNLOCKNOTIFY();
7093 }
7094
7095
7096 /*
7097 * _IOServiceNullNotifier
7098 */
7099
7100 void
taggedRetain(const void * tag) const7101 _IOServiceNullNotifier::taggedRetain(const void *tag) const
7102 {
7103 }
7104 void
taggedRelease(const void * tag,const int when) const7105 _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
7106 {
7107 }
7108 void
free()7109 _IOServiceNullNotifier::free()
7110 {
7111 }
7112 void
wait()7113 _IOServiceNullNotifier::wait()
7114 {
7115 }
7116 void
remove()7117 _IOServiceNullNotifier::remove()
7118 {
7119 }
7120 void
enable(bool was)7121 _IOServiceNullNotifier::enable(bool was)
7122 {
7123 }
7124 bool
disable()7125 _IOServiceNullNotifier::disable()
7126 {
7127 return false;
7128 }
7129
7130 /*
7131 * IOResources
7132 */
7133
7134 IOService *
resources(void)7135 IOResources::resources( void )
7136 {
7137 IOResources * inst;
7138
7139 inst = new IOResources;
7140 if (inst && !inst->init()) {
7141 inst->release();
7142 inst = NULL;
7143 }
7144
7145 return inst;
7146 }
7147
7148 bool
init(OSDictionary * dictionary)7149 IOResources::init( OSDictionary * dictionary )
7150 {
7151 // Do super init first
7152 if (!IOService::init()) {
7153 return false;
7154 }
7155
7156 // Allow PAL layer to publish a value
7157 const char *property_name;
7158 int property_value;
7159
7160 pal_get_resource_property( &property_name, &property_value );
7161
7162 if (property_name) {
7163 OSNumber *num;
7164 const OSSymbol * sym;
7165
7166 if ((num = OSNumber::withNumber(property_value, 32)) != NULL) {
7167 if ((sym = OSSymbol::withCString( property_name)) != NULL) {
7168 this->setProperty( sym, num );
7169 sym->release();
7170 }
7171 num->release();
7172 }
7173 }
7174
7175 return true;
7176 }
7177
7178 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7179 IOResources::newUserClient(task_t owningTask, void * securityID,
7180 UInt32 type, OSDictionary * properties,
7181 IOUserClient ** handler)
7182 {
7183 return kIOReturnUnsupported;
7184 }
7185
7186 IOWorkLoop *
getWorkLoop() const7187 IOResources::getWorkLoop() const
7188 {
7189 // If we are the resource root
7190 // then use the platform's workloop
7191 if (this == (IOResources *) gIOResources) {
7192 return getPlatform()->getWorkLoop();
7193 } else {
7194 return IOService::getWorkLoop();
7195 }
7196 }
7197
7198 static bool
IOResourcesMatchPropertyTable(IOService * resources,OSDictionary * table)7199 IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
7200 {
7201 OSObject * prop;
7202 bool __block ok = true;
7203
7204 prop = table->getObject( gIOResourceMatchKey );
7205 if (prop) {
7206 prop->iterateObjects(^bool (OSObject * obj)
7207 {
7208 OSString *
7209 str = OSDynamicCast(OSString, obj);
7210 ok = (NULL != resources->getProperty(str));
7211 return !ok;
7212 });
7213 } else if ((prop = table->getObject(gIOResourceMatchedKey))) {
7214 OSObject * obj;
7215 OSArray * keys;
7216
7217 obj = resources->copyProperty(gIOResourceMatchedKey);
7218 keys = OSDynamicCast(OSArray, obj);
7219 ok = false;
7220 if (keys) {
7221 // assuming OSSymbol
7222 ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
7223 }
7224 OSSafeReleaseNULL(obj);
7225 }
7226
7227 return ok;
7228 }
7229
7230 bool
matchPropertyTable(OSDictionary * table)7231 IOResources::matchPropertyTable( OSDictionary * table )
7232 {
7233 return IOResourcesMatchPropertyTable(this, table);
7234 }
7235
7236 /*
7237 * IOUserResources
7238 */
7239
7240 IOService *
resources(void)7241 IOUserResources::resources( void )
7242 {
7243 IOUserResources * inst;
7244
7245 inst = OSTypeAlloc(IOUserResources);
7246 if (inst && !inst->init()) {
7247 inst->release();
7248 inst = NULL;
7249 }
7250
7251 return inst;
7252 }
7253
7254 bool
init(OSDictionary * dictionary)7255 IOUserResources::init( OSDictionary * dictionary )
7256 {
7257 // Do super init first
7258 if (!IOService::init()) {
7259 return false;
7260 }
7261 return true;
7262 }
7263
7264 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7265 IOUserResources::newUserClient(task_t owningTask, void * securityID,
7266 UInt32 type, OSDictionary * properties,
7267 IOUserClient ** handler)
7268 {
7269 return kIOReturnUnsupported;
7270 }
7271
7272 IOWorkLoop *
getWorkLoop() const7273 IOUserResources::getWorkLoop() const
7274 {
7275 return getPlatform()->getWorkLoop();
7276 }
7277
7278 bool
matchPropertyTable(OSDictionary * table)7279 IOUserResources::matchPropertyTable( OSDictionary * table )
7280 {
7281 return IOResourcesMatchPropertyTable(this, table);
7282 }
7283
7284 // --
7285
7286 void
consoleLockTimer(thread_call_param_t p0,thread_call_param_t p1)7287 IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
7288 {
7289 IOService::updateConsoleUsers(NULL, 0);
7290 }
7291
7292 void
updateConsoleUsers(OSArray * consoleUsers,IOMessage systemMessage,bool afterUserspaceReboot)7293 IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage, bool afterUserspaceReboot)
7294 {
7295 IORegistryEntry * regEntry;
7296 OSObject * locked = kOSBooleanFalse;
7297 uint32_t idx;
7298 bool publish;
7299 OSDictionary * user;
7300 clock_sec_t now = 0;
7301 clock_usec_t microsecs;
7302
7303 regEntry = IORegistryEntry::getRegistryRoot();
7304
7305 if (!gIOChosenEntry) {
7306 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
7307 }
7308
7309 IOLockLock(gIOConsoleUsersLock);
7310
7311 if (systemMessage) {
7312 sSystemPower = systemMessage;
7313 #if HIBERNATION
7314 if (kIOMessageSystemHasPoweredOn == systemMessage) {
7315 uint32_t lockState = IOHibernateWasScreenLocked();
7316 switch (lockState) {
7317 case 0:
7318 break;
7319 case kIOScreenLockLocked:
7320 case kIOScreenLockFileVaultDialog:
7321 gIOConsoleBooterLockState = kOSBooleanTrue;
7322 break;
7323 case kIOScreenLockNoLock:
7324 gIOConsoleBooterLockState = NULL;
7325 break;
7326 case kIOScreenLockUnlocked:
7327 default:
7328 gIOConsoleBooterLockState = kOSBooleanFalse;
7329 break;
7330 }
7331 }
7332 #endif /* HIBERNATION */
7333 }
7334
7335 if (consoleUsers) {
7336 OSNumber * num = NULL;
7337 bool loginLocked = true;
7338
7339 gIOConsoleLoggedIn = false;
7340 for (idx = 0;
7341 (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
7342 idx++) {
7343 gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
7344 && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
7345
7346 loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
7347 if (!num) {
7348 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
7349 }
7350 }
7351 #if HIBERNATION
7352 if (!loginLocked || afterUserspaceReboot) {
7353 gIOConsoleBooterLockState = NULL;
7354 }
7355 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
7356 (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
7357 gIOConsoleLoggedIn, loginLocked);
7358 #endif /* HIBERNATION */
7359 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
7360 }
7361
7362 if (!gIOConsoleLoggedIn
7363 || (kIOMessageSystemWillSleep == sSystemPower)
7364 || (kIOMessageSystemPagingOff == sSystemPower)) {
7365 if (afterUserspaceReboot) {
7366 // set "locked" to false after a user space reboot
7367 // because the reboot happens directly after a user
7368 // logs into the machine via fvunlock mode.
7369 locked = kOSBooleanFalse;
7370 } else {
7371 locked = kOSBooleanTrue;
7372 }
7373 }
7374 #if HIBERNATION
7375 else if (gIOConsoleBooterLockState) {
7376 locked = gIOConsoleBooterLockState;
7377 }
7378 #endif /* HIBERNATION */
7379 else if (gIOConsoleLockTime) {
7380 clock_get_calendar_microtime(&now, µsecs);
7381 if (gIOConsoleLockTime > now) {
7382 AbsoluteTime deadline;
7383 clock_sec_t interval;
7384 uint32_t interval32;
7385
7386 interval = (gIOConsoleLockTime - now);
7387 interval32 = (uint32_t) interval;
7388 if (interval32 != interval) {
7389 interval32 = UINT_MAX;
7390 }
7391 clock_interval_to_deadline(interval32, kSecondScale, &deadline);
7392 thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
7393 } else {
7394 locked = kOSBooleanTrue;
7395 }
7396 }
7397
7398 publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
7399 if (publish) {
7400 regEntry->setProperty(gIOConsoleLockedKey, locked);
7401 if (consoleUsers) {
7402 regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
7403 }
7404 OSIncrementAtomic( &gIOConsoleUsersSeed );
7405 }
7406
7407 #if HIBERNATION
7408 if (gIOChosenEntry) {
7409 if (locked == kOSBooleanTrue) {
7410 gIOScreenLockState = kIOScreenLockLocked;
7411 } else if (gIOConsoleLockTime) {
7412 gIOScreenLockState = kIOScreenLockUnlocked;
7413 } else {
7414 gIOScreenLockState = kIOScreenLockNoLock;
7415 }
7416 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
7417
7418 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
7419 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
7420 }
7421 #endif /* HIBERNATION */
7422
7423 IOLockUnlock(gIOConsoleUsersLock);
7424
7425 if (publish) {
7426 publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
7427
7428 MessageClientsContext context;
7429
7430 context.service = getServiceRoot();
7431 context.type = kIOMessageConsoleSecurityChange;
7432 context.argument = (void *) regEntry;
7433 context.argSize = 0;
7434
7435 applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
7436 &messageClientsApplier, &context );
7437 }
7438 }
7439
7440 IOReturn
setProperties(OSObject * properties)7441 IOResources::setProperties( OSObject * properties )
7442 {
7443 IOReturn err;
7444 const OSSymbol * key;
7445 OSDictionary * dict;
7446 OSCollectionIterator * iter;
7447
7448 if (!IOCurrentTaskHasEntitlement(kIOResourcesSetPropertyKey)) {
7449 err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
7450 if (kIOReturnSuccess != err) {
7451 return err;
7452 }
7453 }
7454
7455 dict = OSDynamicCast(OSDictionary, properties);
7456 if (NULL == dict) {
7457 return kIOReturnBadArgument;
7458 }
7459
7460 iter = OSCollectionIterator::withCollection( dict);
7461 if (NULL == iter) {
7462 return kIOReturnBadArgument;
7463 }
7464
7465 while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
7466 if (gIOConsoleUsersKey == key) {
7467 do{
7468 OSArray * consoleUsers;
7469 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
7470 if (!consoleUsers) {
7471 continue;
7472 }
7473 IOService::updateConsoleUsers(consoleUsers, 0);
7474 }while (false);
7475 }
7476
7477 publishResource( key, dict->getObject(key));
7478 }
7479
7480 iter->release();
7481
7482 return kIOReturnSuccess;
7483 }
7484
7485 /*
7486 * Helpers for matching dictionaries.
7487 * Keys existing in matching are checked in properties.
7488 * Keys may be a string or OSCollection of IOStrings
7489 */
7490
7491 bool
compareProperty(OSDictionary * matching,const char * key)7492 IOService::compareProperty( OSDictionary * matching,
7493 const char * key )
7494 {
7495 OSObject * value;
7496 OSObject * prop;
7497 bool ok;
7498
7499 value = matching->getObject( key );
7500 if (value) {
7501 prop = copyProperty(key);
7502 ok = value->isEqualTo(prop);
7503 if (prop) {
7504 prop->release();
7505 }
7506 } else {
7507 ok = true;
7508 }
7509
7510 return ok;
7511 }
7512
7513
7514 bool
compareProperty(OSDictionary * matching,const OSString * key)7515 IOService::compareProperty( OSDictionary * matching,
7516 const OSString * key )
7517 {
7518 OSObject * value;
7519 OSObject * prop;
7520 bool ok;
7521
7522 value = matching->getObject( key );
7523 if (value) {
7524 prop = copyProperty(key);
7525 ok = value->isEqualTo(prop);
7526 if (prop) {
7527 prop->release();
7528 }
7529 } else {
7530 ok = true;
7531 }
7532
7533 return ok;
7534 }
7535
7536 #ifndef __clang_analyzer__
7537 // Implementation of this function is hidden from the static analyzer.
7538 // The analyzer was worried about this function's confusing contract over
7539 // the 'keys' parameter. The contract is to either release it or not release it
7540 // depending on whether 'matching' is non-null. Such contracts are discouraged
7541 // but changing it now would break compatibility.
7542 bool
compareProperties(OSDictionary * matching,OSCollection * keys)7543 IOService::compareProperties( OSDictionary * matching,
7544 OSCollection * keys )
7545 {
7546 OSCollectionIterator * iter;
7547 const OSString * key;
7548 bool ok = true;
7549
7550 if (!matching || !keys) {
7551 return false;
7552 }
7553
7554 iter = OSCollectionIterator::withCollection( keys );
7555
7556 if (iter) {
7557 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
7558 ok = compareProperty( matching, key );
7559 }
7560
7561 iter->release();
7562 }
7563 keys->release(); // !! consume a ref !!
7564
7565 return ok;
7566 }
7567 #endif // __clang_analyzer__
7568
7569 /* Helper to add a location matching dict to the table */
7570
7571 OSDictionary *
addLocation(OSDictionary * table)7572 IOService::addLocation( OSDictionary * table )
7573 {
7574 OSDictionary * dict;
7575
7576 if (!table) {
7577 return NULL;
7578 }
7579
7580 dict = OSDictionary::withCapacity( 1 );
7581 if (dict) {
7582 bool ok = table->setObject( gIOLocationMatchKey, dict );
7583 dict->release();
7584 if (!ok) {
7585 dict = NULL;
7586 }
7587 }
7588
7589 return dict;
7590 }
7591
7592 /*
7593 * Go looking for a provider to match a location dict.
7594 */
7595
7596 IOService *
matchLocation(IOService *)7597 IOService::matchLocation( IOService * /* client */ )
7598 {
7599 IOService * parent;
7600
7601 parent = getProvider();
7602
7603 if (parent) {
7604 parent = parent->matchLocation( this );
7605 }
7606
7607 return parent;
7608 }
7609
7610 OSDictionary *
_copyPropertiesForMatching(void)7611 IOService::_copyPropertiesForMatching(void)
7612 {
7613 OSDictionary * matchProps;
7614
7615 matchProps = dictionaryWithProperties();
7616 if (matchProps) {
7617 // merge will check the OSDynamicCast
7618 matchProps->merge((const OSDictionary *)matchProps->getObject(gIOUserServicePropertiesKey));
7619 }
7620 return matchProps;
7621 }
7622
7623 bool
matchInternal(OSDictionary * table,uint32_t options,uint32_t * did)7624 IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
7625 {
7626 OSString * matched;
7627 OSObject * obj;
7628 OSString * str;
7629 OSDictionary * matchProps;
7630 IORegistryEntry * entry;
7631 OSNumber * num;
7632 bool match = true;
7633 bool changesOK = (0 != (kIOServiceChangesOK & options));
7634 uint32_t count;
7635 uint32_t done;
7636
7637 do{
7638 count = table->getCount();
7639 done = 0;
7640 matchProps = NULL;
7641 bool isUser;
7642
7643 isUser = (NULL != table->getObject(gIOServiceNotificationUserKey));
7644 if (isUser) {
7645 done++;
7646 match = (0 == (kIOServiceUserInvisibleMatchState & __state[0]));
7647 if ((!match) || (done == count)) {
7648 break;
7649 }
7650 }
7651
7652 if (propertyExists(gIOExclaveAssignedKey)) {
7653 if (!table->getObject(gIOExclaveProxyKey) && !isUser) {
7654 match = false;
7655 break;
7656 }
7657 } else if (table->getObject(gIOExclaveProxyKey)) {
7658 match = false;
7659 break;
7660 }
7661
7662 if (table->getObject(gIOCompatibilityMatchKey)) {
7663 done++;
7664 obj = copyProperty(gIOCompatibilityPropertiesKey);
7665 matchProps = OSDynamicCast(OSDictionary, obj);
7666 if (!matchProps) {
7667 OSSafeReleaseNULL(obj);
7668 }
7669 }
7670
7671 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
7672 if (str) {
7673 done++;
7674 if (matchProps && (obj = matchProps->getObject(gIOClassKey))) {
7675 match = str->isEqualTo(obj);
7676 } else {
7677 match = ((kIOServiceClassDone & options) || (NULL != metaCast(str)));
7678 }
7679
7680 #if MATCH_DEBUG
7681 match = (0 != metaCast( str ));
7682 if ((kIOServiceClassDone & options) && !match) {
7683 panic("classDone");
7684 }
7685 #endif
7686 if ((!match) || (done == count)) {
7687 break;
7688 }
7689 }
7690
7691 obj = table->getObject( gIONameMatchKey );
7692 if (obj) {
7693 done++;
7694 match = compareNames( obj, changesOK ? &matched : NULL );
7695 if (!match) {
7696 break;
7697 }
7698 if (changesOK && matched) {
7699 // leave a hint as to which name matched
7700 table->setObject( gIONameMatchedKey, matched );
7701 matched->release();
7702 }
7703 if (done == count) {
7704 break;
7705 }
7706 }
7707
7708 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
7709 if (str) {
7710 const OSSymbol * sym;
7711 done++;
7712 match = false;
7713 sym = copyLocation();
7714 if (sym) {
7715 match = sym->isEqualTo( str );
7716 sym->release();
7717 }
7718 if ((!match) || (done == count)) {
7719 break;
7720 }
7721 }
7722
7723 obj = table->getObject( gIOPropertyMatchKey );
7724 if (obj) {
7725 OSDictionary * nextDict;
7726 OSIterator * iter;
7727 done++;
7728 match = false;
7729 if (!matchProps) {
7730 matchProps = _copyPropertiesForMatching();
7731 }
7732 if (matchProps) {
7733 nextDict = OSDynamicCast( OSDictionary, obj);
7734 if (nextDict) {
7735 iter = NULL;
7736 } else {
7737 iter = OSCollectionIterator::withCollection(
7738 OSDynamicCast(OSCollection, obj));
7739 }
7740
7741 while (nextDict
7742 || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
7743 iter->getNextObject()))))) {
7744 match = matchProps->isEqualTo( nextDict, nextDict);
7745 if (match) {
7746 break;
7747 }
7748 nextDict = NULL;
7749 }
7750 if (iter) {
7751 iter->release();
7752 }
7753 }
7754 if ((!match) || (done == count)) {
7755 break;
7756 }
7757 }
7758
7759 obj = table->getObject( gIOPropertyExistsMatchKey );
7760 if (obj) {
7761 OSString * nextKey;
7762 OSIterator * iter;
7763 done++;
7764 match = false;
7765 if (!matchProps) {
7766 matchProps = _copyPropertiesForMatching();
7767 }
7768 if (matchProps) {
7769 nextKey = OSDynamicCast( OSString, obj);
7770 if (nextKey) {
7771 iter = NULL;
7772 } else {
7773 iter = OSCollectionIterator::withCollection(
7774 OSDynamicCast(OSCollection, obj));
7775 }
7776
7777 while (nextKey
7778 || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
7779 iter->getNextObject()))))) {
7780 match = (NULL != matchProps->getObject(nextKey));
7781 if (match) {
7782 break;
7783 }
7784 nextKey = NULL;
7785 }
7786 if (iter) {
7787 iter->release();
7788 }
7789 }
7790 if ((!match) || (done == count)) {
7791 break;
7792 }
7793 }
7794
7795 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
7796 if (str) {
7797 done++;
7798 entry = IORegistryEntry::fromPath( str->getCStringNoCopy());
7799 match = (this == entry);
7800 if (entry) {
7801 entry->release();
7802 }
7803 if (!match && matchProps && (obj = matchProps->getObject(gIOPathKey))) {
7804 match = str->isEqualTo(obj);
7805 }
7806 if ((!match) || (done == count)) {
7807 break;
7808 }
7809 }
7810
7811 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
7812 if (num) {
7813 done++;
7814 match = (getRegistryEntryID() == num->unsigned64BitValue());
7815 if ((!match) || (done == count)) {
7816 break;
7817 }
7818 }
7819
7820 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
7821 if (num) {
7822 OSIterator * iter;
7823 IOService * service = NULL;
7824 UInt32 serviceCount = 0;
7825
7826 done++;
7827 iter = getClientIterator();
7828 if (iter) {
7829 while ((service = (IOService *) iter->getNextObject())) {
7830 if (kIOServiceInactiveState & service->__state[0]) {
7831 continue;
7832 }
7833 if (NULL == service->getProperty( gIOMatchCategoryKey )) {
7834 continue;
7835 }
7836 ++serviceCount;
7837 }
7838 iter->release();
7839 }
7840 match = (serviceCount == num->unsigned32BitValue());
7841 if ((!match) || (done == count)) {
7842 break;
7843 }
7844 }
7845
7846 #define propMatch(key) \
7847 obj = table->getObject(key); \
7848 if (obj) \
7849 { \
7850 OSObject * prop; \
7851 done++; \
7852 prop = copyProperty(key); \
7853 match = obj->isEqualTo(prop); \
7854 if (prop) prop->release(); \
7855 if ((!match) || (done == count)) break; \
7856 }
7857 propMatch(gIOBSDNameKey)
7858 propMatch(gIOBSDMajorKey)
7859 propMatch(gIOBSDMinorKey)
7860 propMatch(gIOBSDUnitKey)
7861 #undef propMatch
7862 }while (false);
7863
7864 OSSafeReleaseNULL(matchProps);
7865
7866 if (did) {
7867 *did = done;
7868 }
7869 return match;
7870 }
7871
7872 bool
passiveMatch(OSDictionary * table,bool changesOK)7873 IOService::passiveMatch( OSDictionary * table, bool changesOK )
7874 {
7875 return matchPassive(table, changesOK ? kIOServiceChangesOK : 0);
7876 }
7877
7878 bool
matchPassive(OSDictionary * table,uint32_t options)7879 IOService::matchPassive(OSDictionary * table, uint32_t options)
7880 {
7881 IOService * where;
7882 OSDictionary * nextTable;
7883 SInt32 score;
7884 OSNumber * newPri;
7885 bool match = true;
7886 bool matchParent = false;
7887 uint32_t count;
7888 uint32_t done;
7889
7890 assert( table );
7891
7892 #if defined(XNU_TARGET_OS_OSX)
7893 OSArray* aliasServiceRegIds = NULL;
7894 IOService* foundAlternateService = NULL;
7895 #endif /* defined(XNU_TARGET_OS_OSX) */
7896
7897 #if MATCH_DEBUG
7898 OSDictionary * root = table;
7899 #endif
7900
7901 where = this;
7902 do{
7903 do{
7904 count = table->getCount();
7905 if (!(kIOServiceInternalDone & options)) {
7906 match = where->matchInternal(table, options, &done);
7907 // don't call family if we've done all the entries in the table
7908 if ((!match) || (done == count)) {
7909 break;
7910 }
7911 }
7912
7913 // pass in score from property table
7914 score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
7915
7916 // do family specific matching
7917 match = where->matchPropertyTable( table, &score );
7918
7919 if (!match) {
7920 #if IOMATCHDEBUG
7921 if (kIOLogMatch & getDebugFlags( table )) {
7922 LOG("%s: family specific matching fails\n", where->getName());
7923 }
7924 #endif
7925 break;
7926 }
7927
7928 if (kIOServiceChangesOK & options) {
7929 // save the score
7930 newPri = OSNumber::withNumber( score, 32 );
7931 if (newPri) {
7932 table->setObject( gIOProbeScoreKey, newPri );
7933 newPri->release();
7934 }
7935 }
7936
7937 options = 0;
7938 matchParent = false;
7939
7940 nextTable = OSDynamicCast(OSDictionary,
7941 table->getObject( gIOParentMatchKey ));
7942 if (nextTable) {
7943 // look for a matching entry anywhere up to root
7944 match = false;
7945 matchParent = true;
7946 table = nextTable;
7947 break;
7948 }
7949
7950 table = OSDynamicCast(OSDictionary,
7951 table->getObject( gIOLocationMatchKey ));
7952 if (table) {
7953 // look for a matching entry at matchLocation()
7954 match = false;
7955 where = where->getProvider();
7956 if (where && (where = where->matchLocation(where))) {
7957 continue;
7958 }
7959 }
7960 break;
7961 }while (true);
7962
7963 if (match == true) {
7964 break;
7965 }
7966
7967 if (matchParent == true) {
7968 #if defined(XNU_TARGET_OS_OSX)
7969 // check if service has an alias to search its other "parents" if a parent match isn't found
7970 OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
7971 OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
7972 if (alternateRegistryID != NULL) {
7973 if (aliasServiceRegIds == NULL) {
7974 aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
7975 }
7976 aliasServiceRegIds->setObject(alternateRegistryID);
7977 }
7978 OSSafeReleaseNULL(prop);
7979 #endif /* defined(XNU_TARGET_OS_OSX) */
7980 } else {
7981 break;
7982 }
7983
7984 where = where->getProvider();
7985 #if defined(XNU_TARGET_OS_OSX)
7986 if (where == NULL) {
7987 // there were no matching parent services, check to see if there are aliased services that have a matching parent
7988 if (aliasServiceRegIds != NULL) {
7989 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
7990 if (numAliasedServices != 0) {
7991 OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
7992 if (alternateRegistryID != NULL) {
7993 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
7994 aliasServiceRegIds->removeObject(numAliasedServices - 1);
7995 if (alternateMatchingDict != NULL) {
7996 OSSafeReleaseNULL(foundAlternateService);
7997 foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
7998 alternateMatchingDict->release();
7999 if (foundAlternateService != NULL) {
8000 where = foundAlternateService;
8001 }
8002 }
8003 }
8004 }
8005 }
8006 }
8007 #endif /* defined(XNU_TARGET_OS_OSX) */
8008 }while (where != NULL);
8009
8010 #if defined(XNU_TARGET_OS_OSX)
8011 OSSafeReleaseNULL(foundAlternateService);
8012 OSSafeReleaseNULL(aliasServiceRegIds);
8013 #endif /* defined(XNU_TARGET_OS_OSX) */
8014
8015 #if MATCH_DEBUG
8016 if (where != this) {
8017 OSSerialize * s = OSSerialize::withCapacity(128);
8018 root->serialize(s);
8019 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
8020 s->release();
8021 }
8022 #endif
8023
8024 return match;
8025 }
8026
8027
8028 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)8029 IOService::newUserClient( task_t owningTask, void * securityID,
8030 UInt32 type, OSDictionary * properties,
8031 IOUserClient ** handler )
8032 {
8033 const OSSymbol *userClientClass = NULL;
8034 IOUserClient *client;
8035 OSObject *prop;
8036 OSObject *temp;
8037
8038 if (reserved && reserved->uvars && reserved->uvars->userServer) {
8039 return reserved->uvars->userServer->serviceNewUserClient(this, owningTask, securityID, type, properties, handler);
8040 }
8041
8042 if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
8043 return kIOReturnSuccess;
8044 }
8045
8046 // First try my own properties for a user client class name
8047 prop = copyProperty(gIOUserClientClassKey);
8048 if (prop) {
8049 if (OSDynamicCast(OSSymbol, prop)) {
8050 userClientClass = (const OSSymbol *) prop;
8051 prop = NULL;
8052 } else if (OSDynamicCast(OSString, prop)) {
8053 userClientClass = OSSymbol::withString((OSString *) prop);
8054 OSSafeReleaseNULL(prop);
8055 if (userClientClass) {
8056 setProperty(gIOUserClientClassKey,
8057 (OSObject *) userClientClass);
8058 }
8059 } else {
8060 OSSafeReleaseNULL(prop);
8061 }
8062 }
8063
8064 // Didn't find one so lets just bomb out now without further ado.
8065 if (!userClientClass) {
8066 return kIOReturnUnsupported;
8067 }
8068
8069 // This reference is consumed by the IOServiceOpen call
8070 temp = OSMetaClass::allocClassWithName(userClientClass);
8071 OSSafeReleaseNULL(userClientClass);
8072 if (!temp) {
8073 return kIOReturnNoMemory;
8074 }
8075
8076 if (OSDynamicCast(IOUserClient, temp)) {
8077 client = (IOUserClient *) temp;
8078 } else {
8079 temp->release();
8080 return kIOReturnUnsupported;
8081 }
8082
8083 if (!client->initWithTask(owningTask, securityID, type, properties)) {
8084 client->release();
8085 return kIOReturnBadArgument;
8086 }
8087
8088 if (!client->attach(this)) {
8089 client->release();
8090 return kIOReturnUnsupported;
8091 }
8092
8093 if (!client->start(this)) {
8094 client->detach(this);
8095 client->release();
8096 return kIOReturnUnsupported;
8097 }
8098
8099 *handler = client;
8100 return kIOReturnSuccess;
8101 }
8102
8103 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,OSSharedPtr<IOUserClient> & handler)8104 IOService::newUserClient( task_t owningTask, void * securityID,
8105 UInt32 type, OSDictionary * properties,
8106 OSSharedPtr<IOUserClient>& handler )
8107 {
8108 IOUserClient* handlerRaw = NULL;
8109 IOReturn result = newUserClient(owningTask, securityID, type, properties, &handlerRaw);
8110 handler.reset(handlerRaw, OSNoRetain);
8111 return result;
8112 }
8113
8114 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,IOUserClient ** handler)8115 IOService::newUserClient( task_t owningTask, void * securityID,
8116 UInt32 type, IOUserClient ** handler )
8117 {
8118 return kIOReturnUnsupported;
8119 }
8120
8121 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSSharedPtr<IOUserClient> & handler)8122 IOService::newUserClient( task_t owningTask, void * securityID,
8123 UInt32 type, OSSharedPtr<IOUserClient>& handler )
8124 {
8125 IOUserClient* handlerRaw = nullptr;
8126 IOReturn result = IOService::newUserClient(owningTask, securityID, type, &handlerRaw);
8127 handler.reset(handlerRaw, OSNoRetain);
8128 return result;
8129 }
8130
8131
8132 IOReturn
requestProbe(IOOptionBits options)8133 IOService::requestProbe( IOOptionBits options )
8134 {
8135 return kIOReturnUnsupported;
8136 }
8137
8138 bool
hasUserServer() const8139 IOService::hasUserServer() const
8140 {
8141 return reserved && reserved->uvars && reserved->uvars->userServer;
8142 }
8143
8144 /*
8145 * Convert an IOReturn to text. Subclasses which add additional
8146 * IOReturn's should override this method and call
8147 * super::stringFromReturn if the desired value is not found.
8148 */
8149
8150 const char *
stringFromReturn(IOReturn rtn)8151 IOService::stringFromReturn( IOReturn rtn )
8152 {
8153 static const IONamedValue IOReturn_values[] = {
8154 {kIOReturnSuccess, "success" },
8155 {kIOReturnError, "general error" },
8156 {kIOReturnNoMemory, "memory allocation error" },
8157 {kIOReturnNoResources, "resource shortage" },
8158 {kIOReturnIPCError, "Mach IPC failure" },
8159 {kIOReturnNoDevice, "no such device" },
8160 {kIOReturnNotPrivileged, "privilege violation" },
8161 {kIOReturnBadArgument, "invalid argument" },
8162 {kIOReturnLockedRead, "device is read locked" },
8163 {kIOReturnLockedWrite, "device is write locked" },
8164 {kIOReturnExclusiveAccess, "device is exclusive access" },
8165 {kIOReturnBadMessageID, "bad IPC message ID" },
8166 {kIOReturnUnsupported, "unsupported function" },
8167 {kIOReturnVMError, "virtual memory error" },
8168 {kIOReturnInternalError, "internal driver error" },
8169 {kIOReturnIOError, "I/O error" },
8170 {kIOReturnCannotLock, "cannot acquire lock" },
8171 {kIOReturnNotOpen, "device is not open" },
8172 {kIOReturnNotReadable, "device is not readable" },
8173 {kIOReturnNotWritable, "device is not writeable" },
8174 {kIOReturnNotAligned, "alignment error" },
8175 {kIOReturnBadMedia, "media error" },
8176 {kIOReturnStillOpen, "device is still open" },
8177 {kIOReturnRLDError, "rld failure" },
8178 {kIOReturnDMAError, "DMA failure" },
8179 {kIOReturnBusy, "device is busy" },
8180 {kIOReturnTimeout, "I/O timeout" },
8181 {kIOReturnOffline, "device is offline" },
8182 {kIOReturnNotReady, "device is not ready" },
8183 {kIOReturnNotAttached, "device/channel is not attached" },
8184 {kIOReturnNoChannels, "no DMA channels available" },
8185 {kIOReturnNoSpace, "no space for data" },
8186 {kIOReturnPortExists, "device port already exists" },
8187 {kIOReturnCannotWire, "cannot wire physical memory" },
8188 {kIOReturnNoInterrupt, "no interrupt attached" },
8189 {kIOReturnNoFrames, "no DMA frames enqueued" },
8190 {kIOReturnMessageTooLarge, "message is too large" },
8191 {kIOReturnNotPermitted, "operation is not permitted" },
8192 {kIOReturnNoPower, "device is without power" },
8193 {kIOReturnNoMedia, "media is not present" },
8194 {kIOReturnUnformattedMedia, "media is not formatted" },
8195 {kIOReturnUnsupportedMode, "unsupported mode" },
8196 {kIOReturnUnderrun, "data underrun" },
8197 {kIOReturnOverrun, "data overrun" },
8198 {kIOReturnDeviceError, "device error" },
8199 {kIOReturnNoCompletion, "no completion routine" },
8200 {kIOReturnAborted, "operation was aborted" },
8201 {kIOReturnNoBandwidth, "bus bandwidth would be exceeded" },
8202 {kIOReturnNotResponding, "device is not responding" },
8203 {kIOReturnInvalid, "unanticipated driver error" },
8204 {0, NULL }
8205 };
8206
8207 return IOFindNameForValue(rtn, IOReturn_values);
8208 }
8209
8210 /*
8211 * Convert an IOReturn to an errno.
8212 */
8213 int
errnoFromReturn(IOReturn rtn)8214 IOService::errnoFromReturn( IOReturn rtn )
8215 {
8216 if (unix_err(err_get_code(rtn)) == rtn) {
8217 return err_get_code(rtn);
8218 }
8219
8220 switch (rtn) {
8221 // (obvious match)
8222 case kIOReturnSuccess:
8223 return 0;
8224 case kIOReturnNoMemory:
8225 return ENOMEM;
8226 case kIOReturnNoDevice:
8227 return ENXIO;
8228 case kIOReturnVMError:
8229 return EFAULT;
8230 case kIOReturnNotPermitted:
8231 return EPERM;
8232 case kIOReturnNotPrivileged:
8233 return EACCES;
8234 case kIOReturnIOError:
8235 return EIO;
8236 case kIOReturnNotWritable:
8237 return EROFS;
8238 case kIOReturnBadArgument:
8239 return EINVAL;
8240 case kIOReturnUnsupported:
8241 return ENOTSUP;
8242 case kIOReturnBusy:
8243 return EBUSY;
8244 case kIOReturnNoPower:
8245 return EPWROFF;
8246 case kIOReturnDeviceError:
8247 return EDEVERR;
8248 case kIOReturnTimeout:
8249 return ETIMEDOUT;
8250 case kIOReturnMessageTooLarge:
8251 return EMSGSIZE;
8252 case kIOReturnNoSpace:
8253 return ENOSPC;
8254 case kIOReturnCannotLock:
8255 return ENOLCK;
8256
8257 // (best match)
8258 case kIOReturnBadMessageID:
8259 case kIOReturnNoCompletion:
8260 case kIOReturnNotAligned:
8261 return EINVAL;
8262 case kIOReturnNotReady:
8263 return EBUSY;
8264 case kIOReturnRLDError:
8265 return EBADMACHO;
8266 case kIOReturnPortExists:
8267 case kIOReturnStillOpen:
8268 return EEXIST;
8269 case kIOReturnExclusiveAccess:
8270 case kIOReturnLockedRead:
8271 case kIOReturnLockedWrite:
8272 case kIOReturnNotOpen:
8273 case kIOReturnNotReadable:
8274 return EACCES;
8275 case kIOReturnCannotWire:
8276 case kIOReturnNoResources:
8277 return ENOMEM;
8278 case kIOReturnAborted:
8279 case kIOReturnOffline:
8280 case kIOReturnNotResponding:
8281 return EBUSY;
8282 case kIOReturnBadMedia:
8283 case kIOReturnNoMedia:
8284 case kIOReturnNotAttached:
8285 case kIOReturnUnformattedMedia:
8286 return ENXIO; // (media error)
8287 case kIOReturnDMAError:
8288 case kIOReturnOverrun:
8289 case kIOReturnUnderrun:
8290 return EIO; // (transfer error)
8291 case kIOReturnNoBandwidth:
8292 case kIOReturnNoChannels:
8293 case kIOReturnNoFrames:
8294 case kIOReturnNoInterrupt:
8295 return EIO; // (hardware error)
8296 case kIOReturnError:
8297 case kIOReturnInternalError:
8298 case kIOReturnInvalid:
8299 return EIO; // (generic error)
8300 case kIOReturnIPCError:
8301 return EIO; // (ipc error)
8302 default:
8303 return EIO; // (all other errors)
8304 }
8305 }
8306
8307 IOReturn
message(UInt32 type,IOService * provider,void * argument)8308 IOService::message( UInt32 type, IOService * provider,
8309 void * argument )
8310 {
8311 /*
8312 * Generic entry point for calls from the provider. A return value of
8313 * kIOReturnSuccess indicates that the message was received, and where
8314 * applicable, that it was successful.
8315 */
8316
8317 return kIOReturnUnsupported;
8318 }
8319
8320 /*
8321 * Device memory
8322 */
8323
8324 IOItemCount
getDeviceMemoryCount(void)8325 IOService::getDeviceMemoryCount( void )
8326 {
8327 OSArray * array;
8328 IOItemCount count;
8329
8330 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8331 if (array) {
8332 count = array->getCount();
8333 } else {
8334 count = 0;
8335 }
8336
8337 return count;
8338 }
8339
8340 IODeviceMemory *
getDeviceMemoryWithIndex(unsigned int index)8341 IOService::getDeviceMemoryWithIndex( unsigned int index )
8342 {
8343 OSArray * array;
8344 IODeviceMemory * range;
8345
8346 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8347 if (array) {
8348 range = (IODeviceMemory *) array->getObject( index );
8349 } else {
8350 range = NULL;
8351 }
8352
8353 return range;
8354 }
8355
8356 IOMemoryMap *
mapDeviceMemoryWithIndex(unsigned int index,IOOptionBits options)8357 IOService::mapDeviceMemoryWithIndex( unsigned int index,
8358 IOOptionBits options )
8359 {
8360 IODeviceMemory * range;
8361 IOMemoryMap * map;
8362
8363 range = getDeviceMemoryWithIndex( index );
8364 if (range) {
8365 map = range->map( options );
8366 } else {
8367 map = NULL;
8368 }
8369
8370 return map;
8371 }
8372
8373 OSArray *
getDeviceMemory(void)8374 IOService::getDeviceMemory( void )
8375 {
8376 return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8377 }
8378
8379
8380 void
setDeviceMemory(OSArray * array)8381 IOService::setDeviceMemory( OSArray * array )
8382 {
8383 setProperty( gIODeviceMemoryKey, array);
8384 }
8385
8386 static void
requireMaxCpuDelay(IOService * service,UInt32 ns,UInt32 delayType)8387 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
8388 {
8389 static const UInt kNoReplace = -1U; // Must be an illegal index
8390 UInt replace = kNoReplace;
8391 bool setCpuDelay = false;
8392
8393 IORecursiveLockLock(sCpuDelayLock);
8394
8395 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8396 __typed_allocators_ignore_push
8397 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8398 __typed_allocators_ignore_pop
8399 IOService * holder = NULL;
8400
8401 if (ns) {
8402 const CpuDelayEntry ne = {service, ns, delayType};
8403 holder = service;
8404 // Set maximum delay.
8405 for (UInt i = 0; i < count; i++) {
8406 IOService *thisService = entries[i].fService;
8407 bool sameType = (delayType == entries[i].fDelayType);
8408 if ((service == thisService) && sameType) {
8409 replace = i;
8410 } else if (!thisService) {
8411 if (kNoReplace == replace) {
8412 replace = i;
8413 }
8414 } else if (sameType) {
8415 const UInt32 thisMax = entries[i].fMaxDelay;
8416 if (thisMax < ns) {
8417 ns = thisMax;
8418 holder = thisService;
8419 }
8420 }
8421 }
8422
8423 setCpuDelay = true;
8424 if (kNoReplace == replace) {
8425 __typed_allocators_ignore_push
8426 sCpuDelayData->appendBytes(&ne, sizeof(ne));
8427 __typed_allocators_ignore_pop
8428 } else {
8429 entries[replace] = ne;
8430 }
8431 } else {
8432 ns = -1U; // Set to max unsigned, i.e. no restriction
8433
8434 for (UInt i = 0; i < count; i++) {
8435 // Clear a maximum delay.
8436 IOService *thisService = entries[i].fService;
8437 if (thisService && (delayType == entries[i].fDelayType)) {
8438 UInt32 thisMax = entries[i].fMaxDelay;
8439 if (service == thisService) {
8440 replace = i;
8441 } else if (thisMax < ns) {
8442 ns = thisMax;
8443 holder = thisService;
8444 }
8445 }
8446 }
8447
8448 // Check if entry found
8449 if (kNoReplace != replace) {
8450 entries[replace].fService = NULL; // Null the entry
8451 setCpuDelay = true;
8452 }
8453 }
8454
8455 if (setCpuDelay) {
8456 if (holder && debug_boot_arg) {
8457 strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
8458 }
8459
8460 // Must be safe to call from locked context
8461 if (delayType == kCpuDelayBusStall) {
8462 #if defined(__x86_64__)
8463 ml_set_maxbusdelay(ns);
8464 #endif /* defined(__x86_64__) */
8465 }
8466 #if defined(__x86_64__)
8467 else if (delayType == kCpuDelayInterrupt) {
8468 ml_set_maxintdelay(ns);
8469 }
8470 #endif /* defined(__x86_64__) */
8471 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
8472 sCPULatencySet[delayType]->setValue(ns);
8473
8474 OSArray * handlers = sCpuLatencyHandlers[delayType];
8475 IOService * target;
8476 if (handlers) {
8477 for (unsigned int idx = 0;
8478 (target = (IOService *) handlers->getObject(idx));
8479 idx++) {
8480 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8481 (void *) (uintptr_t) ns, holder,
8482 NULL, NULL);
8483 }
8484 }
8485 }
8486
8487 IORecursiveLockUnlock(sCpuDelayLock);
8488 }
8489
8490 static IOReturn
setLatencyHandler(UInt32 delayType,IOService * target,bool enable)8491 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
8492 {
8493 IOReturn result = kIOReturnNotFound;
8494 OSArray * array;
8495 unsigned int idx;
8496
8497 IORecursiveLockLock(sCpuDelayLock);
8498
8499 do{
8500 if (enable && !sCpuLatencyHandlers[delayType]) {
8501 sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
8502 }
8503 array = sCpuLatencyHandlers[delayType];
8504 if (!array) {
8505 break;
8506 }
8507 idx = array->getNextIndexOfObject(target, 0);
8508 if (!enable) {
8509 if (-1U != idx) {
8510 array->removeObject(idx);
8511 result = kIOReturnSuccess;
8512 }
8513 } else {
8514 if (-1U != idx) {
8515 result = kIOReturnExclusiveAccess;
8516 break;
8517 }
8518 array->setObject(target);
8519
8520 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8521 __typed_allocators_ignore_push
8522 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8523 __typed_allocators_ignore_pop
8524 UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
8525 IOService * holder = NULL;
8526
8527 for (UInt i = 0; i < count; i++) {
8528 if (entries[i].fService
8529 && (delayType == entries[i].fDelayType)
8530 && (entries[i].fMaxDelay < ns)) {
8531 ns = entries[i].fMaxDelay;
8532 holder = entries[i].fService;
8533 }
8534 }
8535 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8536 (void *) (uintptr_t) ns, holder,
8537 NULL, NULL);
8538 result = kIOReturnSuccess;
8539 }
8540 }while (false);
8541
8542 IORecursiveLockUnlock(sCpuDelayLock);
8543
8544 return result;
8545 }
8546
8547 IOReturn
requireMaxBusStall(UInt32 ns)8548 IOService::requireMaxBusStall(UInt32 ns)
8549 {
8550 #if !defined(__x86_64__)
8551 switch (ns) {
8552 case kIOMaxBusStall40usec:
8553 case kIOMaxBusStall30usec:
8554 case kIOMaxBusStall25usec:
8555 case kIOMaxBusStall20usec:
8556 case kIOMaxBusStall10usec:
8557 case kIOMaxBusStall5usec:
8558 case kIOMaxBusStallNone:
8559 break;
8560 default:
8561 return kIOReturnBadArgument;
8562 }
8563 #endif /* !defined(__x86_64__) */
8564 requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
8565 return kIOReturnSuccess;
8566 }
8567
8568 IOReturn
requireMaxInterruptDelay(uint32_t ns)8569 IOService::requireMaxInterruptDelay(uint32_t ns)
8570 {
8571 #if defined(__x86_64__)
8572 requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
8573 return kIOReturnSuccess;
8574 #else /* defined(__x86_64__) */
8575 return kIOReturnUnsupported;
8576 #endif /* defined(__x86_64__) */
8577 }
8578
8579 /*
8580 * Device interrupts
8581 */
8582
8583 IOReturn
resolveInterrupt(IOService * nub,int source)8584 IOService::resolveInterrupt(IOService *nub, int source)
8585 {
8586 IOInterruptController *interruptController;
8587 OSArray *array;
8588 OSData *data;
8589 OSSymbol *interruptControllerName;
8590 unsigned int numSources;
8591 IOInterruptSource *interruptSources;
8592 IOInterruptSourcePrivate *interruptSourcesPrivate;
8593
8594 // Get the parents list from the nub.
8595 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
8596 if (array == NULL) {
8597 return kIOReturnNoResources;
8598 }
8599
8600 // Allocate space for the IOInterruptSources if needed... then return early.
8601 if (nub->_interruptSources == NULL) {
8602 numSources = array->getCount();
8603 interruptSources = IONewZero(IOInterruptSource, numSources);
8604 interruptSourcesPrivate = IONewZero(IOInterruptSourcePrivate, numSources);
8605
8606 if (interruptSources == NULL || interruptSourcesPrivate == NULL) {
8607 IODelete(interruptSources, IOInterruptSource, numSources);
8608 IODelete(interruptSourcesPrivate, IOInterruptSourcePrivate, numSources);
8609 return kIOReturnNoMemory;
8610 }
8611
8612 nub->_numInterruptSources = numSources;
8613 nub->_interruptSources = interruptSources;
8614 nub->reserved->interruptSourcesPrivate = interruptSourcesPrivate;
8615 return kIOReturnSuccess;
8616 }
8617
8618 interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
8619 if (interruptControllerName == NULL) {
8620 return kIOReturnNoResources;
8621 }
8622
8623 interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
8624 if (interruptController == NULL) {
8625 return kIOReturnNoResources;
8626 }
8627
8628 // Get the interrupt numbers from the nub.
8629 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
8630 if (array == NULL) {
8631 return kIOReturnNoResources;
8632 }
8633 data = OSDynamicCast(OSData, array->getObject(source));
8634 if (data == NULL) {
8635 return kIOReturnNoResources;
8636 }
8637
8638 // Set the interruptController and interruptSource in the nub's table.
8639 interruptSources = nub->_interruptSources;
8640 interruptSources[source].interruptController = interruptController;
8641 interruptSources[source].vectorData = data;
8642
8643 return kIOReturnSuccess;
8644 }
8645
8646 IOReturn
lookupInterrupt(int source,bool resolve,IOInterruptController ** interruptController)8647 IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
8648 {
8649 IOReturn ret;
8650
8651 /* Make sure the _interruptSources are set */
8652 if (_interruptSources == NULL) {
8653 ret = resolveInterrupt(this, source);
8654 if (ret != kIOReturnSuccess) {
8655 return ret;
8656 }
8657 }
8658
8659 /* Make sure the local source number is valid */
8660 if ((source < 0) || (source >= _numInterruptSources)) {
8661 return kIOReturnNoInterrupt;
8662 }
8663
8664 /* Look up the contoller for the local source */
8665 *interruptController = _interruptSources[source].interruptController;
8666
8667 if (*interruptController == NULL) {
8668 if (!resolve) {
8669 return kIOReturnNoInterrupt;
8670 }
8671
8672 /* Try to resolve the interrupt */
8673 ret = resolveInterrupt(this, source);
8674 if (ret != kIOReturnSuccess) {
8675 return ret;
8676 }
8677
8678 *interruptController = _interruptSources[source].interruptController;
8679 }
8680
8681 return kIOReturnSuccess;
8682 }
8683
8684 IOReturn
registerInterrupt(int source,OSObject * target,IOInterruptAction handler,void * refCon)8685 IOService::registerInterrupt(int source, OSObject *target,
8686 IOInterruptAction handler,
8687 void *refCon)
8688 {
8689 IOInterruptController *interruptController;
8690 IOReturn ret;
8691
8692 ret = lookupInterrupt(source, true, &interruptController);
8693 if (ret != kIOReturnSuccess) {
8694 return ret;
8695 }
8696
8697 /* Register the source */
8698 return interruptController->registerInterrupt(this, source, target,
8699 (IOInterruptHandler)handler,
8700 refCon);
8701 }
8702
8703 static void
IOServiceInterruptActionToBlock(OSObject * target,void * refCon,IOService * nub,int source)8704 IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
8705 IOService * nub, int source )
8706 {
8707 ((IOInterruptActionBlock)(refCon))(nub, source);
8708 }
8709
8710 IOReturn
registerInterruptBlock(int source,OSObject * target,IOInterruptActionBlock handler)8711 IOService::registerInterruptBlock(int source, OSObject *target,
8712 IOInterruptActionBlock handler)
8713 {
8714 IOReturn ret;
8715 void * block;
8716
8717 block = Block_copy(handler);
8718 if (!block) {
8719 return kIOReturnNoMemory;
8720 }
8721
8722 ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
8723 if (kIOReturnSuccess != ret) {
8724 Block_release(block);
8725 return ret;
8726 }
8727
8728 reserved->interruptSourcesPrivate[source].vectorBlock = block;
8729
8730 return ret;
8731 }
8732
8733 IOReturn
unregisterInterrupt(int source)8734 IOService::unregisterInterrupt(int source)
8735 {
8736 IOReturn ret;
8737 IOInterruptController *interruptController;
8738 IOInterruptSourcePrivate *priv;
8739 void *block;
8740
8741 ret = lookupInterrupt(source, false, &interruptController);
8742 if (ret != kIOReturnSuccess) {
8743 return ret;
8744 }
8745
8746 /* Unregister the source */
8747 priv = &reserved->interruptSourcesPrivate[source];
8748 block = priv->vectorBlock;
8749 ret = interruptController->unregisterInterrupt(this, source);
8750 if ((kIOReturnSuccess == ret) && (block = priv->vectorBlock)) {
8751 priv->vectorBlock = NULL;
8752 Block_release(block);
8753 }
8754
8755 return ret;
8756 }
8757
8758 void
unregisterAllInterrupts(void)8759 IOService::unregisterAllInterrupts(void)
8760 {
8761 for (int source = 0; source < _numInterruptSources; source++) {
8762 (void) unregisterInterrupt(source);
8763 }
8764 }
8765
8766 IOReturn
addInterruptStatistics(IOInterruptAccountingData * statistics,int source)8767 IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
8768 {
8769 IOReportLegend * legend = NULL;
8770 IOInterruptAccountingData * oldValue = NULL;
8771 IOInterruptAccountingReporter * newArray = NULL;
8772 char subgroupName[64];
8773 int newArraySize = 0;
8774 int i = 0;
8775
8776 if (source < 0) {
8777 return kIOReturnBadArgument;
8778 }
8779
8780 /*
8781 * We support statistics on a maximum of 256 interrupts per nub; if a nub
8782 * has more than 256 interrupt specifiers associated with it, and tries
8783 * to register a high interrupt index with interrupt accounting, panic.
8784 * Having more than 256 interrupts associated with a single nub is
8785 * probably a sign that something fishy is going on.
8786 */
8787 if (source > IA_INDEX_MAX) {
8788 panic("addInterruptStatistics called for an excessively large index (%d)", source);
8789 }
8790
8791 /*
8792 * TODO: This is ugly (wrapping a lock around an allocation). I'm only
8793 * leaving it as is because the likelihood of contention where we are
8794 * actually growing the array is minimal (we would realistically need
8795 * to be starting a driver for the first time, with an IOReporting
8796 * client already in place). Nonetheless, cleanup that can be done
8797 * to adhere to best practices; it'll make the code more complicated,
8798 * unfortunately.
8799 */
8800 IOLockLock(&reserved->interruptStatisticsLock);
8801
8802 /*
8803 * Lazily allocate the statistics array.
8804 */
8805 if (!reserved->interruptStatisticsArray) {
8806 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
8807 assert(reserved->interruptStatisticsArray);
8808 reserved->interruptStatisticsArrayCount = 1;
8809 bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
8810 }
8811
8812 if (source >= reserved->interruptStatisticsArrayCount) {
8813 /*
8814 * We're still within the range of supported indices, but we are out
8815 * of space in the current array. Do a nasty realloc (because
8816 * IORealloc isn't a thing) here. We'll double the size with each
8817 * reallocation.
8818 *
8819 * Yes, the "next power of 2" could be more efficient; but this will
8820 * be invoked incredibly rarely. Who cares.
8821 */
8822 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
8823
8824 while (newArraySize <= source) {
8825 newArraySize = (newArraySize << 1);
8826 }
8827 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
8828
8829 assert(newArray);
8830
8831 /*
8832 * TODO: This even zeroes the memory it is about to overwrite.
8833 * Shameful; fix it. Not particularly high impact, however.
8834 */
8835 bzero(newArray, newArraySize * sizeof(*newArray));
8836 memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
8837 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
8838 reserved->interruptStatisticsArray = newArray;
8839 reserved->interruptStatisticsArrayCount = newArraySize;
8840 }
8841
8842 if (!reserved->interruptStatisticsArray[source].reporter) {
8843 /*
8844 * We don't have a reporter associated with this index yet, so we
8845 * need to create one.
8846 */
8847 /*
8848 * TODO: Some statistics do in fact have common units (time); should this be
8849 * split into separate reporters to communicate this?
8850 */
8851 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
8852
8853 /*
8854 * Each statistic is given an identifier based on the interrupt index (which
8855 * should be unique relative to any single nub) and the statistic involved.
8856 * We should now have a sane (small and positive) index, so start
8857 * constructing the channels for statistics.
8858 */
8859 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
8860 /*
8861 * TODO: Currently, this does not add channels for disabled statistics.
8862 * Will this be confusing for clients? If so, we should just add the
8863 * channels; we can avoid updating the channels even if they exist.
8864 */
8865 if (IA_GET_STATISTIC_ENABLED(i)) {
8866 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
8867 }
8868 }
8869
8870 /*
8871 * We now need to add the legend for this reporter to the registry.
8872 */
8873 OSObject * prop = copyProperty(kIOReportLegendKey);
8874 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
8875 OSSafeReleaseNULL(prop);
8876
8877 /*
8878 * Note that while we compose the subgroup name, we do not need to
8879 * manage its lifecycle (the reporter will handle this).
8880 */
8881 snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
8882 subgroupName[sizeof(subgroupName) - 1] = 0;
8883 legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
8884 setProperty(kIOReportLegendKey, legend->getLegend());
8885 legend->release();
8886
8887 /*
8888 * TODO: Is this a good idea? Probably not; my assumption is it opts
8889 * all entities who register interrupts into public disclosure of all
8890 * IOReporting channels. Unfortunately, this appears to be as fine
8891 * grain as it gets.
8892 */
8893 setProperty(kIOReportLegendPublicKey, true);
8894 }
8895
8896 /*
8897 * Don't stomp existing entries. If we are about to, panic; this
8898 * probably means we failed to tear down our old interrupt source
8899 * correctly.
8900 */
8901 oldValue = reserved->interruptStatisticsArray[source].statistics;
8902
8903 if (oldValue) {
8904 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
8905 }
8906
8907 reserved->interruptStatisticsArray[source].statistics = statistics;
8908
8909 /*
8910 * Inherit the reporter values for each statistic. The target may
8911 * be torn down as part of the runtime of the service (especially
8912 * for sleep/wake), so we inherit in order to avoid having values
8913 * reset for no apparent reason. Our statistics are ultimately
8914 * tied to the index and the sevice, not to an individual target,
8915 * so we should maintain them accordingly.
8916 */
8917 interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8918
8919 IOLockUnlock(&reserved->interruptStatisticsLock);
8920
8921 return kIOReturnSuccess;
8922 }
8923
8924 IOReturn
removeInterruptStatistics(int source)8925 IOService::removeInterruptStatistics(int source)
8926 {
8927 IOInterruptAccountingData * value = NULL;
8928
8929 if (source < 0) {
8930 return kIOReturnBadArgument;
8931 }
8932
8933 IOLockLock(&reserved->interruptStatisticsLock);
8934
8935 /*
8936 * We dynamically grow the statistics array, so an excessively
8937 * large index value has NEVER been registered. This either
8938 * means our cap on the array size is too small (unlikely), or
8939 * that we have been passed a corrupt index (this must be passed
8940 * the plain index into the interrupt specifier list).
8941 */
8942 if (source >= reserved->interruptStatisticsArrayCount) {
8943 panic("removeInterruptStatistics called for index %d, which was never registered", source);
8944 }
8945
8946 assert(reserved->interruptStatisticsArray);
8947
8948 /*
8949 * If there is no existing entry, we are most likely trying to
8950 * free an interrupt owner twice, or we have corrupted the
8951 * index value.
8952 */
8953 value = reserved->interruptStatisticsArray[source].statistics;
8954
8955 if (!value) {
8956 panic("removeInterruptStatistics called for empty index %d", source);
8957 }
8958
8959 /*
8960 * We update the statistics, so that any delta with the reporter
8961 * state is not lost.
8962 */
8963 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8964 reserved->interruptStatisticsArray[source].statistics = NULL;
8965 IOLockUnlock(&reserved->interruptStatisticsLock);
8966
8967 return kIOReturnSuccess;
8968 }
8969
8970 IOReturn
getInterruptType(int source,int * interruptType)8971 IOService::getInterruptType(int source, int *interruptType)
8972 {
8973 IOInterruptController *interruptController;
8974 IOReturn ret;
8975
8976 ret = lookupInterrupt(source, true, &interruptController);
8977 if (ret != kIOReturnSuccess) {
8978 return ret;
8979 }
8980
8981 /* Return the type */
8982 return interruptController->getInterruptType(this, source, interruptType);
8983 }
8984
8985 IOReturn
enableInterrupt(int source)8986 IOService::enableInterrupt(int source)
8987 {
8988 IOInterruptController *interruptController;
8989 IOReturn ret;
8990
8991 ret = lookupInterrupt(source, false, &interruptController);
8992 if (ret != kIOReturnSuccess) {
8993 return ret;
8994 }
8995
8996 /* Enable the source */
8997 return interruptController->enableInterrupt(this, source);
8998 }
8999
9000 IOReturn
disableInterrupt(int source)9001 IOService::disableInterrupt(int source)
9002 {
9003 IOInterruptController *interruptController;
9004 IOReturn ret;
9005
9006 ret = lookupInterrupt(source, false, &interruptController);
9007 if (ret != kIOReturnSuccess) {
9008 return ret;
9009 }
9010
9011 /* Disable the source */
9012 return interruptController->disableInterrupt(this, source);
9013 }
9014
9015 IOReturn
causeInterrupt(int source)9016 IOService::causeInterrupt(int source)
9017 {
9018 IOInterruptController *interruptController;
9019 IOReturn ret;
9020
9021 ret = lookupInterrupt(source, false, &interruptController);
9022 if (ret != kIOReturnSuccess) {
9023 return ret;
9024 }
9025
9026 /* Cause an interrupt for the source */
9027 return interruptController->causeInterrupt(this, source);
9028 }
9029
9030 IOReturn
configureReport(IOReportChannelList * channelList,IOReportConfigureAction action,void * result,void * destination)9031 IOService::configureReport(IOReportChannelList *channelList,
9032 IOReportConfigureAction action,
9033 void *result,
9034 void *destination)
9035 {
9036 unsigned cnt;
9037
9038 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9039 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9040 if (pwrMgt) {
9041 configurePowerStatesReport(action, result);
9042 } else {
9043 return kIOReturnUnsupported;
9044 }
9045 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9046 if (pwrMgt) {
9047 configureSimplePowerReport(action, result);
9048 } else {
9049 return kIOReturnUnsupported;
9050 }
9051 }
9052 }
9053
9054 IOLockLock(&reserved->interruptStatisticsLock);
9055
9056 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9057 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9058 if (reserved->interruptStatisticsArray[cnt].reporter) {
9059 /*
9060 * If the reporter is currently associated with the statistics
9061 * for an event source, we may need to update the reporter.
9062 */
9063 if (reserved->interruptStatisticsArray[cnt].statistics) {
9064 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9065 }
9066
9067 reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
9068 }
9069 }
9070
9071 IOLockUnlock(&reserved->interruptStatisticsLock);
9072
9073 if (hasUserServer()) {
9074 return _ConfigureReport(channelList, action, result, destination);
9075 } else {
9076 return kIOReturnSuccess;
9077 }
9078 }
9079
9080 IOReturn
updateReport(IOReportChannelList * channelList,IOReportUpdateAction action,void * result,void * destination)9081 IOService::updateReport(IOReportChannelList *channelList,
9082 IOReportUpdateAction action,
9083 void *result,
9084 void *destination)
9085 {
9086 unsigned cnt;
9087
9088 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9089 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9090 if (pwrMgt) {
9091 updatePowerStatesReport(action, result, destination);
9092 } else {
9093 return kIOReturnUnsupported;
9094 }
9095 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9096 if (pwrMgt) {
9097 updateSimplePowerReport(action, result, destination);
9098 } else {
9099 return kIOReturnUnsupported;
9100 }
9101 }
9102 }
9103
9104 IOLockLock(&reserved->interruptStatisticsLock);
9105
9106 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9107 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9108 if (reserved->interruptStatisticsArray[cnt].reporter) {
9109 /*
9110 * If the reporter is currently associated with the statistics
9111 * for an event source, we need to update the reporter.
9112 */
9113 if (reserved->interruptStatisticsArray[cnt].statistics) {
9114 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9115 }
9116
9117 reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
9118 }
9119 }
9120
9121 IOLockUnlock(&reserved->interruptStatisticsLock);
9122
9123
9124 if (hasUserServer()) {
9125 return _UpdateReport(channelList, action, result, destination);
9126 } else {
9127 return kIOReturnSuccess;
9128 }
9129 }
9130
9131 uint64_t
getAuthorizationID(void)9132 IOService::getAuthorizationID( void )
9133 {
9134 return reserved->authorizationID;
9135 }
9136
9137 IOReturn
setAuthorizationID(uint64_t authorizationID)9138 IOService::setAuthorizationID( uint64_t authorizationID )
9139 {
9140 OSObject * entitlement;
9141 IOReturn status;
9142
9143 entitlement = IOUserClient::copyClientEntitlement( current_task(), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
9144
9145 if (entitlement) {
9146 if (entitlement == kOSBooleanTrue) {
9147 reserved->authorizationID = authorizationID;
9148
9149 status = kIOReturnSuccess;
9150 } else {
9151 status = kIOReturnNotPrivileged;
9152 }
9153
9154 entitlement->release();
9155 } else {
9156 status = kIOReturnNotPrivileged;
9157 }
9158
9159 return status;
9160 }
9161
9162 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9163
9164 #if __LP64__
9165 OSMetaClassDefineReservedUsedX86(IOService, 0);
9166 OSMetaClassDefineReservedUsedX86(IOService, 1);
9167 OSMetaClassDefineReservedUnused(IOService, 2);
9168 OSMetaClassDefineReservedUnused(IOService, 3);
9169 OSMetaClassDefineReservedUnused(IOService, 4);
9170 OSMetaClassDefineReservedUnused(IOService, 5);
9171 OSMetaClassDefineReservedUnused(IOService, 6);
9172 OSMetaClassDefineReservedUnused(IOService, 7);
9173 #else
9174 OSMetaClassDefineReservedUsedX86(IOService, 0);
9175 OSMetaClassDefineReservedUsedX86(IOService, 1);
9176 OSMetaClassDefineReservedUsedX86(IOService, 2);
9177 OSMetaClassDefineReservedUsedX86(IOService, 3);
9178 OSMetaClassDefineReservedUsedX86(IOService, 4);
9179 OSMetaClassDefineReservedUsedX86(IOService, 5);
9180 OSMetaClassDefineReservedUsedX86(IOService, 6);
9181 OSMetaClassDefineReservedUsedX86(IOService, 7);
9182 #endif
9183 OSMetaClassDefineReservedUnused(IOService, 8);
9184 OSMetaClassDefineReservedUnused(IOService, 9);
9185 OSMetaClassDefineReservedUnused(IOService, 10);
9186 OSMetaClassDefineReservedUnused(IOService, 11);
9187 OSMetaClassDefineReservedUnused(IOService, 12);
9188 OSMetaClassDefineReservedUnused(IOService, 13);
9189 OSMetaClassDefineReservedUnused(IOService, 14);
9190 OSMetaClassDefineReservedUnused(IOService, 15);
9191 OSMetaClassDefineReservedUnused(IOService, 16);
9192 OSMetaClassDefineReservedUnused(IOService, 17);
9193 OSMetaClassDefineReservedUnused(IOService, 18);
9194 OSMetaClassDefineReservedUnused(IOService, 19);
9195 OSMetaClassDefineReservedUnused(IOService, 20);
9196 OSMetaClassDefineReservedUnused(IOService, 21);
9197 OSMetaClassDefineReservedUnused(IOService, 22);
9198 OSMetaClassDefineReservedUnused(IOService, 23);
9199 OSMetaClassDefineReservedUnused(IOService, 24);
9200 OSMetaClassDefineReservedUnused(IOService, 25);
9201 OSMetaClassDefineReservedUnused(IOService, 26);
9202 OSMetaClassDefineReservedUnused(IOService, 27);
9203 OSMetaClassDefineReservedUnused(IOService, 28);
9204 OSMetaClassDefineReservedUnused(IOService, 29);
9205 OSMetaClassDefineReservedUnused(IOService, 30);
9206 OSMetaClassDefineReservedUnused(IOService, 31);
9207 OSMetaClassDefineReservedUnused(IOService, 32);
9208 OSMetaClassDefineReservedUnused(IOService, 33);
9209 OSMetaClassDefineReservedUnused(IOService, 34);
9210 OSMetaClassDefineReservedUnused(IOService, 35);
9211 OSMetaClassDefineReservedUnused(IOService, 36);
9212 OSMetaClassDefineReservedUnused(IOService, 37);
9213 OSMetaClassDefineReservedUnused(IOService, 38);
9214 OSMetaClassDefineReservedUnused(IOService, 39);
9215 OSMetaClassDefineReservedUnused(IOService, 40);
9216 OSMetaClassDefineReservedUnused(IOService, 41);
9217 OSMetaClassDefineReservedUnused(IOService, 42);
9218 OSMetaClassDefineReservedUnused(IOService, 43);
9219 OSMetaClassDefineReservedUnused(IOService, 44);
9220 OSMetaClassDefineReservedUnused(IOService, 45);
9221 OSMetaClassDefineReservedUnused(IOService, 46);
9222 OSMetaClassDefineReservedUnused(IOService, 47);
9223