1 /*
2 * Copyright (c) 1998-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <IOKit/system.h>
30 #include <IOKit/IOService.h>
31 #include <libkern/OSDebug.h>
32 #include <libkern/c++/OSAllocation.h>
33 #include <libkern/c++/OSContainers.h>
34 #include <libkern/c++/OSKext.h>
35 #include <libkern/c++/OSUnserialize.h>
36 #include <libkern/c++/OSKext.h>
37 #include <libkern/c++/OSSharedPtr.h>
38 #include <libkern/Block.h>
39 #include <IOKit/IOCatalogue.h>
40 #include <IOKit/IOCommand.h>
41 #include <IOKit/IODeviceTreeSupport.h>
42 #include <IOKit/IODeviceMemory.h>
43 #include <IOKit/IOInterrupts.h>
44 #include <IOKit/IOInterruptController.h>
45 #include <IOKit/IOPlatformExpert.h>
46 #include <IOKit/IOMessage.h>
47 #include <IOKit/IOLib.h>
48 #include <IOKit/IOKitKeysPrivate.h>
49 #include <IOKit/IOBSD.h>
50 #include <IOKit/IOUserClient.h>
51 #include <IOKit/IOUserServer.h>
52 #include <IOKit/IOWorkLoop.h>
53 #include <IOKit/IOTimeStamp.h>
54 #include <IOKit/IOHibernatePrivate.h>
55 #include <IOKit/IOInterruptAccountingPrivate.h>
56 #include <IOKit/IOKernelReporters.h>
57 #include <IOKit/AppleKeyStoreInterface.h>
58 #include <IOKit/pwr_mgt/RootDomain.h>
59 #include <IOKit/IOCPU.h>
60 #include <Exclaves/Exclaves.h>
61 #include <kern/cs_blobs.h>
62 #include <mach/sync_policy.h>
63 #include <mach/thread_info.h>
64 #include <IOKit/assert.h>
65 #include <sys/errno.h>
66 #include <sys/kdebug.h>
67 #include <string.h>
68
69 #include <machine/pal_routines.h>
70
71 #define LOG kprintf
72 //#define LOG IOLog
73 #define MATCH_DEBUG 0
74 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
75
76 // disabled since lockForArbitration() can be held externally
77 #define DEBUG_NOTIFIER_LOCKED 0
78
79 enum{
80 kIOUserServerCheckInTimeoutSecs = 120ULL
81 };
82
83 #include "IOServicePrivate.h"
84 #include "IOKitKernelInternal.h"
85
86 // take lockForArbitration before LOCKNOTIFY
87
88 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
89
90 #define super IORegistryEntry
91
92 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
93
94 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
95 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
96
97 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
98
99 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
100
101 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
102
103 OSDefineMetaClassAndStructors(IOResources, IOService)
104 OSDefineMetaClassAndStructors(IOUserResources, IOService)
105 OSDefineMetaClassAndStructors(IOExclaveProxy, IOService)
106
107 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
108
109 OSDefineMetaClassAndStructors(_IOServiceStateNotification, IOService)
110
111 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
112
113 OSDefineMetaClassAndStructors(IOServiceCompatibility, IOService)
114
115 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
116
117 static IOPlatformExpert * gIOPlatform;
118 static class IOPMrootDomain * gIOPMRootDomain;
119 const IORegistryPlane * gIOServicePlane;
120 const IORegistryPlane * gIOPowerPlane;
121 const OSSymbol * gIODeviceMemoryKey;
122 const OSSymbol * gIOInterruptControllersKey;
123 const OSSymbol * gIOInterruptSpecifiersKey;
124
125 const OSSymbol * gIOResourcesKey;
126 const OSSymbol * gIOUserResourcesKey;
127 const OSSymbol * gIOResourceMatchKey;
128 const OSSymbol * gIOResourceMatchedKey;
129 const OSSymbol * gIOResourceIOKitKey;
130
131 const OSSymbol * gIOProviderClassKey;
132 const OSSymbol * gIONameMatchKey;
133 const OSSymbol * gIONameMatchedKey;
134 const OSSymbol * gIOPropertyMatchKey;
135 const OSSymbol * gIOPropertyExistsMatchKey;
136 const OSSymbol * gIOLocationMatchKey;
137 const OSSymbol * gIOParentMatchKey;
138 const OSSymbol * gIOPathMatchKey;
139 const OSSymbol * gIOMatchCategoryKey;
140 const OSSymbol * gIODefaultMatchCategoryKey;
141 const OSSymbol * gIOMatchedAtBootKey;
142 const OSSymbol * gIOMatchedServiceCountKey;
143 const OSSymbol * gIOMatchedPersonalityKey;
144 const OSSymbol * gIORematchPersonalityKey;
145 const OSSymbol * gIORematchCountKey;
146 const OSSymbol * gIODEXTMatchCountKey;
147 const OSSymbol * gIOSupportedPropertiesKey;
148 const OSSymbol * gIOUserServicePropertiesKey;
149 #if defined(XNU_TARGET_OS_OSX)
150 const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey;
151 #endif /* defined(XNU_TARGET_OS_OSX) */
152
153 const OSSymbol * gIOCompatibilityMatchKey;
154 const OSSymbol * gIOCompatibilityPropertiesKey;
155 const OSSymbol * gIOPathKey;
156
157 const OSSymbol * gIOMapperIDKey;
158 const OSSymbol * gIOUserClientClassKey;
159
160 const OSSymbol * gIOUserClassKey;
161 const OSSymbol * gIOUserClassesKey;
162 const OSSymbol * gIOUserServerClassKey;
163 const OSSymbol * gIOUserServerNameKey;
164 const OSSymbol * gIOUserServerTagKey;
165 const OSSymbol * gIOUserUserClientKey;
166 const OSSymbol * gIOUserServerOneProcessKey;
167 const OSSymbol * gIOUserServerPreserveUserspaceRebootKey;
168
169 const OSSymbol * gIOKitDebugKey;
170
171 const OSSymbol * gIOCommandPoolSizeKey;
172
173 const OSSymbol * gIOConsoleLockedKey;
174 const OSSymbol * gIOConsoleUsersKey;
175 const OSSymbol * gIOConsoleSessionUIDKey;
176 const OSSymbol * gIOConsoleSessionAuditIDKey;
177 const OSSymbol * gIOConsoleUsersSeedKey;
178 const OSSymbol * gIOConsoleSessionOnConsoleKey;
179 const OSSymbol * gIOConsoleSessionLoginDoneKey;
180 const OSSymbol * gIOConsoleSessionSecureInputPIDKey;
181 const OSSymbol * gIOConsoleSessionScreenLockedTimeKey;
182 const OSSymbol * gIOConsoleSessionScreenIsLockedKey;
183 clock_sec_t gIOConsoleLockTime;
184 static bool gIOConsoleLoggedIn;
185 #if HIBERNATION
186 static OSBoolean * gIOConsoleBooterLockState;
187 static uint32_t gIOScreenLockState;
188 #endif
189 static IORegistryEntry * gIOChosenEntry;
190
191 static int gIOResourceGenerationCount;
192
193 const OSSymbol * gIOServiceKey;
194 const OSSymbol * gIOPublishNotification;
195 const OSSymbol * gIOFirstPublishNotification;
196 const OSSymbol * gIOMatchedNotification;
197 const OSSymbol * gIOFirstMatchNotification;
198 const OSSymbol * gIOTerminatedNotification;
199 const OSSymbol * gIOWillTerminateNotification;
200
201 const OSSymbol * gIOUserClientEntitlementsKey;
202 const OSSymbol * gIOServiceDEXTEntitlementsKey;
203 const OSSymbol * gIODriverKitEntitlementKey;
204 const OSSymbol * gIODriverKitUserClientEntitlementsKey;
205 const OSSymbol * gIODriverKitUserClientEntitlementAllowAnyKey;
206 const OSSymbol * gIODriverKitRequiredEntitlementsKey;
207 const OSSymbol * gIODriverKitTestDriverEntitlementKey;
208 const OSSymbol * gIODriverKitUserClientEntitlementCommunicatesWithDriversKey;
209 const OSSymbol * gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey;
210 const OSSymbol * gIOMatchDeferKey;
211 const OSSymbol * gIOServiceMatchDeferredKey;
212 const OSSymbol * gIOServiceNotificationUserKey;
213
214 const OSSymbol * gIOExclaveAssignedKey;
215 const OSSymbol * gIOExclaveProxyKey;
216
217 const OSSymbol * gIOPrimaryDriverTerminateOptionsKey;
218 const OSSymbol * gIOMediaKey;
219 const OSSymbol * gIOBlockStorageDriverKey;
220 static const OSSymbol * gPhysicalInterconnectKey;
221 static const OSSymbol * gVirtualInterfaceKey;
222
223 const OSSymbol * gIOAllCPUInitializedKey;
224
225 const OSSymbol * gIOGeneralInterest;
226 const OSSymbol * gIOBusyInterest;
227 const OSSymbol * gIOAppPowerStateInterest;
228 const OSSymbol * gIOPriorityPowerStateInterest;
229 const OSSymbol * gIOConsoleSecurityInterest;
230
231 const OSSymbol * gIOBSDKey;
232 const OSSymbol * gIOBSDNameKey;
233 const OSSymbol * gIOBSDMajorKey;
234 const OSSymbol * gIOBSDMinorKey;
235 const OSSymbol * gIOBSDUnitKey;
236
237 const OSSymbol * gAKSGetKey;
238 #if defined(__i386__) || defined(__x86_64__)
239 const OSSymbol * gIOCreateEFIDevicePathSymbol;
240 #endif
241
242 static OSDictionary * gNotifications;
243 static IORecursiveLock * gNotificationLock;
244
245 static IOService * gIOResources;
246 static IOService * gIOUserResources;
247 static IOService * gIOServiceRoot;
248
249 static OSOrderedSet * gJobs;
250 static semaphore_port_t gJobsSemaphore;
251 static IOLock * gJobsLock;
252 static int gOutstandingJobs;
253 static int gNumConfigThreads;
254 static int gHighNumConfigThreads;
255 static int gMaxConfigThreads = kMaxConfigThreads;
256 static int gNumWaitingThreads;
257 static IOLock * gIOServiceBusyLock;
258 bool gCPUsRunning;
259 bool gIOKitWillTerminate;
260 bool gInUserspaceReboot;
261
262 #define kIOServiceRootMediaParentInvalid ((IOService *) -1UL)
263 #if NO_KEXTD
264 static bool gIOServiceHideIOMedia = false;
265 static IOService * gIOServiceRootMediaParent = NULL;
266 #else /* NO_KEXTD */
267 static bool gIOServiceHideIOMedia = true;
268 static IOService * gIOServiceRootMediaParent = kIOServiceRootMediaParentInvalid;
269 #endif /* !NO_KEXTD */
270
271 static thread_t gIOTerminateThread;
272 static thread_t gIOTerminateWorkerThread;
273 static UInt32 gIOTerminateWork;
274 static OSArray * gIOTerminatePhase2List;
275 static OSArray * gIOStopList;
276 static OSArray * gIOStopProviderList;
277 static OSArray * gIOFinalizeList;
278
279 #if !NO_KEXTD
280 static OSArray * gIOMatchDeferList;
281 #endif
282
283 static SInt32 gIOConsoleUsersSeed;
284 static OSData * gIOConsoleUsersSeedValue;
285
286 extern const OSSymbol * gIODTPHandleKey;
287
288 const OSSymbol * gIOPlatformFunctionHandlerSet;
289
290
291 static IOLock * gIOConsoleUsersLock;
292 static thread_call_t gIOConsoleLockCallout;
293 static IONotifier * gIOServiceNullNotifier;
294
295 static SECURITY_READ_ONLY_LATE(uint32_t) gIODextRelaunchMax;
296
297 #if DEVELOPMENT || DEBUG
298 uint64_t driverkit_checkin_timed_out = 0;
299 #endif
300
301 IORecursiveLock * gDriverKitLaunchLock;
302 OSSet * gDriverKitLaunches;
303 const OSSymbol * gIOAssociatedServicesKey;
304
305 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
306
307 #define LOCKREADNOTIFY() \
308 IORecursiveLockLock( gNotificationLock )
309 #define LOCKWRITENOTIFY() \
310 IORecursiveLockLock( gNotificationLock )
311 #define LOCKWRITE2READNOTIFY()
312 #define UNLOCKNOTIFY() \
313 IORecursiveLockUnlock( gNotificationLock )
314 #define SLEEPNOTIFY(event) \
315 IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
316 #define SLEEPNOTIFYTO(event, deadline) \
317 IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
318 #define WAKEUPNOTIFY(event) \
319 IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
320
321 #define randomDelay() \
322 int del = read_processor_clock(); \
323 del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff; \
324 IOSleep( del );
325
326 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
327
328 #define queue_element(entry, element, type, field) do { \
329 vm_address_t __ele = (vm_address_t) (entry); \
330 __ele -= -4 + ((size_t)(&((type) 4)->field)); \
331 (element) = (type) __ele; \
332 } while(0)
333
334 #define iterqueue(que, elt) \
335 for (queue_entry_t elt = queue_first(que); \
336 !queue_end(que, elt); \
337 elt = queue_next(elt))
338
339 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
340
341 struct IOInterruptAccountingReporter {
342 IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
343 IOInterruptAccountingData * statistics; /* The live statistics values, if any */
344 };
345
346 struct ArbitrationLockQueueElement {
347 queue_chain_t link;
348 IOThread thread;
349 IOService * service;
350 unsigned count;
351 bool required;
352 bool aborted;
353 };
354
355 static queue_head_t gArbitrationLockQueueActive;
356 static queue_head_t gArbitrationLockQueueWaiting;
357 static queue_head_t gArbitrationLockQueueFree;
358 static IOLock * gArbitrationLockQueueLock;
359
360 bool
isInactive(void) const361 IOService::isInactive( void ) const
362 {
363 return 0 != (kIOServiceInactiveState & getState());
364 }
365
366 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
367
368 // Only used by the intel implementation of
369 // IOService::requireMaxBusStall(UInt32 ns)
370 // IOService::requireMaxInterruptDelay(uint32_t ns)
371 struct CpuDelayEntry {
372 IOService * fService;
373 UInt32 fMaxDelay;
374 UInt32 fDelayType;
375 };
376
377 enum {
378 kCpuDelayBusStall,
379 #if defined(__x86_64__)
380 kCpuDelayInterrupt,
381 #endif /* defined(__x86_64__) */
382 kCpuNumDelayTypes
383 };
384
385 static OSData *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
386 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
387 static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes];
388 const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes];
389 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
390 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
391 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
392
393 static void
394 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
395 static IOReturn
396 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
397
398 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
399
400 IOCoreAnalyticsSendEventProc gIOCoreAnalyticsSendEventProc;
401
402 kern_return_t
IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)403 IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)
404 {
405 if (gIOCoreAnalyticsSendEventProc) {
406 return kIOReturnNotPermitted;
407 }
408 gIOCoreAnalyticsSendEventProc = proc;
409
410 return kIOReturnSuccess;
411 }
412
413 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
414
415
416 static IOMessage sSystemPower;
417 extern "C" bool restore_boot;
418
419 namespace IOServicePH
420 {
421 IONotifier * fRootNotifier;
422 OSArray * fUserServers;
423 OSArray * fUserServersWait;
424 OSArray * fMatchingWork;
425 OSArray * fMatchingDelayed;
426 IOService * fSystemPowerAckTo;
427 uint32_t fSystemPowerAckRef;
428 uint8_t fSystemOff;
429 uint8_t fUserServerOff;
430 uint8_t fWaitingUserServers;
431 thread_call_t fUserServerAckTimer;
432
433 void lock();
434 void unlock();
435
436 void init(IOPMrootDomain * root);
437
438 IOReturn systemPowerChange(
439 void * target,
440 void * refCon,
441 UInt32 messageType, IOService * service,
442 void * messageArgument, vm_size_t argSize);
443
444 bool matchingStart(IOService * service);
445 void matchingEnd(IOService * service);
446 void userServerAckTimerExpired(void *, void *);
447 };
448
449 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
450
451 void
initialize(void)452 IOService::initialize( void )
453 {
454 kern_return_t err;
455
456 gIOServicePlane = IORegistryEntry::makePlane( kIOServicePlane );
457 gIOPowerPlane = IORegistryEntry::makePlane( kIOPowerPlane );
458
459 gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
460 gIONameMatchKey = OSSymbol::withCStringNoCopy( kIONameMatchKey );
461 gIONameMatchedKey = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
462 gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
463 gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
464 gIOPathMatchKey = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
465 gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
466 gIOParentMatchKey = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
467
468 gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
469 gIODefaultMatchCategoryKey = OSSymbol::withCStringNoCopy(
470 kIODefaultMatchCategoryKey );
471 gIOMatchedAtBootKey = OSSymbol::withCStringNoCopy(
472 kIOMatchedAtBootKey );
473
474 gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy(
475 kIOMatchedServiceCountKey );
476 gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
477 kIOMatchedPersonalityKey );
478 gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
479 kIORematchPersonalityKey );
480 gIORematchCountKey = OSSymbol::withCStringNoCopy(
481 kIORematchCountKey );
482 gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
483 kIODEXTMatchCountKey );
484
485 #if defined(XNU_TARGET_OS_OSX)
486 gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
487 kIOServiceLegacyMatchingRegistryIDKey );
488 #endif /* defined(XNU_TARGET_OS_OSX) */
489
490 if (!PE_parse_boot_argn("dextrelaunch", &gIODextRelaunchMax, sizeof(gIODextRelaunchMax))) {
491 if (restore_boot) {
492 // Limit dext relaunches in the restore environment
493 gIODextRelaunchMax = 10;
494 } else {
495 gIODextRelaunchMax = 1000;
496 }
497 }
498 PE_parse_boot_argn("iocthreads", &gMaxConfigThreads, sizeof(gMaxConfigThreads));
499
500 gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
501
502 gIOUserClassKey = OSSymbol::withCStringNoCopy(kIOUserClassKey);
503 gIOUserClassesKey = OSSymbol::withCStringNoCopy(kIOUserClassesKey);
504
505 gIOUserServerClassKey = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
506 gIOUserServerNameKey = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
507 gIOUserServerTagKey = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
508 gIOUserUserClientKey = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
509
510 gIOUserServerOneProcessKey = OSSymbol::withCStringNoCopy(kIOUserServerOneProcessKey);
511
512 gIOUserServerPreserveUserspaceRebootKey = OSSymbol::withCStringNoCopy(kIOUserServerPreserveUserspaceRebootKey);
513
514 gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass );
515 gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
516 gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
517 gIOResourceIOKitKey = OSSymbol::withCStringNoCopy("IOKit");
518
519 gIODeviceMemoryKey = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
520 gIOInterruptControllersKey
521 = OSSymbol::withCStringNoCopy("IOInterruptControllers");
522 gIOInterruptSpecifiersKey
523 = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
524
525 gIOCompatibilityMatchKey = OSSymbol::withCStringNoCopy(kIOCompatibilityMatchKey);
526 gIOCompatibilityPropertiesKey = OSSymbol::withCStringNoCopy(kIOCompatibilityPropertiesKey);
527 gIOPathKey = OSSymbol::withCStringNoCopy(kIOPathKey);
528 gIOSupportedPropertiesKey = OSSymbol::withCStringNoCopy(kIOSupportedPropertiesKey);
529 gIOUserServicePropertiesKey = OSSymbol::withCStringNoCopy(kIOUserServicePropertiesKey);
530
531 gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
532
533 gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
534
535 gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
536
537 gIOGeneralInterest = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
538 gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest );
539 gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
540 gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
541 gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
542
543 gIOBSDKey = OSSymbol::withCStringNoCopy(kIOBSDKey);
544 gIOBSDNameKey = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
545 gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
546 gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
547 gIOBSDUnitKey = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
548
549 gNotifications = OSDictionary::withCapacity( 1 );
550 gIOPublishNotification = OSSymbol::withCStringNoCopy(
551 kIOPublishNotification );
552 gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
553 kIOFirstPublishNotification );
554 gIOMatchedNotification = OSSymbol::withCStringNoCopy(
555 kIOMatchedNotification );
556 gIOFirstMatchNotification = OSSymbol::withCStringNoCopy(
557 kIOFirstMatchNotification );
558 gIOTerminatedNotification = OSSymbol::withCStringNoCopy(
559 kIOTerminatedNotification );
560 gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
561 kIOWillTerminateNotification );
562 gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass);
563
564
565 gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
566 gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
567 gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
568 gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
569
570 gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
571 gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
572 gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
573 gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
574 gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
575 gIOConsoleSessionScreenIsLockedKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
576
577 gIOConsoleUsersSeedValue = OSData::withValueNoCopy(gIOConsoleUsersSeed);
578
579 gIOUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIOUserClientEntitlementsKey );
580 gIOServiceDEXTEntitlementsKey = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
581 gIODriverKitEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
582 gIODriverKitUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
583 #if XNU_TARGET_OS_OSX
584 gIODriverKitUserClientEntitlementAllowAnyKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementAllowAnyKey );
585 #else
586 gIODriverKitUserClientEntitlementAllowAnyKey = NULL;
587 #endif
588 gIODriverKitRequiredEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitRequiredEntitlementsKey );
589 gIODriverKitTestDriverEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitTestDriverEntitlementKey );
590 gIODriverKitUserClientEntitlementCommunicatesWithDriversKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementCommunicatesWithDriversKey);
591 gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey);
592
593 gIOMatchDeferKey = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
594 gIOServiceMatchDeferredKey = OSSymbol::withCStringNoCopy( kIOServiceMatchDeferredKey );
595 gIOServiceNotificationUserKey = OSSymbol::withCStringNoCopy( kIOServiceNotificationUserKey );
596 gIOExclaveAssignedKey = OSSymbol::withCStringNoCopy(kIOExclaveAssignedKey);
597 gIOExclaveProxyKey = OSSymbol::withCStringNoCopy(kIOExclaveProxyKey);
598
599 gIOPrimaryDriverTerminateOptionsKey = OSSymbol::withCStringNoCopy(kIOPrimaryDriverTerminateOptionsKey);
600 gIOMediaKey = OSSymbol::withCStringNoCopy("IOMedia");
601 gIOBlockStorageDriverKey = OSSymbol::withCStringNoCopy("IOBlockStorageDriver");
602 gPhysicalInterconnectKey = OSSymbol::withCStringNoCopy("Physical Interconnect");
603 gVirtualInterfaceKey = OSSymbol::withCStringNoCopy("Virtual Interface");
604
605 gIOAllCPUInitializedKey = OSSymbol::withCStringNoCopy( kIOAllCPUInitializedKey );
606
607 gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
608 sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
609 #if defined(__x86_64__)
610 sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
611 #endif /* defined(__x86_64__) */
612 uint32_t idx;
613 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
614 sCPULatencySet[idx] = OSNumber::withNumber(UINT_MAX, 32);
615 sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
616 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
617 }
618
619 #if defined(__x86_64__)
620 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
621 #endif /* defined(__x86_64__) */
622
623 gNotificationLock = IORecursiveLockAlloc();
624
625 gAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
626
627 #if CONFIG_EXCLAVES
628 gExclaveProxyStates = OSDictionary::withCapacity( 1 );
629
630 gExclaveProxyStateLock = IORecursiveLockAlloc();
631 assert( gExclaveProxyStates && gExclaveProxyStateLock);
632 #endif /* CONFIG_EXCLAVES */
633
634 assert( gIOServicePlane && gIODeviceMemoryKey
635 && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
636 && gIOResourcesKey && gNotifications && gNotificationLock
637 && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
638 && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
639 && gIOPublishNotification && gIOMatchedNotification
640 && gIOTerminatedNotification && gIOServiceKey
641 && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
642 && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
643 && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
644
645 gJobsLock = IOLockAlloc();
646 gJobs = OSOrderedSet::withCapacity( 10 );
647
648 gIOServiceBusyLock = IOLockAlloc();
649
650 gIOConsoleUsersLock = IOLockAlloc();
651
652 err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
653
654 gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
655
656 IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
657
658 assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
659 && gIOConsoleLockCallout && (err == KERN_SUCCESS));
660
661 gIOResources = IOResources::resources();
662 gIOUserResources = IOUserResources::resources();
663 assert( gIOResources && gIOUserResources );
664
665 gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
666 assert(gIOServiceNullNotifier);
667
668 gArbitrationLockQueueLock = IOLockAlloc();
669 queue_init(&gArbitrationLockQueueActive);
670 queue_init(&gArbitrationLockQueueWaiting);
671 queue_init(&gArbitrationLockQueueFree);
672
673 assert( gArbitrationLockQueueLock );
674
675 allocPMInitLock();
676
677 gIOTerminatePhase2List = OSArray::withCapacity( 2 );
678 gIOStopList = OSArray::withCapacity( 16 );
679 gIOStopProviderList = OSArray::withCapacity( 16 );
680 gIOFinalizeList = OSArray::withCapacity( 16 );
681 #if !NO_KEXTD
682 if (OSKext::iokitDaemonAvailable()) {
683 gIOMatchDeferList = OSArray::withCapacity( 16 );
684 } else {
685 gIOMatchDeferList = NULL;
686 }
687 #endif
688 assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
689
690 gDriverKitLaunches = OSSet::withCapacity(0);
691 gDriverKitLaunchLock = IORecursiveLockAlloc();
692 gIOAssociatedServicesKey = OSSymbol::withCStringNoCopy( "IOAssociatedServices" );
693 #if CONFIG_EXCLAVES
694 gDARTMapperFunctionSetActive = OSSymbol::withCStringNoCopy("setActive");
695 #endif /* CONFIG_EXCLAVES */
696
697 // worker thread that is responsible for terminating / cleaning up threads
698 kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
699 assert(gIOTerminateWorkerThread);
700 thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
701 }
702
703 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
704
705 #if defined(__x86_64__)
706 extern "C" {
707 const char *getCpuDelayBusStallHolderName(void);
708 const char *
getCpuDelayBusStallHolderName(void)709 getCpuDelayBusStallHolderName(void)
710 {
711 return sCPULatencyHolderName[kCpuDelayBusStall];
712 }
713
714 const char *getCpuInterruptDelayHolderName(void);
715 const char *
getCpuInterruptDelayHolderName(void)716 getCpuInterruptDelayHolderName(void)
717 {
718 return sCPULatencyHolderName[kCpuDelayInterrupt];
719 }
720 }
721 #endif /* defined(__x86_64__) */
722
723
724
725 #if IOMATCHDEBUG
726 static UInt64
getDebugFlags(OSDictionary * props)727 getDebugFlags( OSDictionary * props )
728 {
729 OSNumber * debugProp;
730 UInt64 debugFlags;
731
732 debugProp = OSDynamicCast( OSNumber,
733 props->getObject( gIOKitDebugKey ));
734 if (debugProp) {
735 debugFlags = debugProp->unsigned64BitValue();
736 } else {
737 debugFlags = gIOKitDebug;
738 }
739
740 return debugFlags;
741 }
742
743 static UInt64
getDebugFlags(IOService * inst)744 getDebugFlags( IOService * inst )
745 {
746 OSObject * prop;
747 OSNumber * debugProp;
748 UInt64 debugFlags;
749
750 prop = inst->copyProperty(gIOKitDebugKey);
751 debugProp = OSDynamicCast(OSNumber, prop);
752 if (debugProp) {
753 debugFlags = debugProp->unsigned64BitValue();
754 } else {
755 debugFlags = gIOKitDebug;
756 }
757
758 OSSafeReleaseNULL(prop);
759
760 return debugFlags;
761 }
762 #endif
763
764 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
765
766 // Probe a matched service and return an instance to be started.
767 // The default score is from the property table, & may be altered
768 // during probe to change the start order.
769
770 IOService *
probe(IOService * provider,SInt32 * score)771 IOService::probe( IOService * provider,
772 SInt32 * score )
773 {
774 return this;
775 }
776
777 bool
start(IOService * provider)778 IOService::start( IOService * provider )
779 {
780 return true;
781 }
782
783 void
stop(IOService * provider)784 IOService::stop( IOService * provider )
785 {
786 if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
787 reserved->uvars->userServer->serviceStop(this, provider);
788 }
789 }
790
791 bool
init(OSDictionary * dictionary)792 IOService::init( OSDictionary * dictionary )
793 {
794 bool ret;
795
796 ret = super::init(dictionary);
797 if (!ret) {
798 return false;
799 }
800 if (reserved) {
801 return true;
802 }
803
804 reserved = IOMallocType(ExpansionData);
805 IOLockInlineInit(&reserved->interruptStatisticsLock);
806 return true;
807 }
808
809 bool
init(IORegistryEntry * from,const IORegistryPlane * inPlane)810 IOService::init( IORegistryEntry * from,
811 const IORegistryPlane * inPlane )
812 {
813 bool ret;
814
815 ret = super::init(from, inPlane);
816 if (!ret) {
817 return false;
818 }
819 if (reserved) {
820 return true;
821 }
822
823 reserved = IOMallocType(ExpansionData);
824 IOLockInlineInit(&reserved->interruptStatisticsLock);
825
826 return true;
827 }
828
829 void
free(void)830 IOService::free( void )
831 {
832 IOInterruptSourcePrivate *sourcesPrivate = NULL;
833 int i = 0;
834 requireMaxBusStall(0);
835 #if defined(__x86_64__)
836 requireMaxInterruptDelay(0);
837 #endif /* defined(__x86_64__) */
838 if (getPropertyTable()) {
839 unregisterAllInterest();
840 }
841 PMfree();
842
843 if (reserved) {
844 if (reserved->interruptStatisticsArray) {
845 for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
846 if (reserved->interruptStatisticsArray[i].reporter) {
847 reserved->interruptStatisticsArray[i].reporter->release();
848 }
849 }
850
851 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
852 }
853
854 if (reserved->uvars && reserved->uvars->userServer) {
855 reserved->uvars->userServer->serviceFree(this);
856 }
857 sourcesPrivate = reserved->interruptSourcesPrivate;
858 IOLockInlineDestroy(&reserved->interruptStatisticsLock);
859 IOFreeType(reserved, ExpansionData);
860 }
861
862 if (_numInterruptSources && _interruptSources) {
863 assert(sourcesPrivate);
864 for (i = 0; i < _numInterruptSources; i++) {
865 void * block = sourcesPrivate[i].vectorBlock;
866 if (block) {
867 Block_release(block);
868 }
869 }
870 IODelete(_interruptSources, IOInterruptSource, _numInterruptSources);
871 _interruptSources = NULL;
872 IODelete(sourcesPrivate, IOInterruptSourcePrivate, _numInterruptSources);
873 }
874
875 super::free();
876 }
877
878 /*
879 * Attach in service plane
880 */
881 bool
attach(IOService * provider)882 IOService::attach( IOService * provider )
883 {
884 bool ok;
885 uint32_t count;
886 AbsoluteTime deadline;
887 int waitResult = THREAD_AWAKENED;
888 bool wait, computeDeadline = true;
889
890 if (provider) {
891 if (gIOKitDebug & kIOLogAttach) {
892 LOG( "%s[0x%qx]::attach(%s[0x%qx])\n", getName(), getRegistryEntryID(),
893 provider->getName(), provider->getRegistryEntryID());
894 }
895
896 ok = false;
897 do{
898 wait = false;
899 provider->lockForArbitration();
900 if (provider->__state[0] & kIOServiceInactiveState) {
901 ok = false;
902 } else {
903 count = provider->getChildCount(gIOServicePlane);
904 wait = (count > (kIOServiceBusyMax - 4));
905 if (!wait) {
906 ok = attachToParent(provider, gIOServicePlane);
907 } else {
908 IOLog("stalling for detach from %s\n", provider->getName());
909 IOLockLock( gIOServiceBusyLock );
910 provider->__state[1] |= kIOServiceWaitDetachState;
911 }
912 }
913 provider->unlockForArbitration();
914 if (wait) {
915 if (computeDeadline) {
916 clock_interval_to_deadline(15, kSecondScale, &deadline);
917 computeDeadline = false;
918 }
919 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
920 IOLockUnlock( gIOServiceBusyLock );
921 waitResult = thread_block(THREAD_CONTINUE_NULL);
922 wait = (waitResult != THREAD_TIMED_OUT);
923 }
924 }while (wait);
925 } else {
926 gIOServiceRoot = this;
927 ok = attachToParent( getRegistryRoot(), gIOServicePlane);
928 }
929
930 if (ok && !__provider) {
931 (void) getProvider();
932 }
933
934 return ok;
935 }
936
937 IOService *
getServiceRoot(void)938 IOService::getServiceRoot( void )
939 {
940 return gIOServiceRoot;
941 }
942
943 void
detach(IOService * provider)944 IOService::detach( IOService * provider )
945 {
946 IOService * newProvider = NULL;
947 SInt32 busy;
948 bool adjParent;
949
950 if (gIOKitDebug & kIOLogAttach) {
951 LOG("%s[0x%qx]::detach(%s[0x%qx])\n", getName(), getRegistryEntryID(), provider->getName(), provider->getRegistryEntryID());
952 }
953
954 #if !NO_KEXTD
955 IOLockLock(gJobsLock);
956 if (gIOMatchDeferList) {
957 auto idx = gIOMatchDeferList->getNextIndexOfObject(this, 0);
958 if (-1U != idx) {
959 gIOMatchDeferList->removeObject(idx);
960 }
961 }
962 if (IOServicePH::fMatchingDelayed) {
963 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(this, 0);
964 if (-1U != idx) {
965 IOServicePH::fMatchingDelayed->removeObject(idx);
966 }
967 }
968 IOLockUnlock(gJobsLock);
969 #endif /* NO_KEXTD */
970
971 lockForArbitration();
972
973 uint64_t regID1 = provider->getRegistryEntryID();
974 uint64_t regID2 = getRegistryEntryID();
975 IOServiceTrace(
976 IOSERVICE_DETACH,
977 (uintptr_t) regID1,
978 (uintptr_t) (regID1 >> 32),
979 (uintptr_t) regID2,
980 (uintptr_t) (regID2 >> 32));
981
982 adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
983 && (provider == getProvider()));
984
985 detachFromParent( provider, gIOServicePlane );
986
987 if (busy) {
988 newProvider = getProvider();
989 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
990 _adjustBusy( -busy );
991 }
992 }
993
994 if (kIOServiceInactiveState & __state[0]) {
995 getMetaClass()->removeInstance(this);
996 IORemoveServicePlatformActions(this);
997 }
998
999 unlockForArbitration();
1000
1001 if (newProvider && adjParent) {
1002 newProvider->lockForArbitration();
1003 newProvider->_adjustBusy(1);
1004 newProvider->unlockForArbitration();
1005 }
1006
1007 // check for last client detach from a terminated service
1008 if (provider->lockForArbitration( true )) {
1009 if (kIOServiceStartState & __state[1]) {
1010 provider->scheduleTerminatePhase2();
1011 }
1012 if (adjParent) {
1013 provider->_adjustBusy( -1 );
1014 }
1015 if ((provider->__state[1] & kIOServiceTermPhase3State)
1016 && (NULL == provider->getClient())) {
1017 provider->scheduleFinalize(false);
1018 }
1019
1020 IOLockLock( gIOServiceBusyLock );
1021 if (kIOServiceWaitDetachState & provider->__state[1]) {
1022 provider->__state[1] &= ~kIOServiceWaitDetachState;
1023 thread_wakeup(&provider->__provider);
1024 }
1025 IOLockUnlock( gIOServiceBusyLock );
1026
1027 provider->unlockForArbitration();
1028 }
1029
1030 if (kIOServiceRematchOnDetach & __state[1]) {
1031 provider->registerService();
1032 }
1033 }
1034
1035 /*
1036 * Register instance - publish it for matching
1037 */
1038
1039 void
registerService(IOOptionBits options)1040 IOService::registerService( IOOptionBits options )
1041 {
1042 OSDataAllocation<char> pathBuf;
1043 const char * path;
1044 const char * skip;
1045 int len;
1046 enum { kMaxPathLen = 256 };
1047 enum { kMaxChars = 63 };
1048
1049 IORegistryEntry * parent = this;
1050 IORegistryEntry * root = getRegistryRoot();
1051 while (parent && (parent != root)) {
1052 parent = parent->getParentEntry( gIOServicePlane);
1053 }
1054
1055 if (parent != root) {
1056 IOLog("%s: not registry member at registerService()\n", getName());
1057 return;
1058 }
1059
1060 // Allow the Platform Expert to adjust this node.
1061 if (gIOPlatform && (!gIOPlatform->platformAdjustService(this))) {
1062 return;
1063 }
1064
1065 IOInstallServicePlatformActions(this);
1066 IOInstallServiceSleepPlatformActions(this);
1067
1068 if ((this != gIOResources)
1069 && (kIOLogRegister & gIOKitDebug)) {
1070 pathBuf = OSDataAllocation<char>( kMaxPathLen, OSAllocateMemory );
1071
1072 IOLog( "Registering: " );
1073
1074 len = kMaxPathLen;
1075 if (pathBuf && getPath( pathBuf.data(), &len, gIOServicePlane)) {
1076 path = pathBuf.data();
1077 if (len > kMaxChars) {
1078 IOLog("..");
1079 len -= kMaxChars;
1080 path += len;
1081 if ((skip = strchr( path, '/'))) {
1082 path = skip;
1083 }
1084 }
1085 } else {
1086 path = getName();
1087 }
1088
1089 IOLog( "%s\n", path );
1090 }
1091
1092 startMatching( options );
1093 }
1094
1095 void
startMatching(IOOptionBits options)1096 IOService::startMatching( IOOptionBits options )
1097 {
1098 IOService * provider;
1099 UInt32 prevBusy = 0;
1100 bool needConfig;
1101 bool needWake = false;
1102 bool sync;
1103 bool waitAgain;
1104 bool releaseAssertion = false;
1105
1106 if (options & kIOServiceDextRequirePowerForMatching) {
1107 bool ok = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
1108 if (!ok) {
1109 panic("%s: Failed to acquire power assertion for matching", getName());
1110 }
1111 releaseAssertion = true;
1112 }
1113
1114 lockForArbitration();
1115
1116 sync = (options & kIOServiceSynchronous)
1117 || ((provider = getProvider())
1118 && (provider->__state[1] & kIOServiceSynchronousState));
1119
1120 if (options & kIOServiceAsynchronous) {
1121 sync = false;
1122 }
1123
1124 needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1125 && (0 == (__state[0] & kIOServiceInactiveState));
1126
1127 __state[1] |= kIOServiceNeedConfigState;
1128
1129 // __state[0] &= ~kIOServiceInactiveState;
1130
1131 // if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1132 // OSKernelStackRemaining(), getName());
1133
1134 if (needConfig) {
1135 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1136 }
1137
1138 if (sync) {
1139 __state[1] |= kIOServiceSynchronousState;
1140 } else {
1141 __state[1] &= ~kIOServiceSynchronousState;
1142 }
1143
1144 if (needConfig) {
1145 prevBusy = _adjustBusy( 1 );
1146 }
1147
1148 unlockForArbitration();
1149
1150 if (needConfig) {
1151 if (needWake) {
1152 IOLockLock( gIOServiceBusyLock );
1153 thread_wakeup((event_t) this /*&__state[1]*/ );
1154 IOLockUnlock( gIOServiceBusyLock );
1155 } else if (!sync || (kIOServiceAsynchronous & options)) {
1156 // assertion will be released when matching job is complete
1157 releaseAssertion = false;
1158 _IOServiceJob::startJob( this, kMatchNubJob, options );
1159 } else {
1160 do {
1161 if ((__state[1] & kIOServiceNeedConfigState)) {
1162 doServiceMatch( options );
1163 }
1164
1165 lockForArbitration();
1166 IOLockLock( gIOServiceBusyLock );
1167
1168 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1169 && (0 == (__state[0] & kIOServiceInactiveState)));
1170
1171 if (waitAgain) {
1172 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1173 } else {
1174 __state[1] &= ~kIOServiceSyncPubState;
1175 }
1176
1177 unlockForArbitration();
1178
1179 if (waitAgain) {
1180 assert_wait((event_t) this /*&__state[1]*/, THREAD_UNINT);
1181 }
1182
1183 IOLockUnlock( gIOServiceBusyLock );
1184 if (waitAgain) {
1185 thread_block(THREAD_CONTINUE_NULL);
1186 }
1187 } while (waitAgain);
1188 }
1189 }
1190
1191 if (releaseAssertion) {
1192 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
1193 }
1194 }
1195
1196
1197 void
startDeferredMatches(void)1198 IOService::startDeferredMatches(void)
1199 {
1200 #if !NO_KEXTD
1201 OSArray * array;
1202
1203 IOLockLock(gJobsLock);
1204 array = gIOMatchDeferList;
1205 gIOMatchDeferList = NULL;
1206 IOLockUnlock(gJobsLock);
1207
1208 if (array) {
1209 IOLog("deferred rematching count %d\n", array->getCount());
1210 array->iterateObjects(^bool (OSObject * obj)
1211 {
1212 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
1213 return false;
1214 });
1215 array->release();
1216 }
1217 #endif /* !NO_KEXTD */
1218 }
1219
1220 void
iokitDaemonLaunched(void)1221 IOService::iokitDaemonLaunched(void)
1222 {
1223 #if !NO_KEXTD
1224 if (!OSKext::iokitDaemonAvailable()) {
1225 panic(kIOKitDaemonName " is unavailable in this environment, but it was launched");
1226 }
1227 IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1228 startDeferredMatches();
1229 getServiceRoot()->adjustBusy(-1);
1230 IOService::publishUserResource(gIOResourceIOKitKey);
1231 #endif /* !NO_KEXTD */
1232 }
1233
1234 /*
1235 * Possibly called with IORWLock from IOCatalog held.
1236 * This means that no calls to OSKext that could take
1237 * sKextLock can be performed from this function.
1238 */
1239 IOReturn
catalogNewDrivers(OSOrderedSet * newTables)1240 IOService::catalogNewDrivers( OSOrderedSet * newTables )
1241 {
1242 OSDictionary * table;
1243 OSSet * set;
1244 OSSet * allSet = NULL;
1245 IOService * service;
1246 #if IOMATCHDEBUG
1247 SInt32 count = 0;
1248 #endif
1249
1250 newTables->retain();
1251
1252 while ((table = (OSDictionary *) newTables->getFirstObject())) {
1253 LOCKWRITENOTIFY();
1254 set = (OSSet *) copyExistingServices( table,
1255 kIOServiceRegisteredState,
1256 kIOServiceExistingSet);
1257 UNLOCKNOTIFY();
1258 if (set) {
1259 #if IOMATCHDEBUG
1260 count += set->getCount();
1261 #endif
1262 if (allSet) {
1263 allSet->merge((const OSSet *) set);
1264 set->release();
1265 } else {
1266 allSet = set;
1267 }
1268 }
1269
1270 #if IOMATCHDEBUG
1271 if (getDebugFlags( table ) & kIOLogMatch) {
1272 LOG("Matching service count = %ld\n", (long)count);
1273 }
1274 #endif
1275 newTables->removeObject(table);
1276 }
1277
1278 if (allSet) {
1279 while ((service = (IOService *) allSet->getAnyObject())) {
1280 service->startMatching(kIOServiceAsynchronous);
1281 allSet->removeObject(service);
1282 }
1283 allSet->release();
1284 }
1285
1286 newTables->release();
1287
1288 return kIOReturnSuccess;
1289 }
1290
1291 _IOServiceJob *
startJob(IOService * nub,int type,IOOptionBits options)1292 _IOServiceJob::startJob( IOService * nub, int type,
1293 IOOptionBits options )
1294 {
1295 _IOServiceJob * job;
1296
1297 job = new _IOServiceJob;
1298 if (job && !job->init()) {
1299 job->release();
1300 job = NULL;
1301 }
1302
1303 if (job) {
1304 job->type = type;
1305 job->nub = nub;
1306 job->options = options;
1307 nub->retain(); // thread will release()
1308 pingConfig( job );
1309 }
1310
1311 return job;
1312 }
1313
1314 /*
1315 * Called on a registered service to see if it matches
1316 * a property table.
1317 */
1318
1319 bool
matchPropertyTable(OSDictionary * table,SInt32 * score)1320 IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1321 {
1322 return matchPropertyTable(table);
1323 }
1324
1325 bool
matchPropertyTable(OSDictionary * table)1326 IOService::matchPropertyTable( OSDictionary * table )
1327 {
1328 return true;
1329 }
1330
1331 /*
1332 * Called on a matched service to allocate resources
1333 * before first driver is attached.
1334 */
1335
1336 IOReturn
getResources(void)1337 IOService::getResources( void )
1338 {
1339 return kIOReturnSuccess;
1340 }
1341
1342 /*
1343 * Client/provider accessors
1344 */
1345
1346 IOService *
getProvider(void) const1347 IOService::getProvider( void ) const
1348 {
1349 IOService * self = (IOService *) this;
1350 IOService * parent;
1351 SInt32 generation;
1352
1353 generation = getRegistryEntryParentGenerationCount();
1354 if (__providerGeneration == generation) {
1355 return __provider;
1356 }
1357
1358 parent = (IOService *) getParentEntry( gIOServicePlane);
1359 if (parent == IORegistryEntry::getRegistryRoot()) {
1360 /* root is not an IOService */
1361 parent = NULL;
1362 }
1363
1364 self->__provider = parent;
1365 OSMemoryBarrier();
1366 // save the count from before call to getParentEntry()
1367 self->__providerGeneration = generation;
1368
1369 return parent;
1370 }
1371
1372 IOWorkLoop *
getWorkLoop() const1373 IOService::getWorkLoop() const
1374 {
1375 IOService *provider = getProvider();
1376
1377 if (provider) {
1378 return provider->getWorkLoop();
1379 } else {
1380 return NULL;
1381 }
1382 }
1383
1384 OSIterator *
getProviderIterator(void) const1385 IOService::getProviderIterator( void ) const
1386 {
1387 return getParentIterator( gIOServicePlane);
1388 }
1389
1390 IOService *
getClient(void) const1391 IOService::getClient( void ) const
1392 {
1393 return (IOService *) getChildEntry( gIOServicePlane);
1394 }
1395
1396 OSIterator *
getClientIterator(void) const1397 IOService::getClientIterator( void ) const
1398 {
1399 return getChildIterator( gIOServicePlane);
1400 }
1401
1402 OSIterator *
iterator(OSIterator * _iter,const IOService * client,const IOService * provider)1403 _IOOpenServiceIterator::iterator( OSIterator * _iter,
1404 const IOService * client,
1405 const IOService * provider )
1406 {
1407 _IOOpenServiceIterator * inst;
1408
1409 if (!_iter) {
1410 return NULL;
1411 }
1412
1413 inst = new _IOOpenServiceIterator;
1414
1415 if (inst && !inst->init()) {
1416 inst->release();
1417 inst = NULL;
1418 }
1419 if (inst) {
1420 inst->iter = _iter;
1421 inst->client = client;
1422 inst->provider = provider;
1423 } else {
1424 OSSafeReleaseNULL(_iter);
1425 }
1426
1427 return inst;
1428 }
1429
1430 void
free()1431 _IOOpenServiceIterator::free()
1432 {
1433 iter->release();
1434 if (last) {
1435 last->unlockForArbitration();
1436 }
1437 OSIterator::free();
1438 }
1439
1440 OSObject *
getNextObject()1441 _IOOpenServiceIterator::getNextObject()
1442 {
1443 IOService * next;
1444
1445 if (last) {
1446 last->unlockForArbitration();
1447 }
1448
1449 while ((next = (IOService *) iter->getNextObject())) {
1450 next->lockForArbitration();
1451 if ((client && (next->isOpen( client )))
1452 || (provider && (provider->isOpen( next )))) {
1453 break;
1454 }
1455 next->unlockForArbitration();
1456 }
1457
1458 last = next;
1459
1460 return next;
1461 }
1462
1463 bool
isValid()1464 _IOOpenServiceIterator::isValid()
1465 {
1466 return iter->isValid();
1467 }
1468
1469 void
reset()1470 _IOOpenServiceIterator::reset()
1471 {
1472 if (last) {
1473 last->unlockForArbitration();
1474 last = NULL;
1475 }
1476 iter->reset();
1477 }
1478
1479 OSIterator *
getOpenProviderIterator(void) const1480 IOService::getOpenProviderIterator( void ) const
1481 {
1482 return _IOOpenServiceIterator::iterator( getProviderIterator(), this, NULL );
1483 }
1484
1485 OSIterator *
getOpenClientIterator(void) const1486 IOService::getOpenClientIterator( void ) const
1487 {
1488 return _IOOpenServiceIterator::iterator( getClientIterator(), NULL, this );
1489 }
1490
1491
1492 IOReturn
callPlatformFunction(const OSSymbol * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1493 IOService::callPlatformFunction( const OSSymbol * functionName,
1494 bool waitForFunction,
1495 void *param1, void *param2,
1496 void *param3, void *param4 )
1497 {
1498 IOReturn result = kIOReturnUnsupported;
1499 IOService *provider;
1500
1501 if (functionName == gIOPlatformQuiesceActionKey ||
1502 functionName == gIOPlatformActiveActionKey ||
1503 functionName == gIOPlatformPanicActionKey) {
1504 /*
1505 * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction / IOPlatformPanicAction
1506 * must consume that event themselves, without passing it up to super/IOService.
1507 */
1508 if (gEnforcePlatformActionSafety) {
1509 panic("Class %s passed the %s action to IOService",
1510 getMetaClass()->getClassName(), functionName->getCStringNoCopy());
1511 }
1512 }
1513
1514 if (gIOPlatformFunctionHandlerSet == functionName) {
1515 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1516 IOService * target = (IOService *) param2;
1517 bool enable = (param3 != NULL);
1518
1519 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1520 result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1521 }
1522 #if defined(__x86_64__)
1523 else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1524 result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1525 }
1526 #endif /* defined(__x86_64__) */
1527 }
1528
1529 if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1530 result = provider->callPlatformFunction(functionName, waitForFunction,
1531 param1, param2, param3, param4);
1532 }
1533
1534 return result;
1535 }
1536
1537 IOReturn
callPlatformFunction(const char * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1538 IOService::callPlatformFunction( const char * functionName,
1539 bool waitForFunction,
1540 void *param1, void *param2,
1541 void *param3, void *param4 )
1542 {
1543 IOReturn result = kIOReturnNoMemory;
1544 const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1545
1546 if (functionSymbol != NULL) {
1547 result = callPlatformFunction(functionSymbol, waitForFunction,
1548 param1, param2, param3, param4);
1549 functionSymbol->release();
1550 }
1551
1552 return result;
1553 }
1554
1555
1556 /*
1557 * Accessors for global services
1558 */
1559
1560 IOPlatformExpert *
getPlatform(void)1561 IOService::getPlatform( void )
1562 {
1563 return gIOPlatform;
1564 }
1565
1566 class IOPMrootDomain *
getPMRootDomain(void)1567 IOService::getPMRootDomain( void )
1568 {
1569 return gIOPMRootDomain;
1570 }
1571
1572 IOService *
getResourceService(void)1573 IOService::getResourceService( void )
1574 {
1575 return gIOResources;
1576 }
1577
1578 IOService * gIOSystemStateNotificationService;
1579
1580 IOService *
getSystemStateNotificationService(void)1581 IOService::getSystemStateNotificationService(void)
1582 {
1583 return gIOSystemStateNotificationService;
1584 }
1585
1586 void
setPlatform(IOPlatformExpert * platform)1587 IOService::setPlatform( IOPlatformExpert * platform)
1588 {
1589 gIOPlatform = platform;
1590 gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1591
1592 gIOUserResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1593
1594 #if DEVELOPMENT || DEBUG
1595 // Test object that will be terminated for dext to match
1596 {
1597 IOService * ios;
1598 ios = OSTypeAlloc(IOService);
1599 ios->init();
1600 ios->attach(gIOUserResources);
1601 ios->setProperty(gIOMatchCategoryKey->getCStringNoCopy(), "com.apple.iokit.test");
1602 ios->setProperty(gIOModuleIdentifierKey->getCStringNoCopy(), "com.apple.kpi.iokit");
1603 ios->setProperty(gIOMatchedAtBootKey, kOSBooleanTrue);
1604 ios->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
1605 ios->release();
1606 }
1607 #endif
1608
1609 gIOSystemStateNotificationService = IOSystemStateNotification::initialize();
1610 gIOSystemStateNotificationService->attachToParent(platform, gIOServicePlane);
1611 gIOSystemStateNotificationService->registerService();
1612
1613 static const char * keys[kCpuNumDelayTypes] = {
1614 kIOPlatformMaxBusDelay,
1615 #if defined(__x86_64__)
1616 kIOPlatformMaxInterruptDelay
1617 #endif /* defined(__x86_64__) */
1618 };
1619 const OSObject * objs[2];
1620 OSArray * array;
1621 uint32_t idx;
1622
1623 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1624 objs[0] = sCPULatencySet[idx];
1625 objs[1] = sCPULatencyHolder[idx];
1626 array = OSArray::withObjects(objs, 2);
1627 if (!array) {
1628 break;
1629 }
1630 platform->setProperty(keys[idx], array);
1631 array->release();
1632 }
1633 }
1634
1635 void
setPMRootDomain(class IOPMrootDomain * rootDomain)1636 IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1637 {
1638 gIOPMRootDomain = rootDomain;
1639 }
1640
1641 void
publishPMRootDomain(void)1642 IOService::publishPMRootDomain( void )
1643 {
1644 assert(getPMRootDomain() != NULL);
1645 publishResource(gIOResourceIOKitKey);
1646 #if NO_KEXTD
1647 // Publish IOUserResources now since there is no IOKit daemon.
1648 publishUserResource(gIOResourceIOKitKey);
1649 #endif
1650 IOServicePH::init(getPMRootDomain());
1651 }
1652
1653 /*
1654 * Stacking change
1655 */
1656
1657 bool
lockForArbitration(bool isSuccessRequired)1658 IOService::lockForArbitration( bool isSuccessRequired )
1659 {
1660 bool found;
1661 bool success;
1662 ArbitrationLockQueueElement * element;
1663 ArbitrationLockQueueElement * owner;
1664 ArbitrationLockQueueElement * active;
1665 ArbitrationLockQueueElement * waiting;
1666
1667 enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1668
1669 // lock global access
1670 IOTakeLock( gArbitrationLockQueueLock );
1671
1672 // obtain an unused queue element
1673 if (!queue_empty( &gArbitrationLockQueueFree )) {
1674 queue_remove_first( &gArbitrationLockQueueFree,
1675 element,
1676 ArbitrationLockQueueElement *,
1677 link );
1678 } else {
1679 element = IOMallocType(ArbitrationLockQueueElement);
1680 assert( element );
1681 }
1682
1683 // prepare the queue element
1684 element->thread = IOThreadSelf();
1685 element->service = this;
1686 element->count = 1;
1687 element->required = isSuccessRequired;
1688 element->aborted = false;
1689
1690 // determine whether this object is already locked (ie. on active queue)
1691 found = false;
1692 queue_iterate( &gArbitrationLockQueueActive,
1693 owner,
1694 ArbitrationLockQueueElement *,
1695 link )
1696 {
1697 if (owner->service == element->service) {
1698 found = true;
1699 break;
1700 }
1701 }
1702
1703 if (found) { // this object is already locked
1704 active = owner;
1705 // determine whether it is the same or a different thread trying to lock
1706 if (active->thread != element->thread) { // it is a different thread
1707 ArbitrationLockQueueElement * victim = NULL;
1708
1709 // before placing this new thread on the waiting queue, we look for
1710 // a deadlock cycle...
1711
1712 while (1) {
1713 // determine whether the active thread holding the object we
1714 // want is waiting for another object to be unlocked
1715 found = false;
1716 queue_iterate( &gArbitrationLockQueueWaiting,
1717 waiting,
1718 ArbitrationLockQueueElement *,
1719 link )
1720 {
1721 if (waiting->thread == active->thread) {
1722 assert( false == waiting->aborted );
1723 found = true;
1724 break;
1725 }
1726 }
1727
1728 if (found) { // yes, active thread waiting for another object
1729 // this may be a candidate for rejection if the required
1730 // flag is not set, should we detect a deadlock later on
1731 if (false == waiting->required) {
1732 victim = waiting;
1733 }
1734
1735 // find the thread that is holding this other object, that
1736 // is blocking the active thread from proceeding (fun :-)
1737 found = false;
1738 queue_iterate( &gArbitrationLockQueueActive,
1739 active, // (reuse active queue element)
1740 ArbitrationLockQueueElement *,
1741 link )
1742 {
1743 if (active->service == waiting->service) {
1744 found = true;
1745 break;
1746 }
1747 }
1748
1749 // someone must be holding it or it wouldn't be waiting
1750 assert( found );
1751
1752 if (active->thread == element->thread) {
1753 // doh, it's waiting for the thread that originated
1754 // this whole lock (ie. current thread) -> deadlock
1755 if (false == element->required) { // willing to fail?
1756 // the originating thread doesn't have the required
1757 // flag, so it can fail
1758 success = false; // (fail originating lock request)
1759 break; // (out of while)
1760 } else { // originating thread is not willing to fail
1761 // see if we came across a waiting thread that did
1762 // not have the 'required' flag set: we'll fail it
1763 if (victim) {
1764 // we do have a willing victim, fail it's lock
1765 victim->aborted = true;
1766
1767 // take the victim off the waiting queue
1768 queue_remove( &gArbitrationLockQueueWaiting,
1769 victim,
1770 ArbitrationLockQueueElement *,
1771 link );
1772
1773 // wake the victim
1774 wakeup_thread_with_inheritor(&victim->service->__machPortHoldDestroy, // event
1775 THREAD_AWAKENED, LCK_WAKE_DEFAULT, victim->thread);
1776
1777 // allow this thread to proceed (ie. wait)
1778 success = true; // (put request on wait queue)
1779 break; // (out of while)
1780 } else {
1781 // all the waiting threads we came across in
1782 // finding this loop had the 'required' flag
1783 // set, so we've got a deadlock we can't avoid
1784 panic("I/O Kit: Unrecoverable deadlock.");
1785 }
1786 }
1787 } else {
1788 // repeat while loop, redefining active thread to be the
1789 // thread holding "this other object" (see above), and
1790 // looking for threads waiting on it; note the active
1791 // variable points to "this other object" already... so
1792 // there nothing to do in this else clause.
1793 }
1794 } else { // no, active thread is not waiting for another object
1795 success = true; // (put request on wait queue)
1796 break; // (out of while)
1797 }
1798 } // while forever
1799
1800 if (success) { // put the request on the waiting queue?
1801 kern_return_t wait_result;
1802
1803 // place this thread on the waiting queue and put it to sleep;
1804 // we place it at the tail of the queue...
1805 queue_enter( &gArbitrationLockQueueWaiting,
1806 element,
1807 ArbitrationLockQueueElement *,
1808 link );
1809
1810 // declare that this thread will wait for a given event
1811 restart_sleep:
1812 // unlock global access
1813 // & put thread to sleep, waiting for our event to fire...
1814 wait_result = IOLockSleepWithInheritor(gArbitrationLockQueueLock,
1815 LCK_SLEEP_UNLOCK,
1816 &element->service->__machPortHoldDestroy, // event
1817 owner->thread,
1818 element->required ? THREAD_UNINT : THREAD_INTERRUPTIBLE, TIMEOUT_WAIT_FOREVER);
1819
1820 // ...and we've been woken up; we might be in one of two states:
1821 // (a) we've been aborted and our queue element is not on
1822 // any of the three queues, but is floating around
1823 // (b) we're allowed to proceed with the lock and we have
1824 // already been moved from the waiting queue to the
1825 // active queue.
1826 // ...plus a 3rd state, should the thread have been interrupted:
1827 // (c) we're still on the waiting queue
1828
1829 // determine whether we were interrupted out of our sleep
1830 if (THREAD_INTERRUPTED == wait_result) {
1831 // re-lock global access
1832 IOTakeLock( gArbitrationLockQueueLock );
1833
1834 // determine whether we're still on the waiting queue
1835 found = false;
1836 queue_iterate( &gArbitrationLockQueueWaiting,
1837 waiting, // (reuse waiting queue element)
1838 ArbitrationLockQueueElement *,
1839 link )
1840 {
1841 if (waiting == element) {
1842 found = true;
1843 break;
1844 }
1845 }
1846
1847 if (found) { // yes, we're still on the waiting queue
1848 // determine whether we're willing to fail
1849 if (false == element->required) {
1850 // mark us as aborted
1851 element->aborted = true;
1852
1853 // take us off the waiting queue
1854 queue_remove( &gArbitrationLockQueueWaiting,
1855 element,
1856 ArbitrationLockQueueElement *,
1857 link );
1858 } else { // we are not willing to fail
1859 // ignore interruption, go back to sleep
1860 goto restart_sleep;
1861 }
1862 }
1863
1864 // unlock global access
1865 IOUnlock( gArbitrationLockQueueLock );
1866
1867 // proceed as though this were a normal wake up
1868 wait_result = THREAD_AWAKENED;
1869 }
1870
1871 assert( THREAD_AWAKENED == wait_result );
1872
1873 // determine whether we've been aborted while we were asleep
1874 if (element->aborted) {
1875 assert( false == element->required );
1876 // re-lock global access
1877 IOTakeLock( gArbitrationLockQueueLock );
1878
1879 action = kPutOnFreeQueue;
1880 success = false;
1881 } else { // we weren't aborted, so we must be ready to go :-)
1882 // we've already been moved from waiting to active queue
1883 return true;
1884 }
1885 } else { // the lock request is to be failed
1886 // return unused queue element to queue
1887 action = kPutOnFreeQueue;
1888 }
1889 } else { // it is the same thread, recursive access is allowed
1890 // add one level of recursion
1891 active->count++;
1892
1893 // return unused queue element to queue
1894 action = kPutOnFreeQueue;
1895 success = true;
1896 }
1897 } else { // this object is not already locked, so let this thread through
1898 action = kPutOnActiveQueue;
1899 success = true;
1900 }
1901
1902 // put the new element on a queue
1903 if (kPutOnActiveQueue == action) {
1904 queue_enter( &gArbitrationLockQueueActive,
1905 element,
1906 ArbitrationLockQueueElement *,
1907 link );
1908 } else if (kPutOnFreeQueue == action) {
1909 queue_enter( &gArbitrationLockQueueFree,
1910 element,
1911 ArbitrationLockQueueElement *,
1912 link );
1913 } else {
1914 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1915 }
1916
1917 // unlock global access
1918 IOUnlock( gArbitrationLockQueueLock );
1919
1920 return success;
1921 }
1922
1923 void
unlockForArbitration(void)1924 IOService::unlockForArbitration( void )
1925 {
1926 bool found;
1927 ArbitrationLockQueueElement * element;
1928
1929 // lock global access
1930 IOTakeLock( gArbitrationLockQueueLock );
1931
1932 // find the lock element for this object (ie. on active queue)
1933 found = false;
1934 queue_iterate( &gArbitrationLockQueueActive,
1935 element,
1936 ArbitrationLockQueueElement *,
1937 link )
1938 {
1939 if (element->service == this) {
1940 found = true;
1941 break;
1942 }
1943 }
1944
1945 assert( found );
1946
1947 // determine whether the lock has been taken recursively
1948 if (element->count > 1) {
1949 // undo one level of recursion
1950 element->count--;
1951 } else {
1952 // remove it from the active queue
1953 queue_remove( &gArbitrationLockQueueActive,
1954 element,
1955 ArbitrationLockQueueElement *,
1956 link );
1957
1958 // put it on the free queue
1959 queue_enter( &gArbitrationLockQueueFree,
1960 element,
1961 ArbitrationLockQueueElement *,
1962 link );
1963
1964 // determine whether a thread is waiting for object
1965 thread_t woken;
1966 kern_return_t kr =
1967 wakeup_one_with_inheritor(&__machPortHoldDestroy, // event
1968 THREAD_AWAKENED, LCK_WAKE_DEFAULT, &woken);
1969
1970 if (KERN_SUCCESS == kr) {
1971 found = false;
1972 queue_iterate( &gArbitrationLockQueueWaiting,
1973 element,
1974 ArbitrationLockQueueElement *,
1975 link )
1976 {
1977 if (element->thread == woken) {
1978 found = true;
1979 break;
1980 }
1981 }
1982 assert(found); // we found an interested thread on waiting queue
1983 // remove it from the waiting queue
1984 queue_remove( &gArbitrationLockQueueWaiting,
1985 element,
1986 ArbitrationLockQueueElement *,
1987 link );
1988
1989 // put it on the active queue
1990 queue_enter( &gArbitrationLockQueueActive,
1991 element,
1992 ArbitrationLockQueueElement *,
1993 link );
1994
1995 thread_deallocate(woken);
1996 }
1997 }
1998
1999 // unlock global access
2000 IOUnlock( gArbitrationLockQueueLock );
2001 }
2002
2003 uint32_t
isLockedForArbitration(IOService * service)2004 IOService::isLockedForArbitration(IOService * service)
2005 {
2006 #if DEBUG_NOTIFIER_LOCKED
2007 uint32_t count;
2008 ArbitrationLockQueueElement * active;
2009
2010 // lock global access
2011 IOLockLock(gArbitrationLockQueueLock);
2012
2013 // determine whether this object is already locked (ie. on active queue)
2014 count = 0;
2015 queue_iterate(&gArbitrationLockQueueActive,
2016 active,
2017 ArbitrationLockQueueElement *,
2018 link)
2019 {
2020 if ((active->thread == IOThreadSelf())
2021 && (!service || (active->service == service))) {
2022 count += 0x10000;
2023 count += active->count;
2024 }
2025 }
2026
2027 IOLockUnlock(gArbitrationLockQueueLock);
2028
2029 return count;
2030
2031 #else /* DEBUG_NOTIFIER_LOCKED */
2032
2033 return 0;
2034
2035 #endif /* DEBUG_NOTIFIER_LOCKED */
2036 }
2037
2038 void
setMachPortHoldDestroy(bool holdDestroy)2039 IOService::setMachPortHoldDestroy(bool holdDestroy)
2040 {
2041 __machPortHoldDestroy = holdDestroy;
2042 }
2043
2044 bool
machPortHoldDestroy()2045 IOService::machPortHoldDestroy()
2046 {
2047 return __machPortHoldDestroy;
2048 }
2049
2050 void
applyToProviders(IOServiceApplierFunction applier,void * context)2051 IOService::applyToProviders( IOServiceApplierFunction applier,
2052 void * context )
2053 {
2054 applyToParents((IORegistryEntryApplierFunction) applier,
2055 context, gIOServicePlane );
2056 }
2057
2058 void
applyToClients(IOServiceApplierFunction applier,void * context)2059 IOService::applyToClients( IOServiceApplierFunction applier,
2060 void * context )
2061 {
2062 applyToChildren((IORegistryEntryApplierFunction) applier,
2063 context, gIOServicePlane );
2064 }
2065
2066
2067 static void
IOServiceApplierToBlock(IOService * next,void * context)2068 IOServiceApplierToBlock(IOService * next, void * context)
2069 {
2070 IOServiceApplierBlock block = (IOServiceApplierBlock) context;
2071 block(next);
2072 }
2073
2074 void
applyToProviders(IOServiceApplierBlock applier)2075 IOService::applyToProviders(IOServiceApplierBlock applier)
2076 {
2077 applyToProviders(&IOServiceApplierToBlock, applier);
2078 }
2079
2080 void
applyToClients(IOServiceApplierBlock applier)2081 IOService::applyToClients(IOServiceApplierBlock applier)
2082 {
2083 applyToClients(&IOServiceApplierToBlock, applier);
2084 }
2085
2086 /*
2087 * Client messages
2088 */
2089
2090
2091 // send a message to a client or interested party of this service
2092 IOReturn
messageClient(UInt32 type,OSObject * client,void * argument,vm_size_t argSize)2093 IOService::messageClient( UInt32 type, OSObject * client,
2094 void * argument, vm_size_t argSize )
2095 {
2096 IOReturn ret;
2097 IOService * service;
2098 _IOServiceInterestNotifier * notify;
2099
2100 if ((service = OSDynamicCast( IOService, client))) {
2101 ret = service->message( type, this, argument );
2102 } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
2103 _IOServiceNotifierInvocation invocation;
2104 bool willNotify;
2105
2106 invocation.thread = current_thread();
2107
2108 LOCKWRITENOTIFY();
2109 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2110
2111 if (willNotify) {
2112 queue_enter( ¬ify->handlerInvocations, &invocation,
2113 _IOServiceNotifierInvocation *, link );
2114 }
2115 UNLOCKNOTIFY();
2116
2117 if (willNotify) {
2118 ret = (*notify->handler)( notify->target, notify->ref,
2119 type, this, argument, argSize );
2120
2121 LOCKWRITENOTIFY();
2122 queue_remove( ¬ify->handlerInvocations, &invocation,
2123 _IOServiceNotifierInvocation *, link );
2124 if (kIOServiceNotifyWaiter & notify->state) {
2125 notify->state &= ~kIOServiceNotifyWaiter;
2126 WAKEUPNOTIFY( notify );
2127 }
2128 UNLOCKNOTIFY();
2129 } else {
2130 ret = kIOReturnSuccess;
2131 }
2132 } else {
2133 ret = kIOReturnBadArgument;
2134 }
2135
2136 return ret;
2137 }
2138
2139 static void
applyToInterestNotifiers(const IORegistryEntry * target,const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2140 applyToInterestNotifiers(const IORegistryEntry *target,
2141 const OSSymbol * typeOfInterest,
2142 OSObjectApplierFunction applier,
2143 void * context )
2144 {
2145 OSArray * copyArray = NULL;
2146 OSObject * prop;
2147
2148 LOCKREADNOTIFY();
2149
2150 prop = target->copyProperty(typeOfInterest);
2151 IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
2152
2153 if (notifyList) {
2154 copyArray = OSArray::withCapacity(1);
2155
2156 // iterate over queue, entry is set to each element in the list
2157 iterqueue(¬ifyList->fCommandChain, entry) {
2158 _IOServiceInterestNotifier * notify;
2159
2160 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2161 copyArray->setObject(notify);
2162 }
2163 }
2164 UNLOCKNOTIFY();
2165
2166 if (copyArray) {
2167 unsigned int index;
2168 OSObject * next;
2169
2170 for (index = 0; (next = copyArray->getObject( index )); index++) {
2171 (*applier)(next, context);
2172 }
2173 copyArray->release();
2174 }
2175
2176 OSSafeReleaseNULL(prop);
2177 }
2178
2179 void
applyToInterested(const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2180 IOService::applyToInterested( const OSSymbol * typeOfInterest,
2181 OSObjectApplierFunction applier,
2182 void * context )
2183 {
2184 if (gIOGeneralInterest == typeOfInterest) {
2185 applyToClients((IOServiceApplierFunction) applier, context );
2186 }
2187 applyToInterestNotifiers(this, typeOfInterest, applier, context);
2188 }
2189
2190 struct MessageClientsContext {
2191 IOService * service;
2192 UInt32 type;
2193 void * argument;
2194 vm_size_t argSize;
2195 IOReturn ret;
2196 };
2197
2198 static void
messageClientsApplier(OSObject * object,void * ctx)2199 messageClientsApplier( OSObject * object, void * ctx )
2200 {
2201 IOReturn ret;
2202 MessageClientsContext * context = (MessageClientsContext *) ctx;
2203
2204 ret = context->service->messageClient( context->type,
2205 object, context->argument, context->argSize );
2206 if (kIOReturnSuccess != ret) {
2207 context->ret = ret;
2208 }
2209 }
2210
2211 // send a message to all clients
2212 IOReturn
messageClients(UInt32 type,void * argument,vm_size_t argSize)2213 IOService::messageClients( UInt32 type,
2214 void * argument, vm_size_t argSize )
2215 {
2216 MessageClientsContext context;
2217
2218 context.service = this;
2219 context.type = type;
2220 context.argument = argument;
2221 context.argSize = argSize;
2222 context.ret = kIOReturnSuccess;
2223
2224 applyToInterested( gIOGeneralInterest,
2225 &messageClientsApplier, &context );
2226
2227 return context.ret;
2228 }
2229
2230 IOReturn
acknowledgeNotification(IONotificationRef notification,IOOptionBits response)2231 IOService::acknowledgeNotification( IONotificationRef notification,
2232 IOOptionBits response )
2233 {
2234 return kIOReturnUnsupported;
2235 }
2236
2237 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2238 IOService::registerInterest( const OSSymbol * typeOfInterest,
2239 IOServiceInterestHandler handler, void * target, void * ref )
2240 {
2241 _IOServiceInterestNotifier * notify = NULL;
2242 IOReturn rc = kIOReturnError;
2243
2244 notify = new _IOServiceInterestNotifier;
2245 if (!notify) {
2246 return NULL;
2247 }
2248
2249 if (notify->init()) {
2250 rc = registerInterestForNotifier(notify, typeOfInterest,
2251 handler, target, ref);
2252 }
2253
2254 if (rc != kIOReturnSuccess) {
2255 notify->release();
2256 notify = NULL;
2257 }
2258
2259 return notify;
2260 }
2261
2262
2263
2264 static IOReturn
IOServiceInterestHandlerToBlock(void * target __unused,void * refCon,UInt32 messageType,IOService * provider,void * messageArgument,vm_size_t argSize)2265 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2266 UInt32 messageType, IOService * provider,
2267 void * messageArgument, vm_size_t argSize )
2268 {
2269 return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2270 }
2271
2272 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandlerBlock handler)2273 IOService::registerInterest(const OSSymbol * typeOfInterest,
2274 IOServiceInterestHandlerBlock handler)
2275 {
2276 IONotifier * notify;
2277 void * block;
2278
2279 block = Block_copy(handler);
2280 if (!block) {
2281 return NULL;
2282 }
2283
2284 notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
2285
2286 if (!notify) {
2287 Block_release(block);
2288 }
2289
2290 return notify;
2291 }
2292
2293 IOReturn
registerInterestForNotifier(IONotifier * svcNotify,const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2294 IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2295 IOServiceInterestHandler handler, void * target, void * ref )
2296 {
2297 IOReturn rc = kIOReturnSuccess;
2298 _IOServiceInterestNotifier *notify = NULL;
2299
2300
2301 if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)) || !handler) {
2302 return kIOReturnBadArgument;
2303 }
2304
2305 notify->handler = handler;
2306 notify->target = target;
2307 notify->ref = ref;
2308
2309 if ((typeOfInterest != gIOGeneralInterest)
2310 && (typeOfInterest != gIOBusyInterest)
2311 && (typeOfInterest != gIOAppPowerStateInterest)
2312 && (typeOfInterest != gIOConsoleSecurityInterest)
2313 && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2314 return kIOReturnBadArgument;
2315 }
2316
2317 lockForArbitration();
2318 if (0 == (__state[0] & kIOServiceInactiveState)) {
2319 notify->state = kIOServiceNotifyEnable;
2320
2321 ////// queue
2322
2323 LOCKWRITENOTIFY();
2324
2325 // Get the head of the notifier linked list
2326 IOCommand * notifyList;
2327 OSObject * obj = copyProperty( typeOfInterest );
2328 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2329 notifyList = OSTypeAlloc(IOCommand);
2330 if (notifyList) {
2331 notifyList->init();
2332 bool ok = setProperty( typeOfInterest, notifyList);
2333 notifyList->release();
2334 if (!ok) {
2335 notifyList = NULL;
2336 }
2337 }
2338 }
2339 if (obj) {
2340 obj->release();
2341 }
2342
2343 if (notifyList) {
2344 enqueue(¬ifyList->fCommandChain, ¬ify->chain);
2345 notify->retain(); // ref'ed while in list
2346 }
2347
2348 UNLOCKNOTIFY();
2349 } else {
2350 rc = kIOReturnNotReady;
2351 }
2352 unlockForArbitration();
2353
2354 return rc;
2355 }
2356
2357 static void
cleanInterestList(OSObject * head)2358 cleanInterestList( OSObject * head )
2359 {
2360 IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2361 if (!notifyHead) {
2362 return;
2363 }
2364
2365 LOCKWRITENOTIFY();
2366 while (queue_entry_t entry = dequeue(¬ifyHead->fCommandChain)) {
2367 queue_next(entry) = queue_prev(entry) = NULL;
2368
2369 _IOServiceInterestNotifier * notify;
2370
2371 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2372 notify->release();
2373 }
2374 UNLOCKNOTIFY();
2375 }
2376
2377 void
unregisterAllInterest(void)2378 IOService::unregisterAllInterest( void )
2379 {
2380 OSObject * prop;
2381
2382 prop = copyProperty(gIOGeneralInterest);
2383 cleanInterestList(prop);
2384 OSSafeReleaseNULL(prop);
2385
2386 prop = copyProperty(gIOBusyInterest);
2387 cleanInterestList(prop);
2388 OSSafeReleaseNULL(prop);
2389
2390 prop = copyProperty(gIOAppPowerStateInterest);
2391 cleanInterestList(prop);
2392 OSSafeReleaseNULL(prop);
2393
2394 prop = copyProperty(gIOPriorityPowerStateInterest);
2395 cleanInterestList(prop);
2396 OSSafeReleaseNULL(prop);
2397
2398 prop = copyProperty(gIOConsoleSecurityInterest);
2399 cleanInterestList(prop);
2400 OSSafeReleaseNULL(prop);
2401 }
2402
2403 /*
2404 * _IOServiceInterestNotifier
2405 */
2406
2407 // wait for all threads, other than the current one,
2408 // to exit the handler
2409
2410 void
wait()2411 _IOServiceInterestNotifier::wait()
2412 {
2413 _IOServiceNotifierInvocation * next;
2414 bool doWait;
2415
2416 do {
2417 doWait = false;
2418 queue_iterate( &handlerInvocations, next,
2419 _IOServiceNotifierInvocation *, link) {
2420 if (next->thread != current_thread()) {
2421 doWait = true;
2422 break;
2423 }
2424 }
2425 if (doWait) {
2426 state |= kIOServiceNotifyWaiter;
2427 SLEEPNOTIFY(this);
2428 }
2429 } while (doWait);
2430 }
2431
2432 void
free()2433 _IOServiceInterestNotifier::free()
2434 {
2435 assert( queue_empty( &handlerInvocations ));
2436
2437 if (handler == &IOServiceInterestHandlerToBlock) {
2438 Block_release(ref);
2439 }
2440
2441 OSObject::free();
2442 }
2443
2444 void
remove()2445 _IOServiceInterestNotifier::remove()
2446 {
2447 LOCKWRITENOTIFY();
2448
2449 if (queue_next( &chain )) {
2450 remqueue(&chain);
2451 queue_next( &chain) = queue_prev( &chain) = NULL;
2452 release();
2453 }
2454
2455 state &= ~kIOServiceNotifyEnable;
2456
2457 wait();
2458
2459 UNLOCKNOTIFY();
2460
2461 release();
2462 }
2463
2464 bool
disable()2465 _IOServiceInterestNotifier::disable()
2466 {
2467 bool ret;
2468
2469 LOCKWRITENOTIFY();
2470
2471 ret = (0 != (kIOServiceNotifyEnable & state));
2472 state &= ~kIOServiceNotifyEnable;
2473 if (ret) {
2474 wait();
2475 }
2476
2477 UNLOCKNOTIFY();
2478
2479 return ret;
2480 }
2481
2482 void
enable(bool was)2483 _IOServiceInterestNotifier::enable( bool was )
2484 {
2485 LOCKWRITENOTIFY();
2486 if (was) {
2487 state |= kIOServiceNotifyEnable;
2488 } else {
2489 state &= ~kIOServiceNotifyEnable;
2490 }
2491 UNLOCKNOTIFY();
2492 }
2493
2494 bool
init()2495 _IOServiceInterestNotifier::init()
2496 {
2497 queue_init( &handlerInvocations );
2498 return OSObject::init();
2499 }
2500 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2501
2502 /*
2503 * Termination
2504 */
2505
2506 #define tailQ(o) setObject(o)
2507 #define headQ(o) setObject(0, o)
2508 #define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2509
2510 static void
_workLoopAction(IOWorkLoop::Action action,IOService * service,void * p0=NULL,void * p1=NULL,void * p2=NULL,void * p3=NULL)2511 _workLoopAction( IOWorkLoop::Action action,
2512 IOService * service,
2513 void * p0 = NULL, void * p1 = NULL,
2514 void * p2 = NULL, void * p3 = NULL )
2515 {
2516 IOWorkLoop * wl;
2517
2518 if ((wl = service->getWorkLoop())) {
2519 wl->retain();
2520 wl->runAction( action, service, p0, p1, p2, p3 );
2521 wl->release();
2522 } else {
2523 (*action)( service, p0, p1, p2, p3 );
2524 }
2525 }
2526
2527 bool
requestTerminate(IOService * provider,IOOptionBits options)2528 IOService::requestTerminate( IOService * provider, IOOptionBits options )
2529 {
2530 bool ok;
2531
2532 // if its our only provider
2533 ok = isParent( provider, gIOServicePlane, true);
2534
2535 // -- compat
2536 if (ok) {
2537 provider->terminateClient( this, options | kIOServiceRecursing );
2538 ok = (0 != (kIOServiceInactiveState & __state[0]));
2539 }
2540 // --
2541
2542 return ok;
2543 }
2544
2545 bool
terminatePhase1(IOOptionBits options)2546 IOService::terminatePhase1( IOOptionBits options )
2547 {
2548 IOService * victim;
2549 IOService * client;
2550 IOService * rematchProvider;
2551 OSIterator * iter;
2552 OSArray * makeInactive;
2553 OSArray * waitingInactive;
2554 IOOptionBits callerOptions;
2555 int waitResult = THREAD_AWAKENED;
2556 bool wait;
2557 bool ok;
2558 bool didInactive;
2559 bool startPhase2 = false;
2560
2561 TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2562
2563 callerOptions = options;
2564 rematchProvider = NULL;
2565 uint64_t regID = getRegistryEntryID();
2566 IOServiceTrace(
2567 IOSERVICE_TERMINATE_PHASE1,
2568 (uintptr_t) regID,
2569 (uintptr_t) (regID >> 32),
2570 (uintptr_t) this,
2571 (uintptr_t) options);
2572
2573 // -- compat
2574 if (options & kIOServiceRecursing) {
2575 lockForArbitration();
2576 if (0 == (kIOServiceInactiveState & __state[0])) {
2577 __state[0] |= kIOServiceInactiveState;
2578 __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2579 }
2580 unlockForArbitration();
2581
2582 return true;
2583 }
2584 // --
2585
2586 makeInactive = OSArray::withCapacity( 16 );
2587 waitingInactive = OSArray::withCapacity( 16 );
2588 if (!makeInactive || !waitingInactive) {
2589 OSSafeReleaseNULL(makeInactive);
2590 OSSafeReleaseNULL(waitingInactive);
2591 return false;
2592 }
2593
2594 victim = this;
2595 victim->retain();
2596
2597 while (victim) {
2598 didInactive = victim->lockForArbitration( true );
2599 if (didInactive) {
2600 uint64_t regID1 = victim->getRegistryEntryID();
2601 IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2602 (uintptr_t) regID1,
2603 (uintptr_t) (regID1 >> 32),
2604 (uintptr_t) victim->__state[1],
2605 (uintptr_t) 0);
2606
2607 enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2608 didInactive = (kRP1 == (victim->__state[1] & kRP1))
2609 || (0 == (victim->__state[0] & kIOServiceInactiveState));
2610
2611 if (!didInactive) {
2612 // a multiply attached IOService can be visited twice
2613 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) {
2614 do{
2615 IOLockLock(gIOServiceBusyLock);
2616 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2617 if (wait) {
2618 TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2619 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2620 victim->__state[1] |= kIOServiceTerm1WaiterState;
2621 victim->unlockForArbitration();
2622 assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2623 }
2624 IOLockUnlock(gIOServiceBusyLock);
2625 if (wait) {
2626 waitResult = thread_block(THREAD_CONTINUE_NULL);
2627 TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2628 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2629 victim->lockForArbitration();
2630 }
2631 }while (wait && (waitResult != THREAD_TIMED_OUT));
2632 }
2633 } else {
2634 victim->__state[0] |= kIOServiceInactiveState;
2635 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2636 | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2637 victim->__state[1] &= ~kIOServiceRecursing;
2638 victim->__state[1] |= kIOServiceTermPhase1State;
2639 waitingInactive->headQ(victim);
2640 if (victim == this) {
2641 if (kIOServiceTerminateNeedWillTerminate & options) {
2642 victim->__state[1] |= kIOServiceNeedWillTerminate;
2643 }
2644 }
2645 victim->_adjustBusy( 1 );
2646
2647 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2648 if ((options & kIOServiceTerminateWithRematchCurrentDext)) {
2649 OSObject * obj;
2650 OSObject * rematchProps;
2651 OSNumber * num;
2652 uint32_t count;
2653
2654 rematchProvider = getProvider();
2655 if (rematchProvider) {
2656 obj = rematchProvider->copyProperty(gIORematchCountKey);
2657 num = OSDynamicCast(OSNumber, obj);
2658 count = 0;
2659 if (num) {
2660 count = num->unsigned32BitValue();
2661 count++;
2662 }
2663 num = OSNumber::withNumber(count, 32);
2664 rematchProvider->setProperty(gIORematchCountKey, num);
2665 rematchProps = copyProperty(gIOMatchedPersonalityKey);
2666 rematchProvider->setProperty(gIORematchPersonalityKey, rematchProps);
2667 OSSafeReleaseNULL(num);
2668 OSSafeReleaseNULL(rematchProps);
2669 OSSafeReleaseNULL(obj);
2670 }
2671 }
2672 victim->__state[1] |= kIOServiceRematchOnDetach;
2673 }
2674 }
2675 victim->unlockForArbitration();
2676 }
2677 if (victim == this) {
2678 options &= ~(kIOServiceTerminateWithRematch | kIOServiceTerminateWithRematchCurrentDext);
2679 startPhase2 = didInactive;
2680 }
2681 if (didInactive) {
2682 OSArray * notifiers;
2683 notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2684 victim->invokeNotifiers(¬ifiers);
2685
2686 IOUserClient::destroyUserReferences( victim );
2687
2688 iter = victim->getClientIterator();
2689 if (iter) {
2690 while ((client = (IOService *) iter->getNextObject())) {
2691 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2692 client->getName(), client->getRegistryEntryID(),
2693 victim->getName(), victim->getRegistryEntryID(), (long long)options);
2694 ok = client->requestTerminate( victim, options );
2695 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2696 client->getName(), client->getRegistryEntryID(),
2697 victim->getName(), victim->getRegistryEntryID(), ok);
2698
2699 uint64_t regID1 = client->getRegistryEntryID();
2700 uint64_t regID2 = victim->getRegistryEntryID();
2701 IOServiceTrace(
2702 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2703 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2704 (uintptr_t) regID1,
2705 (uintptr_t) (regID1 >> 32),
2706 (uintptr_t) regID2,
2707 (uintptr_t) (regID2 >> 32));
2708
2709 if (ok) {
2710 makeInactive->setObject( client );
2711 }
2712 }
2713 iter->release();
2714 }
2715 }
2716 victim->release();
2717 victim = (IOService *) makeInactive->getObject(0);
2718 if (victim) {
2719 victim->retain();
2720 makeInactive->removeObject(0);
2721 }
2722 }
2723
2724 makeInactive->release();
2725
2726 while ((victim = (IOService *) waitingInactive->getObject(0))) {
2727 victim->retain();
2728 waitingInactive->removeObject(0);
2729
2730 victim->lockForArbitration();
2731 victim->__state[1] &= ~kIOServiceTermPhase1State;
2732 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2733 victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2734 TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2735 IOLockLock( gIOServiceBusyLock );
2736 thread_wakeup((event_t) &victim->__state[1]);
2737 IOLockUnlock( gIOServiceBusyLock );
2738 }
2739 victim->unlockForArbitration();
2740 victim->release();
2741 }
2742
2743 waitingInactive->release();
2744
2745 if (startPhase2) {
2746 retain();
2747 lockForArbitration();
2748 scheduleTerminatePhase2(options);
2749 unlockForArbitration();
2750 release();
2751 }
2752
2753 if (rematchProvider) {
2754 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2755 }
2756
2757 return true;
2758 }
2759
2760 void
setTerminateDefer(IOService * provider,bool defer)2761 IOService::setTerminateDefer(IOService * provider, bool defer)
2762 {
2763 lockForArbitration();
2764 if (defer) {
2765 __state[1] |= kIOServiceStartState;
2766 } else {
2767 __state[1] &= ~kIOServiceStartState;
2768 }
2769 unlockForArbitration();
2770
2771 if (provider && !defer) {
2772 provider->lockForArbitration();
2773 provider->scheduleTerminatePhase2();
2774 provider->unlockForArbitration();
2775 }
2776 }
2777
2778 // Must call this while holding gJobsLock
2779 void
waitToBecomeTerminateThread(void)2780 IOService::waitToBecomeTerminateThread(void)
2781 {
2782 IOLockAssert(gJobsLock, kIOLockAssertOwned);
2783 bool wait;
2784 do {
2785 wait = (gIOTerminateThread != THREAD_NULL);
2786 if (wait) {
2787 IOLockSleepWithInheritor(
2788 gJobsLock,
2789 LCK_SLEEP_DEFAULT,
2790 &gIOTerminateThread,
2791 gIOTerminateThread,
2792 THREAD_UNINT,
2793 TIMEOUT_WAIT_FOREVER);
2794 }
2795 } while (wait);
2796 gIOTerminateThread = current_thread();
2797 }
2798
2799 // call with lockForArbitration
2800 void
scheduleTerminatePhase2(IOOptionBits options)2801 IOService::scheduleTerminatePhase2( IOOptionBits options )
2802 {
2803 AbsoluteTime deadline;
2804 uint64_t regID1;
2805 int waitResult = THREAD_AWAKENED;
2806 bool wait = false, haveDeadline = false;
2807
2808 if (!(__state[0] & kIOServiceInactiveState)) {
2809 return;
2810 }
2811
2812 regID1 = getRegistryEntryID();
2813 IOServiceTrace(
2814 IOSERVICE_TERM_SCHED_PHASE2,
2815 (uintptr_t) regID1,
2816 (uintptr_t) (regID1 >> 32),
2817 (uintptr_t) __state[1],
2818 (uintptr_t) options);
2819
2820 if (__state[1] & kIOServiceTermPhase1State) {
2821 return;
2822 }
2823
2824 retain();
2825 unlockForArbitration();
2826 options |= kIOServiceRequired;
2827 IOLockLock( gJobsLock );
2828
2829 if ((options & kIOServiceSynchronous)
2830 && (current_thread() != gIOTerminateThread)) {
2831 waitToBecomeTerminateThread();
2832 gIOTerminatePhase2List->setObject( this );
2833 gIOTerminateWork++;
2834
2835 do {
2836 while (gIOTerminateWork) {
2837 terminateWorker( options );
2838 }
2839 wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2840 if (wait) {
2841 /* wait for the victim to go non-busy */
2842 if (!haveDeadline) {
2843 clock_interval_to_deadline( 15, kSecondScale, &deadline );
2844 haveDeadline = true;
2845 }
2846 /* let others do work while we wait */
2847 gIOTerminateThread = NULL;
2848 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2849 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2850 deadline, THREAD_UNINT );
2851 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2852 IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2853 "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2854 }
2855 waitToBecomeTerminateThread();
2856 }
2857 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2858
2859 gIOTerminateThread = NULL;
2860 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2861 } else {
2862 // ! kIOServiceSynchronous
2863
2864 gIOTerminatePhase2List->setObject( this );
2865 if (0 == gIOTerminateWork++) {
2866 assert(gIOTerminateWorkerThread);
2867 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2868 }
2869 }
2870
2871 IOLockUnlock( gJobsLock );
2872 lockForArbitration();
2873 release();
2874 }
2875
2876 __attribute__((__noreturn__))
2877 void
terminateThread(void * arg,wait_result_t waitResult)2878 IOService::terminateThread( void * arg, wait_result_t waitResult )
2879 {
2880 // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2881 IOLockLock(gJobsLock);
2882 while (true) {
2883 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2884 waitToBecomeTerminateThread();
2885 }
2886
2887 while (gIOTerminateWork) {
2888 terminateWorker((IOOptionBits)(uintptr_t)arg );
2889 }
2890
2891 gIOTerminateThread = NULL;
2892 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2893 IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2894 }
2895 }
2896
2897 void
scheduleStop(IOService * provider)2898 IOService::scheduleStop( IOService * provider )
2899 {
2900 uint64_t regID1 = getRegistryEntryID();
2901 uint64_t regID2 = provider->getRegistryEntryID();
2902
2903 TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2904 IOServiceTrace(
2905 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2906 (uintptr_t) regID1,
2907 (uintptr_t) (regID1 >> 32),
2908 (uintptr_t) regID2,
2909 (uintptr_t) (regID2 >> 32));
2910
2911 IOLockLock( gJobsLock );
2912 gIOStopList->tailQ( this );
2913 gIOStopProviderList->tailQ( provider );
2914
2915 if (0 == gIOTerminateWork++) {
2916 assert(gIOTerminateWorkerThread);
2917 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2918 }
2919
2920 IOLockUnlock( gJobsLock );
2921 }
2922
2923 void
scheduleFinalize(bool now)2924 IOService::scheduleFinalize(bool now)
2925 {
2926 uint64_t regID1 = getRegistryEntryID();
2927
2928 TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2929 IOServiceTrace(
2930 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2931 (uintptr_t) regID1,
2932 (uintptr_t) (regID1 >> 32),
2933 0, 0);
2934
2935 if (now || IOUserClient::finalizeUserReferences(this)) {
2936 IOLockLock( gJobsLock );
2937 gIOFinalizeList->tailQ(this);
2938 if (0 == gIOTerminateWork++) {
2939 assert(gIOTerminateWorkerThread);
2940 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2941 }
2942 IOLockUnlock( gJobsLock );
2943 }
2944 }
2945
2946 bool
willTerminate(IOService * provider,IOOptionBits options)2947 IOService::willTerminate( IOService * provider, IOOptionBits options )
2948 {
2949 if (reserved->uvars) {
2950 IOUserServer::serviceWillTerminate(this, provider, options);
2951 } else if (kIODKLogSetup & gIODKDebug) {
2952 DKLOG("%s[0x%qx]::willTerminate serviceWillTerminate not called\n", getName(), getRegistryEntryID());
2953 }
2954 return true;
2955 }
2956
2957 bool
didTerminate(IOService * provider,IOOptionBits options,bool * defer)2958 IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2959 {
2960 if (reserved->uvars) {
2961 IOUserServer::serviceDidTerminate(this, provider, options, defer);
2962 }
2963
2964 if (false == *defer) {
2965 if (lockForArbitration( true )) {
2966 if (false == provider->handleIsOpen( this )) {
2967 scheduleStop( provider );
2968 }
2969 // -- compat
2970 else {
2971 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2972 if (false == provider->handleIsOpen( this )) {
2973 scheduleStop( provider );
2974 }
2975 }
2976 // --
2977 unlockForArbitration();
2978 }
2979 }
2980
2981 return true;
2982 }
2983
2984 void
actionWillTerminate(IOService * victim,IOOptionBits options,OSArray * doPhase2List,bool user,void * unused3 __unused)2985 IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2986 OSArray * doPhase2List,
2987 bool user,
2988 void *unused3 __unused)
2989 {
2990 OSIterator * iter;
2991 IOService * client;
2992 bool ok;
2993 uint64_t regID1, regID2 = victim->getRegistryEntryID();
2994
2995 iter = victim->getClientIterator();
2996 if (iter) {
2997 while ((client = (IOService *) iter->getNextObject())) {
2998 if (user != (NULL != client->reserved->uvars)) {
2999 continue;
3000 }
3001 regID1 = client->getRegistryEntryID();
3002 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3003 client->getName(), regID1,
3004 victim->getName(), regID2, (long long)options);
3005 IOServiceTrace(
3006 IOSERVICE_TERMINATE_WILL,
3007 (uintptr_t) regID1,
3008 (uintptr_t) (regID1 >> 32),
3009 (uintptr_t) regID2,
3010 (uintptr_t) (regID2 >> 32));
3011
3012 ok = client->willTerminate( victim, options );
3013 doPhase2List->tailQ( client );
3014 }
3015 iter->release();
3016 }
3017 }
3018
3019 void
actionDidTerminate(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3020 IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
3021 void *unused1 __unused, void *unused2 __unused,
3022 void *unused3 __unused )
3023 {
3024 OSIterator * iter;
3025 IOService * client;
3026 bool defer;
3027 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3028
3029 victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
3030
3031 iter = victim->getClientIterator();
3032 if (iter) {
3033 while ((client = (IOService *) iter->getNextObject())) {
3034 regID1 = client->getRegistryEntryID();
3035 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3036 client->getName(), regID1,
3037 victim->getName(), regID2, (long long)options);
3038 defer = false;
3039 client->didTerminate( victim, options, &defer );
3040
3041 IOServiceTrace(
3042 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3043 : IOSERVICE_TERMINATE_DID),
3044 (uintptr_t) regID1,
3045 (uintptr_t) (regID1 >> 32),
3046 (uintptr_t) regID2,
3047 (uintptr_t) (regID2 >> 32));
3048
3049 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3050 client->getName(), regID1,
3051 victim->getName(), regID2, defer);
3052 }
3053 iter->release();
3054 }
3055 }
3056
3057
3058 void
actionWillStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3059 IOService::actionWillStop( IOService * victim, IOOptionBits options,
3060 void *unused1 __unused, void *unused2 __unused,
3061 void *unused3 __unused )
3062 {
3063 OSIterator * iter;
3064 IOService * provider;
3065 bool ok;
3066 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3067
3068 iter = victim->getProviderIterator();
3069 if (iter) {
3070 while ((provider = (IOService *) iter->getNextObject())) {
3071 regID1 = provider->getRegistryEntryID();
3072 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3073 victim->getName(), regID2,
3074 provider->getName(), regID1, (long long)options);
3075 IOServiceTrace(
3076 IOSERVICE_TERMINATE_WILL,
3077 (uintptr_t) regID2,
3078 (uintptr_t) (regID2 >> 32),
3079 (uintptr_t) regID1,
3080 (uintptr_t) (regID1 >> 32));
3081
3082 ok = victim->willTerminate( provider, options );
3083 }
3084 iter->release();
3085 }
3086 }
3087
3088 void
actionDidStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3089 IOService::actionDidStop( IOService * victim, IOOptionBits options,
3090 void *unused1 __unused, void *unused2 __unused,
3091 void *unused3 __unused )
3092 {
3093 OSIterator * iter;
3094 IOService * provider;
3095 bool defer = false;
3096 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3097
3098 iter = victim->getProviderIterator();
3099 if (iter) {
3100 while ((provider = (IOService *) iter->getNextObject())) {
3101 regID1 = provider->getRegistryEntryID();
3102 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3103 victim->getName(), regID2,
3104 provider->getName(), regID1, (long long)options);
3105 victim->didTerminate( provider, options, &defer );
3106
3107 IOServiceTrace(
3108 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3109 : IOSERVICE_TERMINATE_DID),
3110 (uintptr_t) regID2,
3111 (uintptr_t) (regID2 >> 32),
3112 (uintptr_t) regID1,
3113 (uintptr_t) (regID1 >> 32));
3114
3115 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3116 victim->getName(), regID2,
3117 provider->getName(), regID1, defer);
3118 }
3119 iter->release();
3120 }
3121 }
3122
3123
3124 void
actionFinalize(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3125 IOService::actionFinalize( IOService * victim, IOOptionBits options,
3126 void *unused1 __unused, void *unused2 __unused,
3127 void *unused3 __unused )
3128 {
3129 uint64_t regID1 = victim->getRegistryEntryID();
3130 TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
3131 IOServiceTrace(
3132 IOSERVICE_TERMINATE_FINALIZE,
3133 (uintptr_t) regID1,
3134 (uintptr_t) (regID1 >> 32),
3135 0, 0);
3136
3137 victim->finalize( options );
3138 }
3139
3140 void
actionStop(IOService * provider,IOService * client,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3141 IOService::actionStop( IOService * provider, IOService * client,
3142 void *unused1 __unused, void *unused2 __unused,
3143 void *unused3 __unused )
3144 {
3145 uint64_t regID1 = provider->getRegistryEntryID();
3146 uint64_t regID2 = client->getRegistryEntryID();
3147
3148 TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3149 IOServiceTrace(
3150 IOSERVICE_TERMINATE_STOP,
3151 (uintptr_t) regID1,
3152 (uintptr_t) (regID1 >> 32),
3153 (uintptr_t) regID2,
3154 (uintptr_t) (regID2 >> 32));
3155
3156 client->stop( provider );
3157 if (provider->isOpen( client )) {
3158 provider->close( client );
3159 }
3160
3161 TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3162 client->detach( provider );
3163 }
3164
3165 #pragma clang diagnostic push
3166 #pragma clang diagnostic ignored "-Wcast-function-type"
3167
3168 void
terminateWorker(IOOptionBits options)3169 IOService::terminateWorker( IOOptionBits options )
3170 {
3171 OSArray * doPhase2List;
3172 OSArray * didPhase2List;
3173 OSSet * freeList;
3174 OSIterator * iter;
3175 UInt32 workDone;
3176 IOService * victim;
3177 IOService * client;
3178 IOService * provider;
3179 unsigned int idx;
3180 bool moreToDo;
3181 bool doPhase2;
3182 bool doPhase3;
3183
3184 options |= kIOServiceRequired;
3185
3186 doPhase2List = OSArray::withCapacity( 16 );
3187 didPhase2List = OSArray::withCapacity( 16 );
3188 freeList = OSSet::withCapacity( 16 );
3189 if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
3190 OSSafeReleaseNULL(doPhase2List);
3191 OSSafeReleaseNULL(didPhase2List);
3192 OSSafeReleaseNULL(freeList);
3193 return;
3194 }
3195
3196 do {
3197 workDone = gIOTerminateWork;
3198
3199 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(0))) {
3200 victim->retain();
3201 gIOTerminatePhase2List->removeObject(0);
3202 IOLockUnlock( gJobsLock );
3203
3204 uint64_t regID1 = victim->getRegistryEntryID();
3205 IOServiceTrace(
3206 IOSERVICE_TERM_START_PHASE2,
3207 (uintptr_t) regID1,
3208 (uintptr_t) (regID1 >> 32),
3209 (uintptr_t) 0,
3210 (uintptr_t) 0);
3211
3212 while (victim) {
3213 doPhase2 = victim->lockForArbitration( true );
3214 if (doPhase2) {
3215 doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
3216 if (doPhase2) {
3217 uint64_t regID1 = victim->getRegistryEntryID();
3218 IOServiceTrace(
3219 IOSERVICE_TERM_TRY_PHASE2,
3220 (uintptr_t) regID1,
3221 (uintptr_t) (regID1 >> 32),
3222 (uintptr_t) victim->__state[1],
3223 (uintptr_t) 0);
3224
3225 doPhase2 = (0 == (victim->__state[1] &
3226 (kIOServiceTermPhase1State
3227 | kIOServiceTermPhase2State
3228 | kIOServiceConfigState)));
3229
3230 if (doPhase2 && (iter = victim->getClientIterator())) {
3231 while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3232 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3233 if (!doPhase2) {
3234 uint64_t regID1 = client->getRegistryEntryID();
3235 IOServiceTrace(
3236 IOSERVICE_TERM_UC_DEFER,
3237 (uintptr_t) regID1,
3238 (uintptr_t) (regID1 >> 32),
3239 (uintptr_t) client->__state[1],
3240 (uintptr_t) 0);
3241 TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3242 victim->getName(), victim->getRegistryEntryID(),
3243 client->getName(), client->getRegistryEntryID());
3244 }
3245 }
3246 iter->release();
3247 }
3248 if (doPhase2) {
3249 victim->__state[1] |= kIOServiceTermPhase2State;
3250 }
3251 }
3252 victim->unlockForArbitration();
3253 }
3254 if (doPhase2) {
3255 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3256 if (NULL == victim->reserved->uvars) {
3257 _workLoopAction((IOWorkLoop::Action) &actionWillStop,
3258 victim, (void *)(uintptr_t) options);
3259 } else {
3260 actionWillStop(victim, options, NULL, NULL, NULL);
3261 }
3262 }
3263
3264 OSArray * notifiers;
3265 notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
3266 victim->invokeNotifiers(¬ifiers);
3267
3268 _workLoopAction((IOWorkLoop::Action) &actionWillTerminate,
3269 victim,
3270 (void *)(uintptr_t) options,
3271 (void *)(uintptr_t) doPhase2List,
3272 (void *)(uintptr_t) false);
3273
3274 actionWillTerminate(
3275 victim, options, doPhase2List, true, NULL);
3276
3277 didPhase2List->headQ( victim );
3278 }
3279 victim->release();
3280 victim = (IOService *) doPhase2List->getObject(0);
3281 if (victim) {
3282 victim->retain();
3283 doPhase2List->removeObject(0);
3284 }
3285 }
3286
3287 while ((victim = (IOService *) didPhase2List->getObject(0))) {
3288 bool scheduleFinalize = false;
3289 if (victim->lockForArbitration( true )) {
3290 victim->__state[1] |= kIOServiceTermPhase3State;
3291 scheduleFinalize = (NULL == victim->getClient());
3292 victim->unlockForArbitration();
3293 }
3294 _workLoopAction((IOWorkLoop::Action) &actionDidTerminate,
3295 victim, (void *)(uintptr_t) options );
3296 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3297 _workLoopAction((IOWorkLoop::Action) &actionDidStop,
3298 victim, (void *)(uintptr_t) options, NULL );
3299 }
3300 // no clients - will go to finalize
3301 if (scheduleFinalize) {
3302 victim->scheduleFinalize(false);
3303 }
3304 didPhase2List->removeObject(0);
3305 }
3306 IOLockLock( gJobsLock );
3307 }
3308
3309 // phase 3
3310 do {
3311 doPhase3 = false;
3312 // finalize leaves
3313 while ((victim = (IOService *) gIOFinalizeList->getObject(0))) {
3314 bool sendFinal = false;
3315 IOLockUnlock( gJobsLock );
3316 if (victim->lockForArbitration(true)) {
3317 sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3318 if (sendFinal) {
3319 victim->__state[1] |= kIOServiceFinalized;
3320 }
3321 victim->unlockForArbitration();
3322 }
3323 if (sendFinal) {
3324 _workLoopAction((IOWorkLoop::Action) &actionFinalize,
3325 victim, (void *)(uintptr_t) options );
3326 }
3327 IOLockLock( gJobsLock );
3328 // hold off free
3329 freeList->setObject( victim );
3330 // safe if finalize list is append only
3331 gIOFinalizeList->removeObject(0);
3332 }
3333
3334 for (idx = 0;
3335 (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx));) {
3336 provider = (IOService *) gIOStopProviderList->getObject(idx);
3337 assert( provider );
3338
3339 uint64_t regID1 = provider->getRegistryEntryID();
3340 uint64_t regID2 = client->getRegistryEntryID();
3341
3342 if (!provider->isChild( client, gIOServicePlane )) {
3343 // may be multiply queued - nop it
3344 TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3345 IOServiceTrace(
3346 IOSERVICE_TERMINATE_STOP_NOP,
3347 (uintptr_t) regID1,
3348 (uintptr_t) (regID1 >> 32),
3349 (uintptr_t) regID2,
3350 (uintptr_t) (regID2 >> 32));
3351 } else {
3352 // a terminated client is not ready for stop if it has clients, skip it
3353 bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3354 IOLockUnlock( gJobsLock );
3355 if (deferStop && client->lockForArbitration(true)) {
3356 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3357 //deferStop = (!deferStop && (0 != client->getClient()));
3358 //deferStop = (0 != client->getClient());
3359 client->unlockForArbitration();
3360 if (deferStop) {
3361 TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3362 IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3363 (uintptr_t) regID1,
3364 (uintptr_t) (regID1 >> 32),
3365 (uintptr_t) regID2,
3366 (uintptr_t) (regID2 >> 32));
3367
3368 idx++;
3369 IOLockLock( gJobsLock );
3370 continue;
3371 }
3372 }
3373 _workLoopAction((IOWorkLoop::Action) &actionStop,
3374 provider, (void *) client );
3375 IOLockLock( gJobsLock );
3376 // check the finalize list now
3377 doPhase3 = true;
3378 }
3379 // hold off free
3380 freeList->setObject( client );
3381 freeList->setObject( provider );
3382
3383 // safe if stop list is append only
3384 gIOStopList->removeObject( idx );
3385 gIOStopProviderList->removeObject( idx );
3386 idx = 0;
3387 }
3388 } while (doPhase3);
3389
3390 gIOTerminateWork -= workDone;
3391 moreToDo = (gIOTerminateWork != 0);
3392
3393 if (!moreToDo) {
3394 TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3395 IOServiceTrace(
3396 IOSERVICE_TERMINATE_DONE,
3397 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3398 }
3399 } while (moreToDo);
3400
3401 IOLockUnlock( gJobsLock );
3402
3403 freeList->release();
3404 doPhase2List->release();
3405 didPhase2List->release();
3406
3407 IOLockLock( gJobsLock );
3408 }
3409
3410 #pragma clang diagnostic pop
3411
3412 bool
finalize(IOOptionBits options)3413 IOService::finalize( IOOptionBits options )
3414 {
3415 OSIterator * iter;
3416 IOService * provider;
3417 uint64_t regID1, regID2 = getRegistryEntryID();
3418
3419 iter = getProviderIterator();
3420 assert( iter );
3421
3422 if (iter) {
3423 while ((provider = (IOService *) iter->getNextObject())) {
3424 // -- compat
3425 if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3426 /* we come down here on programmatic terminate */
3427
3428 regID1 = provider->getRegistryEntryID();
3429 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3430 IOServiceTrace(
3431 IOSERVICE_TERMINATE_STOP,
3432 (uintptr_t) regID1,
3433 (uintptr_t) (regID1 >> 32),
3434 (uintptr_t) regID2,
3435 (uintptr_t) (regID2 >> 32));
3436
3437 stop( provider );
3438 if (provider->isOpen( this )) {
3439 provider->close( this );
3440 }
3441 detach( provider );
3442 } else {
3443 //--
3444 if (provider->lockForArbitration( true )) {
3445 if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3446 scheduleStop( provider );
3447 }
3448 provider->unlockForArbitration();
3449 }
3450 }
3451 }
3452 iter->release();
3453 }
3454
3455 return true;
3456 }
3457
3458 #undef tailQ
3459 #undef headQ
3460
3461 /*
3462 * Terminate
3463 */
3464
3465 void
doServiceTerminate(IOOptionBits options)3466 IOService::doServiceTerminate( IOOptionBits options )
3467 {
3468 }
3469
3470 // a method in case someone needs to override it
3471 bool
terminateClient(IOService * client,IOOptionBits options)3472 IOService::terminateClient( IOService * client, IOOptionBits options )
3473 {
3474 bool ok;
3475
3476 if (client->isParent( this, gIOServicePlane, true)) {
3477 // we are the clients only provider
3478 ok = client->terminate( options );
3479 } else {
3480 ok = true;
3481 }
3482
3483 return ok;
3484 }
3485
3486 bool
terminate(IOOptionBits options)3487 IOService::terminate( IOOptionBits options )
3488 {
3489 options |= kIOServiceTerminate;
3490
3491 return terminatePhase1( options );
3492 }
3493
3494 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3495
3496 /*
3497 * Open & close
3498 */
3499
3500 struct ServiceOpenMessageContext {
3501 IOService * service;
3502 UInt32 type;
3503 IOService * excludeClient;
3504 IOOptionBits options;
3505 };
3506
3507 static void
serviceOpenMessageApplier(OSObject * object,void * ctx)3508 serviceOpenMessageApplier( OSObject * object, void * ctx )
3509 {
3510 ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3511
3512 if (object != context->excludeClient) {
3513 context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
3514 }
3515 }
3516
3517 bool
open(IOService * forClient,IOOptionBits options,void * arg)3518 IOService::open( IOService * forClient,
3519 IOOptionBits options,
3520 void * arg )
3521 {
3522 bool ok;
3523 kern_return_t ret = kIOReturnSuccess;
3524 ServiceOpenMessageContext context;
3525
3526 context.service = this;
3527 context.type = kIOMessageServiceIsAttemptingOpen;
3528 context.excludeClient = forClient;
3529 context.options = options;
3530
3531 applyToInterested( gIOGeneralInterest,
3532 &serviceOpenMessageApplier, &context );
3533
3534 if (false == lockForArbitration(false)) {
3535 return false;
3536 }
3537
3538 ok = (0 == (__state[0] & kIOServiceInactiveState));
3539
3540 if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3541 ret = forClient->reserved->uvars->userServer->serviceOpen(this, forClient);
3542 if (ret != kIOReturnSuccess) {
3543 ok = false;
3544 }
3545 }
3546
3547 if (ok) {
3548 ok = handleOpen( forClient, options, arg );
3549
3550 if (!ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3551 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3552 }
3553 }
3554
3555 unlockForArbitration();
3556
3557 return ok;
3558 }
3559
3560 void
close(IOService * forClient,IOOptionBits options)3561 IOService::close( IOService * forClient,
3562 IOOptionBits options )
3563 {
3564 bool wasClosed;
3565 bool last = false;
3566
3567 lockForArbitration();
3568
3569 wasClosed = handleIsOpen( forClient );
3570 if (wasClosed) {
3571 handleClose( forClient, options );
3572 last = (__state[1] & kIOServiceTermPhase3State);
3573
3574 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3575 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3576 }
3577 }
3578
3579 unlockForArbitration();
3580
3581 if (last) {
3582 forClient->scheduleStop( this );
3583 } else if (wasClosed) {
3584 ServiceOpenMessageContext context;
3585
3586 context.service = this;
3587 context.type = kIOMessageServiceWasClosed;
3588 context.excludeClient = forClient;
3589 context.options = options;
3590
3591 applyToInterested( gIOGeneralInterest,
3592 &serviceOpenMessageApplier, &context );
3593 }
3594 }
3595
3596 bool
isOpen(const IOService * forClient) const3597 IOService::isOpen( const IOService * forClient ) const
3598 {
3599 IOService * self = (IOService *) this;
3600 bool ok;
3601
3602 self->lockForArbitration();
3603
3604 ok = handleIsOpen( forClient );
3605
3606 self->unlockForArbitration();
3607
3608 return ok;
3609 }
3610
3611 bool
handleOpen(IOService * forClient,IOOptionBits options,void * arg)3612 IOService::handleOpen( IOService * forClient,
3613 IOOptionBits options,
3614 void * arg )
3615 {
3616 bool ok;
3617
3618 ok = (NULL == __owner);
3619 if (ok) {
3620 __owner = forClient;
3621 } else if (options & kIOServiceSeize) {
3622 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3623 __owner, (void *)(uintptr_t) options ));
3624 if (ok && (NULL == __owner)) {
3625 __owner = forClient;
3626 } else {
3627 ok = false;
3628 }
3629 }
3630 return ok;
3631 }
3632
3633 void
handleClose(IOService * forClient,IOOptionBits options)3634 IOService::handleClose( IOService * forClient,
3635 IOOptionBits options )
3636 {
3637 if (__owner == forClient) {
3638 __owner = NULL;
3639 }
3640 }
3641
3642 bool
handleIsOpen(const IOService * forClient) const3643 IOService::handleIsOpen( const IOService * forClient ) const
3644 {
3645 if (forClient) {
3646 return __owner == forClient;
3647 } else {
3648 return __owner != forClient;
3649 }
3650 }
3651
3652 /*
3653 * Probing & starting
3654 */
3655 static SInt32
IONotifyOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3656 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3657 {
3658 const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3659 const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3660 SInt32 val1;
3661 SInt32 val2;
3662
3663 val1 = 0;
3664 val2 = 0;
3665 if (obj1) {
3666 val1 = obj1->priority;
3667 }
3668 if (obj2) {
3669 val2 = obj2->priority;
3670 }
3671 if (val1 > val2) {
3672 return 1;
3673 }
3674 if (val1 < val2) {
3675 return -1;
3676 }
3677 return 0;
3678 }
3679
3680 static SInt32
IOServiceObjectOrder(const OSObject * entry,void * ref)3681 IOServiceObjectOrder( const OSObject * entry, void * ref)
3682 {
3683 OSDictionary * dict;
3684 IOService * service;
3685 _IOServiceNotifier * notify;
3686 OSSymbol * key = (OSSymbol *) ref;
3687 OSNumber * offset;
3688 OSObject * prop;
3689 SInt32 result;
3690
3691 prop = NULL;
3692 result = kIODefaultProbeScore;
3693 if ((dict = OSDynamicCast( OSDictionary, entry))) {
3694 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3695 } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3696 return notify->priority;
3697 } else if ((service = OSDynamicCast( IOService, entry))) {
3698 prop = service->copyProperty(key);
3699 offset = OSDynamicCast(OSNumber, prop);
3700 } else {
3701 assert( false );
3702 offset = NULL;
3703 }
3704
3705 if (offset) {
3706 result = offset->unsigned32BitValue();
3707 }
3708
3709 OSSafeReleaseNULL(prop);
3710
3711 return result;
3712 }
3713
3714 __attribute__((no_sanitize("signed-integer-overflow"))) SInt32
IOServiceOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3715 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3716 {
3717 const OSObject * obj1 = (const OSObject *) inObj1;
3718 const OSObject * obj2 = (const OSObject *) inObj2;
3719 SInt32 val1;
3720 SInt32 val2;
3721
3722 val1 = 0;
3723 val2 = 0;
3724
3725 if (obj1) {
3726 val1 = IOServiceObjectOrder( obj1, ref );
3727 }
3728
3729 if (obj2) {
3730 val2 = IOServiceObjectOrder( obj2, ref );
3731 }
3732
3733 return val1 - val2;
3734 }
3735
3736 IOService *
copyClientWithCategory(const OSSymbol * category)3737 IOService::copyClientWithCategory( const OSSymbol * category )
3738 {
3739 IOService * service = NULL;
3740 OSIterator * iter;
3741 const OSSymbol * nextCat;
3742
3743 iter = getClientIterator();
3744 if (iter) {
3745 while ((service = (IOService *) iter->getNextObject())) {
3746 if (kIOServiceInactiveState & service->__state[0]) {
3747 if (!(kIOServiceRematchOnDetach & service->__state[1])) {
3748 continue;
3749 }
3750 }
3751 nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3752 service->getProperty( gIOMatchCategoryKey ));
3753 if (category == nextCat) {
3754 service->retain();
3755 break;
3756 }
3757 }
3758 iter->release();
3759 }
3760 return service;
3761 }
3762
3763 IOService *
getClientWithCategory(const OSSymbol * category)3764 IOService::getClientWithCategory( const OSSymbol * category )
3765 {
3766 IOService *
3767 service = copyClientWithCategory(category);
3768 if (service) {
3769 service->release();
3770 }
3771 return service;
3772 }
3773
3774 bool
invokeNotifier(_IOServiceNotifier * notify)3775 IOService::invokeNotifier( _IOServiceNotifier * notify )
3776 {
3777 _IOServiceNotifierInvocation invocation;
3778 bool willNotify;
3779 bool ret = true;
3780 invocation.thread = current_thread();
3781
3782 #if DEBUG_NOTIFIER_LOCKED
3783 uint32_t count;
3784 if ((count = isLockedForArbitration(0))) {
3785 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3786 panic("[%s, 0x%x]", notify->type->getCStringNoCopy(), count);
3787 }
3788 #endif /* DEBUG_NOTIFIER_LOCKED */
3789
3790 LOCKWRITENOTIFY();
3791 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3792
3793 if (willNotify) {
3794 queue_enter( ¬ify->handlerInvocations, &invocation,
3795 _IOServiceNotifierInvocation *, link );
3796 }
3797 UNLOCKNOTIFY();
3798
3799 if (willNotify) {
3800 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3801
3802 LOCKWRITENOTIFY();
3803 queue_remove( ¬ify->handlerInvocations, &invocation,
3804 _IOServiceNotifierInvocation *, link );
3805 if (kIOServiceNotifyWaiter & notify->state) {
3806 notify->state &= ~kIOServiceNotifyWaiter;
3807 WAKEUPNOTIFY( notify );
3808 }
3809 UNLOCKNOTIFY();
3810 }
3811
3812 return ret;
3813 }
3814
3815 bool
invokeNotifiers(OSArray * willSend[])3816 IOService::invokeNotifiers(OSArray * willSend[])
3817 {
3818 OSArray * array;
3819 _IOServiceNotifier * notify;
3820 bool ret = true;
3821
3822 array = *willSend;
3823 if (!array) {
3824 return true;
3825 }
3826 *willSend = NULL;
3827
3828 for (unsigned int idx = 0;
3829 (notify = (_IOServiceNotifier *) array->getObject(idx));
3830 idx++) {
3831 ret &= invokeNotifier(notify);
3832 }
3833 array->release();
3834
3835 return ret;
3836 }
3837
3838
3839 TUNABLE(bool, iokit_print_verbose_match_logs, "iokit_print_verbose_match_logs", false);
3840
3841 /*
3842 * Alloc and probe matching classes,
3843 * called on the provider instance
3844 */
3845
3846 void
probeCandidates(OSOrderedSet * matches)3847 IOService::probeCandidates( OSOrderedSet * matches )
3848 {
3849 OSDictionary * match = NULL;
3850 OSSymbol * symbol;
3851 IOService * inst;
3852 IOService * newInst;
3853 OSDictionary * props;
3854 SInt32 score;
3855 OSNumber * newPri;
3856 OSOrderedSet * familyMatches = NULL;
3857 OSOrderedSet * startList;
3858 OSSet * kexts = NULL;
3859 OSObject * kextRef;
3860
3861 OSDictionary * startDict = NULL;
3862 const OSSymbol * category;
3863 OSIterator * iter;
3864 _IOServiceNotifier * notify;
3865 OSObject * nextMatch = NULL;
3866 bool started;
3867 bool needReloc = false;
3868 bool matchDeferred = false;
3869 #if IOMATCHDEBUG
3870 SInt64 debugFlags;
3871 #endif
3872 IOService * client = NULL;
3873 OSObject * prop1;
3874 OSObject * rematchCountProp;
3875 OSDictionary * rematchPersonality;
3876 OSNumber * num;
3877 uint32_t count;
3878 uint32_t dextCount;
3879 bool isDext;
3880 bool categoryConsumed;
3881
3882 rematchCountProp = NULL;
3883 count = 0;
3884 prop1 = copyProperty(gIORematchPersonalityKey);
3885 rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3886 if (rematchPersonality) {
3887 rematchCountProp = copyProperty(gIORematchCountKey);
3888 num = OSDynamicCast(OSNumber, rematchCountProp);
3889 if (num) {
3890 count = num->unsigned32BitValue();
3891 }
3892 removeProperty(gIORematchPersonalityKey);
3893 }
3894 dextCount = 0;
3895
3896 assert( matches );
3897 while (!needReloc
3898 && (nextMatch = matches->getFirstObject())) {
3899 nextMatch->retain();
3900 matches->removeObject(nextMatch);
3901
3902 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3903 if (0 == (__state[0] & kIOServiceInactiveState)) {
3904 invokeNotifier( notify );
3905 }
3906 nextMatch->release();
3907 nextMatch = NULL;
3908 continue;
3909 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3910 nextMatch->release();
3911 nextMatch = NULL;
3912 continue;
3913 }
3914
3915 props = NULL;
3916 #if IOMATCHDEBUG
3917 debugFlags = getDebugFlags( match );
3918 #endif
3919
3920 bool newIsBoot = false;
3921 bool existingIsBoot = false;
3922 bool isReplacementCandidate = false;
3923
3924 do {
3925 client = NULL;
3926 isDext = (NULL != match->getObject(gIOUserServerNameKey));
3927 if (isDext && !(kIODKEnable & gIODKDebug)) {
3928 continue;
3929 }
3930 if (isDext && !OSKext::iokitDaemonAvailable()) {
3931 continue;
3932 }
3933 if (isDext && !gIODextRelaunchMax && rematchCountProp) {
3934 continue;
3935 }
3936 if (isDext && isInactive()) {
3937 continue;
3938 }
3939 newIsBoot = gIOCatalogue->personalityIsBoot(match);
3940
3941 category = OSDynamicCast( OSSymbol,
3942 match->getObject( gIOMatchCategoryKey ));
3943 if (NULL == category) {
3944 category = gIODefaultMatchCategoryKey;
3945 }
3946 client = copyClientWithCategory(category);
3947
3948 categoryConsumed = (client != NULL);
3949 if (categoryConsumed) {
3950 #if IOMATCHDEBUG
3951 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3952 LOG("%s: match category %s exists\n", getName(),
3953 category->getCStringNoCopy());
3954 }
3955 #endif
3956 existingIsBoot = client->propertyExists(gIOMatchedAtBootKey);
3957 isReplacementCandidate = existingIsBoot && !newIsBoot;
3958 if (!isDext && !isReplacementCandidate) {
3959 break;
3960 }
3961 }
3962
3963 // create a copy now in case its modified during matching
3964 props = OSDictionary::withDictionary(match, match->getCount());
3965 if (NULL == props) {
3966 break;
3967 }
3968 props->setCapacityIncrement(1);
3969
3970 // check the nub matches
3971 if (false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) {
3972 break;
3973 }
3974 if (isReplacementCandidate) {
3975 if (canTerminateForReplacement(client)) {
3976 client->terminate(kIOServiceTerminateNeedWillTerminate | kIOServiceTerminateWithRematch);
3977 break;
3978 }
3979 }
3980
3981 if (isDext || isReplacementCandidate) {
3982 if (isDext) {
3983 dextCount++;
3984 }
3985 if (categoryConsumed) {
3986 break;
3987 }
3988 }
3989 if (rematchPersonality) {
3990 bool personalityMatch = match->isEqualTo(rematchPersonality);
3991 if (count > gIODextRelaunchMax) {
3992 personalityMatch = !personalityMatch;
3993 }
3994 if (!personalityMatch) {
3995 break;
3996 }
3997 }
3998
3999 // Check to see if driver reloc has been loaded.
4000 needReloc = (false == gIOCatalogue->isModuleLoaded( match, &kextRef ));
4001 if (needReloc) {
4002 #if IOMATCHDEBUG
4003 if (debugFlags & kIOLogCatalogue) {
4004 LOG("%s: stalling for module\n", getName());
4005 }
4006 #endif
4007 // If reloc hasn't been loaded, exit;
4008 // reprobing will occur after reloc has been loaded.
4009 break;
4010 }
4011 if (kextRef) {
4012 if (NULL == kexts) {
4013 kexts = OSSet::withCapacity(1);
4014 }
4015 if (kexts) {
4016 kexts->setObject(kextRef);
4017 kextRef->release();
4018 }
4019 }
4020 if (newIsBoot) {
4021 props->setObject(gIOMatchedAtBootKey, kOSBooleanTrue);
4022 }
4023 if (isDext) {
4024 // copy saved for rematchng
4025 props->setObject(gIOMatchedPersonalityKey, match);
4026 }
4027 // reorder on family matchPropertyTable score.
4028 if (NULL == familyMatches) {
4029 familyMatches = OSOrderedSet::withCapacity( 1,
4030 IOServiceOrdering, (void *) gIOProbeScoreKey );
4031 }
4032 if (familyMatches) {
4033 familyMatches->setObject( props );
4034 }
4035 } while (false);
4036
4037 OSSafeReleaseNULL(client);
4038 OSSafeReleaseNULL(nextMatch);
4039 OSSafeReleaseNULL(props);
4040 }
4041 OSSafeReleaseNULL(matches);
4042 OSSafeReleaseNULL(rematchCountProp);
4043
4044 if (familyMatches) {
4045 while (!needReloc
4046 && (props = (OSDictionary *) familyMatches->getFirstObject())) {
4047 props->retain();
4048 familyMatches->removeObject( props );
4049
4050 inst = NULL;
4051 newInst = NULL;
4052 #if IOMATCHDEBUG
4053 debugFlags = getDebugFlags( props );
4054 #endif
4055 do {
4056 symbol = OSDynamicCast( OSSymbol,
4057 props->getObject( gIOClassKey));
4058 if (!symbol) {
4059 continue;
4060 }
4061
4062 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
4063
4064 // alloc the driver instance
4065 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
4066
4067 if (!inst || !OSDynamicCast(IOService, inst)) {
4068 IOLog("Couldn't alloc class \"%s\"\n",
4069 symbol->getCStringNoCopy());
4070 continue;
4071 }
4072
4073 // init driver instance
4074 if (!(inst->init( props ))) {
4075 #if IOMATCHDEBUG
4076 if (debugFlags & kIOLogStart) {
4077 IOLog("%s::init in provider %s[0x%qx] fails\n", symbol->getCStringNoCopy(), getName(), getRegistryEntryID());
4078 }
4079 #endif
4080 continue;
4081 }
4082 if (__state[1] & kIOServiceSynchronousState) {
4083 inst->__state[1] |= kIOServiceSynchronousState;
4084 }
4085
4086 // give the driver the default match category if not specified
4087 category = OSDynamicCast( OSSymbol,
4088 props->getObject( gIOMatchCategoryKey ));
4089 if (NULL == category) {
4090 category = gIODefaultMatchCategoryKey;
4091 }
4092 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
4093 // attach driver instance
4094 if (!(inst->attach( this ))) {
4095 continue;
4096 }
4097
4098 // pass in score from property table
4099 score = familyMatches->orderObject( props );
4100
4101 // & probe the new driver instance
4102 #if IOMATCHDEBUG
4103 if (debugFlags & kIOLogProbe) {
4104 LOG("%s[0x%qx]::probe(%s)\n",
4105 inst->getMetaClass()->getClassName(), inst->getRegistryEntryID(), getName());
4106 }
4107 #endif
4108 newInst = inst->probe( this, &score );
4109 inst->detach( this );
4110 if (NULL == newInst) {
4111 #if IOMATCHDEBUG
4112 if (debugFlags & kIOLogProbe) {
4113 IOLog("%s[0x%qx]::probe fails\n", symbol->getCStringNoCopy(), inst->getRegistryEntryID());
4114 }
4115 #endif
4116 continue;
4117 }
4118
4119 // save the score
4120 newPri = OSNumber::withNumber( score, 32 );
4121 if (newPri) {
4122 newInst->setProperty( gIOProbeScoreKey, newPri );
4123 newPri->release();
4124 }
4125
4126 // add to start list for the match category
4127 if (NULL == startDict) {
4128 startDict = OSDictionary::withCapacity( 1 );
4129 }
4130 assert( startDict );
4131 startList = (OSOrderedSet *)
4132 startDict->getObject( category );
4133 if (NULL == startList) {
4134 startList = OSOrderedSet::withCapacity( 1,
4135 IOServiceOrdering, (void *) gIOProbeScoreKey );
4136 if (startDict && startList) {
4137 startDict->setObject( category, startList );
4138 startList->release();
4139 }
4140 }
4141 assert( startList );
4142 if (startList) {
4143 startList->setObject( newInst );
4144 }
4145 } while (false);
4146
4147 props->release();
4148 if (inst) {
4149 inst->release();
4150 }
4151 }
4152 familyMatches->release();
4153 familyMatches = NULL;
4154 }
4155
4156 if ((debugFlags & kIOLogMatch) && iokit_print_verbose_match_logs && startDict != NULL) {
4157 IOLog("%s(0x%qx): %u categories\n", getName(), getRegistryEntryID(), startList->getCount());
4158 startDict->iterateObjects(^(const OSSymbol *key, OSObject *value) {
4159 OSOrderedSet *startList = OSDynamicCast(OSOrderedSet, value);
4160 if (startList) {
4161 IOLog("%s(0x%qx): category %s, %u matches\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), startList->getCount());
4162 startList->iterateObjects(^(OSObject *obj) {
4163 IOService *match = OSDynamicCast(IOService, obj);
4164 OSNumber *probeScore = OSDynamicCast(OSNumber, match->getProperty(gIOProbeScoreKey));
4165
4166 if (match && probeScore) {
4167 IOLog("%s(0x%qx): category %s: matched %s, probe score %qd\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), match->getName(), probeScore->unsigned64BitValue());
4168 }
4169 return false;
4170 });
4171 }
4172 return false;
4173 });
4174 }
4175
4176 // start the best (until success) of each category
4177
4178 iter = OSCollectionIterator::withCollection( startDict );
4179 assert(startDict || !iter);
4180 if (iter) {
4181 while ((category = (const OSSymbol *) iter->getNextObject())) {
4182 startList = (OSOrderedSet *) startDict->getObject( category );
4183 assert( startList );
4184 if (!startList) {
4185 continue;
4186 }
4187 started = false;
4188 while (true // (!started)
4189 && !matchDeferred
4190 && (inst = (IOService *)startList->getFirstObject())) {
4191 inst->retain();
4192 startList->removeObject(inst);
4193 #if IOMATCHDEBUG
4194 debugFlags = getDebugFlags( inst );
4195
4196 if (debugFlags & kIOLogStart) {
4197 if (started) {
4198 LOG( "match category exists, skipping " );
4199 }
4200 LOG( "%s[0x%qx]::start(%s) <%d>\n", inst->getName(), inst->getRegistryEntryID(),
4201 getName(), inst->getRetainCount());
4202 }
4203 #endif
4204 if (false == started) {
4205 #if !NO_KEXTD
4206 IOLockLock(gJobsLock);
4207 matchDeferred = (gIOMatchDeferList
4208 && kOSBooleanTrue == inst->getProperty(gIOMatchDeferKey));
4209 if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(this, 0))) {
4210 gIOMatchDeferList->setObject(this);
4211 }
4212 if (matchDeferred) {
4213 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
4214 IOLog("%s(0x%qx): matching deferred by %s%s\n",
4215 getName(), getRegistryEntryID(),
4216 symbol ? symbol->getCStringNoCopy() : "",
4217 gInUserspaceReboot ? " in userspace reboot" : "");
4218 // rematching will occur after the IOKit daemon loads all plists
4219 }
4220 IOLockUnlock(gJobsLock);
4221 #endif
4222 if (!matchDeferred) {
4223 /* TODO
4224 * If a dext fails to start because an upgrade happened
4225 * concurrently, then the matching process has to restart
4226 */
4227 started = startCandidate( inst );
4228 #if IOMATCHDEBUG
4229 if ((debugFlags & kIOLogStart) && (false == started)) {
4230 LOG( "%s[0x%qx]::start(%s[0x%qx]) <%d> failed\n", inst->getName(), inst->getRegistryEntryID(), getName(), getRegistryEntryID(),
4231 inst->getRetainCount());
4232 }
4233 #endif
4234 if (!started && inst->propertyExists(gIOServiceMatchDeferredKey)) {
4235 matchDeferred = true;
4236 }
4237 }
4238 }
4239 inst->release();
4240 }
4241 }
4242 iter->release();
4243 }
4244
4245 OSSafeReleaseNULL(prop1);
4246
4247 if (dextCount) {
4248 num = OSNumber::withNumber(dextCount, 32);
4249 setProperty(gIODEXTMatchCountKey, num);
4250 OSSafeReleaseNULL(num);
4251 } else if (rematchPersonality) {
4252 removeProperty(gIODEXTMatchCountKey);
4253 }
4254
4255 // now that instances are created, drop the refs on any kexts allowing unload
4256 if (kexts) {
4257 OSKext::dropMatchingReferences(kexts);
4258 OSSafeReleaseNULL(kexts);
4259 }
4260
4261 // adjust the busy count by +1 if matching is stalled for a module,
4262 // or -1 if a previously stalled matching is complete.
4263 lockForArbitration();
4264 SInt32 adjBusy = 0;
4265 uint64_t regID = getRegistryEntryID();
4266
4267 if (needReloc) {
4268 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
4269 if (adjBusy) {
4270 IOServiceTrace(
4271 IOSERVICE_MODULESTALL,
4272 (uintptr_t) regID,
4273 (uintptr_t) (regID >> 32),
4274 (uintptr_t) this,
4275 0);
4276
4277 __state[1] |= kIOServiceModuleStallState;
4278 }
4279 } else if (__state[1] & kIOServiceModuleStallState) {
4280 IOServiceTrace(
4281 IOSERVICE_MODULEUNSTALL,
4282 (uintptr_t) regID,
4283 (uintptr_t) (regID >> 32),
4284 (uintptr_t) this,
4285 0);
4286
4287 __state[1] &= ~kIOServiceModuleStallState;
4288 adjBusy = -1;
4289 }
4290 if (adjBusy) {
4291 _adjustBusy( adjBusy );
4292 }
4293 unlockForArbitration();
4294
4295 if (startDict) {
4296 startDict->release();
4297 }
4298 }
4299
4300 /*
4301 * Wait for a IOUserServer to check in
4302 */
4303
4304 static
4305 __attribute__((noinline, not_tail_called))
4306 IOUserServer *
__WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)4307 __WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)
4308 {
4309 IOUserServer * result = NULL;
4310 IOService * server = NULL;
4311 const OSSymbol * serverName = token->copyServerName();
4312 OSNumber * serverTag = token->copyServerTag();
4313 OSDictionary * matching = IOService::serviceMatching(gIOUserServerClassKey);
4314
4315 if (!matching || !serverName || !serverTag) {
4316 goto finish;
4317 }
4318 IOService::propertyMatching(gIOUserServerNameKey, serverName, matching);
4319 if (!(kIODKDisableDextTag & gIODKDebug)) {
4320 IOService::propertyMatching(gIOUserServerTagKey, serverTag, matching);
4321 }
4322
4323 server = IOService::waitForMatchingServiceWithToken(matching, kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC, token);
4324 result = OSDynamicCast(IOUserServer, server);
4325 if (!result) {
4326 OSSafeReleaseNULL(server);
4327 token->cancel();
4328 }
4329
4330 finish:
4331 OSSafeReleaseNULL(matching);
4332 OSSafeReleaseNULL(serverName);
4333 OSSafeReleaseNULL(serverTag);
4334
4335 return result;
4336 }
4337
4338 void
willShutdown()4339 IOService::willShutdown()
4340 {
4341 gIOKitWillTerminate = true;
4342 #if !NO_KEXTD
4343 IOUserServerCheckInToken::cancelAll();
4344 #endif
4345 OSKext::willShutdown();
4346 }
4347
4348 void
userSpaceWillReboot()4349 IOService::userSpaceWillReboot()
4350 {
4351 IOLockLock(gJobsLock);
4352 #if !NO_KEXTD
4353 IOService * provider;
4354 IOService * service;
4355 OSIterator * iter;
4356
4357 // Recreate the defer list if it does not exist
4358 if (!gIOMatchDeferList && OSKext::iokitDaemonAvailable()) {
4359 gIOMatchDeferList = OSArray::withCapacity( 16 );
4360 }
4361
4362 if (gIOMatchDeferList) {
4363 iter = IORegistryIterator::iterateOver(gIOServicePlane, kIORegistryIterateRecursively);
4364 if (iter) {
4365 do {
4366 iter->reset();
4367 while ((service = (IOService *)iter->getNextObject())) {
4368 /* Rematch providers of services that will be terminated on userspace reboot, after the userspace reboot
4369 * is complete. This normally happens automatically as the IOKit daemon sends personalities to the kernel
4370 * which triggers rematching. But if this doesn't happen (for example, if a feature flag is turned off),
4371 * then these services will never get rematched.
4372 */
4373 if (service->propertyHasValue(gIOMatchDeferKey, kOSBooleanTrue) || service->hasUserServer()) {
4374 provider = service->getProvider();
4375 IOLog("deferring %s-%llx (provider of %s-%llx) matching after userspace reboot\n",
4376 provider->getName(), provider->getRegistryEntryID(), service->getName(), service->getRegistryEntryID());
4377 gIOMatchDeferList->setObject(provider);
4378 }
4379 }
4380 } while (!service && !iter->isValid());
4381
4382 OSSafeReleaseNULL(iter);
4383 }
4384 }
4385 #endif
4386 gInUserspaceReboot = true;
4387 IOLockUnlock(gJobsLock);
4388 }
4389
4390 void
userSpaceDidReboot()4391 IOService::userSpaceDidReboot()
4392 {
4393 IOLockLock(gJobsLock);
4394 gInUserspaceReboot = false;
4395 IOLockUnlock(gJobsLock);
4396 }
4397
4398 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4399
4400 void
init(IOPMrootDomain * root)4401 IOServicePH::init(IOPMrootDomain * root)
4402 {
4403 fUserServers = OSArray::withCapacity(4);
4404 fMatchingWork = OSArray::withCapacity(4);
4405
4406 assert(fUserServers && fMatchingWork);
4407
4408 fRootNotifier = root->registerInterest(
4409 gIOPriorityPowerStateInterest, &IOServicePH::systemPowerChange, NULL, NULL);
4410
4411 assert(fRootNotifier);
4412
4413 fUserServerAckTimer = thread_call_allocate(&IOServicePH::userServerAckTimerExpired, (thread_call_param_t)NULL);
4414 }
4415
4416 void
lock()4417 IOServicePH::lock()
4418 {
4419 IOLockLock(gJobsLock);
4420 }
4421
4422 void
unlock()4423 IOServicePH::unlock()
4424 {
4425 IOLockUnlock(gJobsLock);
4426 }
4427
4428 void
serverAdd(IOUserServer * server)4429 IOServicePH::serverAdd(IOUserServer * server)
4430 {
4431 uint32_t idx;
4432
4433 lock();
4434 idx = fUserServers->getNextIndexOfObject(server, 0);
4435 if (idx == -1U) {
4436 fUserServers->setObject(server);
4437 }
4438 unlock();
4439 }
4440
4441 void
serverRemove(IOUserServer * server)4442 IOServicePH::serverRemove(IOUserServer * server)
4443 {
4444 uint32_t idx;
4445
4446 lock();
4447 idx = fUserServers->getNextIndexOfObject(server, 0);
4448 if (idx != -1U) {
4449 fUserServers->removeObject(idx);
4450 }
4451
4452 if (fWaitingUserServers) {
4453 fWaitingUserServers = false;
4454 IOLockWakeup(gJobsLock, &fWaitingUserServers, /* one-thread */ false);
4455 }
4456
4457 unlock();
4458 }
4459
4460 void
serverAck(IOUserServer * server)4461 IOServicePH::serverAck(IOUserServer * server)
4462 {
4463 uint32_t idx;
4464 IOService * ackTo;
4465 uint32_t ackToRef;
4466
4467 ackTo = NULL;
4468 lock();
4469 if (server && fUserServersWait) {
4470 idx = fUserServersWait->getNextIndexOfObject(server, 0);
4471 if (idx != -1U) {
4472 fUserServersWait->removeObject(idx);
4473 if (0 == fUserServersWait->getCount()) {
4474 OSSafeReleaseNULL(fUserServersWait);
4475 }
4476 }
4477 }
4478 if (!fUserServersWait && !fMatchingWork->getCount()) {
4479 ackTo = fSystemPowerAckTo;
4480 ackToRef = fSystemPowerAckRef;
4481 fSystemPowerAckTo = NULL;
4482 if (ackTo) {
4483 thread_call_cancel(fUserServerAckTimer);
4484 }
4485 }
4486 if (fUserServersWait && fUserServersWait->getCount() > 0 && fMatchingWork && fMatchingWork->getCount() > 0) {
4487 DKLOG("Waiting for %u user servers, %u matching work\n", fUserServersWait->getCount(), fMatchingWork->getCount());
4488 }
4489 unlock();
4490
4491 if (ackTo) {
4492 DKLOG("allowPowerChange\n");
4493 ackTo->allowPowerChange((uintptr_t) ackToRef);
4494 }
4495 }
4496
4497 bool
matchingStart(IOService * service)4498 IOServicePH::matchingStart(IOService * service)
4499 {
4500 uint32_t idx;
4501 bool assertionActive = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
4502
4503 lock();
4504 bool matchNow = !fSystemOff && assertionActive;
4505 if (matchNow) {
4506 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4507 if (idx == -1U) {
4508 fMatchingWork->setObject(service);
4509 }
4510 } else {
4511 // Delay matching if system is transitioning to sleep
4512 if (!fMatchingDelayed) {
4513 fMatchingDelayed = OSArray::withObjects((const OSObject **) &service, 1, 1);
4514 } else {
4515 idx = fMatchingDelayed->getNextIndexOfObject(service, 0);
4516 if (idx == -1U) {
4517 fMatchingDelayed->setObject(service);
4518 }
4519 }
4520 }
4521 unlock();
4522 if (!matchNow && assertionActive) {
4523 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4524 }
4525
4526 return matchNow;
4527 }
4528
4529 void
matchingEnd(IOService * service)4530 IOServicePH::matchingEnd(IOService * service)
4531 {
4532 uint32_t idx;
4533 OSArray * notifyServers;
4534 OSArray * deferredMatches;
4535
4536 notifyServers = NULL;
4537 deferredMatches = NULL;
4538
4539 if (service) {
4540 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4541 }
4542
4543 lock();
4544
4545 if (service) {
4546 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4547 if (idx != -1U) {
4548 fMatchingWork->removeObject(idx);
4549 }
4550 }
4551
4552
4553 if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4554 if (fSystemOff) {
4555 if (0 == fMatchingWork->getCount()) {
4556 fUserServersWait = OSArray::withArray(fUserServers);
4557 notifyServers = OSArray::withArray(fUserServers);
4558 fUserServerOff = fSystemOff;
4559 }
4560 } else {
4561 notifyServers = OSArray::withArray(fUserServers);
4562 fUserServerOff = fSystemOff;
4563 }
4564 }
4565
4566 if (!fSystemOff && fMatchingDelayed) {
4567 deferredMatches = fMatchingDelayed;
4568 fMatchingDelayed = NULL;
4569 }
4570
4571 unlock();
4572
4573 if (notifyServers) {
4574 uint32_t sleepType = 0;
4575 uint32_t standbyTimer = 0;
4576 bool hibernate = false;
4577 if (fSystemOff && IOService::getPMRootDomain()->getSystemSleepType(&sleepType, &standbyTimer) == kIOReturnSuccess) {
4578 hibernate = (sleepType == kIOPMSleepTypeHibernate);
4579 }
4580 notifyServers->iterateObjects(^bool (OSObject * obj) {
4581 IOUserServer * us;
4582 us = (typeof(us))obj;
4583 us->systemPower(fSystemOff, hibernate);
4584 return false;
4585 });
4586 OSSafeReleaseNULL(notifyServers);
4587 }
4588
4589 if (deferredMatches) {
4590 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4591 deferredMatches->iterateObjects(^bool (OSObject * obj)
4592 {
4593 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
4594 return false;
4595 });
4596 deferredMatches->release();
4597 }
4598
4599 serverAck(NULL);
4600 }
4601
4602 TUNABLE(uint32_t, dk_shutdown_timeout_ms, "dk_shutdown_timeout_ms", 30000);
4603 TUNABLE(bool, dk_panic_on_shutdown_hang, "dk_panic_on_shutdown_hang", false);
4604 #if DEVELOPMENT || DEBUG
4605 #define DK_SETPOWERSTATE_HANG true
4606 #else
4607 #define DK_SETPOWERSTATE_HANG false
4608 #endif
4609 TUNABLE(bool, dk_panic_on_setpowerstate_hang, "dk_panic_on_setpowerstate_hang", DK_SETPOWERSTATE_HANG);
4610
4611 void
userServerAckTimerExpired(void *,void *)4612 IOServicePH::userServerAckTimerExpired(void *, void *)
4613 {
4614 OSArray * userServers = NULL;
4615 lock();
4616 if (fSystemPowerAckTo) {
4617 DKLOG("ack timer expired\n");
4618 if (dk_panic_on_setpowerstate_hang) {
4619 panic("DK ack timer expired after %u ms", dk_shutdown_timeout_ms);
4620 }
4621 userServers = fUserServersWait;
4622 fUserServersWait = NULL;
4623 }
4624 unlock();
4625
4626 if (userServers != NULL) {
4627 userServers->iterateObjects(^bool (OSObject *obj) {
4628 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4629 if (us) {
4630 DKLOG(DKS " power state transition failed\n", DKN(us));
4631 us->kill("Power Management Failed");
4632 }
4633 return false;
4634 });
4635 OSSafeReleaseNULL(userServers);
4636 }
4637
4638 serverAck(NULL);
4639 }
4640
4641 void
systemHalt(int howto)4642 IOServicePH::systemHalt(int howto)
4643 {
4644 OSArray * notifyServers;
4645 uint64_t deadline;
4646
4647 lock();
4648 notifyServers = OSArray::withArray(fUserServers);
4649 unlock();
4650
4651 if (notifyServers) {
4652 notifyServers->iterateObjects(^bool (OSObject * obj) {
4653 IOUserServer * us;
4654 us = (typeof(us))obj;
4655 us->systemHalt(howto);
4656 return false;
4657 });
4658 OSSafeReleaseNULL(notifyServers);
4659 }
4660
4661 lock();
4662 clock_interval_to_deadline(dk_shutdown_timeout_ms, kMillisecondScale, &deadline);
4663 while (0 < fUserServers->getCount()) {
4664 fWaitingUserServers = true;
4665 __assert_only int waitResult =
4666 IOLockSleepDeadline(gJobsLock, &fWaitingUserServers, deadline, THREAD_UNINT);
4667 assert((THREAD_AWAKENED == waitResult) || (THREAD_TIMED_OUT == waitResult));
4668 if (THREAD_TIMED_OUT == waitResult) {
4669 IOUserServer::beginLeakingObjects();
4670 #if DEVELOPMENT || DEBUG
4671 if (dk_panic_on_shutdown_hang) {
4672 panic("Shutdown timed out waiting for DK drivers to stop");
4673 }
4674 #endif /* DEVELOPMENT || DEBUG */
4675 break;
4676 }
4677 }
4678 unlock();
4679 }
4680
4681 bool
serverSlept(void)4682 IOServicePH::serverSlept(void)
4683 {
4684 bool ret;
4685
4686 lock();
4687 ret = (kIOMessageSystemWillSleep == sSystemPower)
4688 || (kIOMessageSystemWillPowerOff == sSystemPower)
4689 || (kIOMessageSystemWillRestart == sSystemPower);
4690 unlock();
4691
4692 return ret;
4693 }
4694
4695 TUNABLE(uint32_t, dk_power_state_timeout_ms, "dk_power_state_timeout_ms", 30000);
4696
4697 IOReturn
systemPowerChange(void * target,void * refCon,UInt32 messageType,IOService * service,void * messageArgument,vm_size_t argSize)4698 IOServicePH::systemPowerChange(
4699 void * target,
4700 void * refCon,
4701 UInt32 messageType, IOService * service,
4702 void * messageArgument, vm_size_t argSize)
4703 {
4704 IOReturn ret;
4705 IOUserServer * us;
4706 IOPMSystemCapabilityChangeParameters * params;
4707 AbsoluteTime deadline;
4708
4709 us = NULL;
4710
4711 switch (messageType) {
4712 case kIOMessageSystemCapabilityChange:
4713
4714 params = (typeof params)messageArgument;
4715
4716 if (kIODKLogPM & gIODKDebug) {
4717 IOLog("IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4718 params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4719 params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4720 params->fromCapabilities,
4721 params->toCapabilities);
4722 }
4723
4724 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4725 (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4726 ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4727 lock();
4728 DKLOG("arming ack timer, %u ms\n", dk_power_state_timeout_ms);
4729 clock_interval_to_deadline(dk_power_state_timeout_ms, kMillisecondScale, &deadline);
4730 fSystemOff = true;
4731 fSystemPowerAckRef = params->notifyRef;
4732 fSystemPowerAckTo = service;
4733 thread_call_enter_delayed(fUserServerAckTimer, deadline);
4734 unlock();
4735
4736 matchingEnd(NULL);
4737
4738 params->maxWaitForReply = dk_power_state_timeout_ms * 2 * 1000;
4739 ret = kIOReturnSuccess;
4740 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4741 ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4742 (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4743 lock();
4744 fSystemOff = false;
4745 unlock();
4746
4747 matchingEnd(NULL);
4748
4749 params->maxWaitForReply = 0;
4750 ret = kIOReturnSuccess;
4751 } else {
4752 params->maxWaitForReply = 0;
4753 ret = kIOReturnSuccess;
4754 }
4755 break;
4756
4757 default:
4758 ret = kIOReturnUnsupported;
4759 break;
4760 }
4761
4762 return ret;
4763 }
4764
4765 bool
checkPMReady(void)4766 IOServicePH::checkPMReady(void)
4767 {
4768 bool __block ready = true;
4769
4770 lock();
4771 fUserServers->iterateObjects(^bool (OSObject *obj) {
4772 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4773 if (us) {
4774 if (!us->checkPMReady()) {
4775 ready = false;
4776 return true;
4777 }
4778 }
4779 return false;
4780 });
4781 unlock();
4782
4783 return ready;
4784 }
4785
4786 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4787
4788 /*
4789 * Start a previously attached & probed instance,
4790 * called on exporting object instance
4791 */
4792
4793 bool
startCandidate(IOService * service)4794 IOService::startCandidate( IOService * service )
4795 {
4796 bool ok;
4797 OSObject * obj;
4798 OSObject * prop;
4799 IOUserServer * userServer;
4800 bool ph;
4801
4802 userServer = NULL;
4803 obj = service->copyProperty(gIOUserServerNameKey);
4804
4805 if (obj && (this == gIOResources)) {
4806 ok = false;
4807 } else {
4808 ok = service->attach( this );
4809 }
4810 if (!ok) {
4811 OSSafeReleaseNULL(obj);
4812 return false;
4813 }
4814
4815 if ((this != gIOResources) && (this != gIOUserResources)) {
4816 // stall for any nub resources
4817 checkResources();
4818 // stall for any driver resources
4819 service->checkResources();
4820 }
4821 ph = false;
4822 {
4823 OSString * bundleID;
4824 OSString * serverName;
4825 OSString * str;
4826 const OSSymbol * sym;
4827 OSNumber * serverTag;
4828 uint64_t entryID;
4829 IOUserServerCheckInToken * token;
4830 OSData * serverDUI;
4831
4832 if ((serverName = OSDynamicCast(OSString, obj))) {
4833 obj = service->copyProperty(gIOModuleIdentifierKey);
4834 bundleID = OSDynamicCast(OSString, obj);
4835 entryID = service->getRegistryEntryID();
4836 serverTag = OSNumber::withNumber(entryID, 64);
4837 token = NULL;
4838
4839 if (kIODKDisableDextLaunch & gIODKDebug) {
4840 DKLOG(DKS " dext launches are disabled \n", DKN(service));
4841 service->detach(this);
4842 OSSafeReleaseNULL(serverName);
4843 OSSafeReleaseNULL(obj);
4844 OSSafeReleaseNULL(serverTag);
4845 return false;
4846 }
4847
4848 if (gIOKitWillTerminate) {
4849 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4850 service->detach(this);
4851 OSSafeReleaseNULL(serverName);
4852 OSSafeReleaseNULL(obj);
4853 OSSafeReleaseNULL(serverTag);
4854 return false;
4855 }
4856
4857 ph = IOServicePH::matchingStart(this);
4858 if (!ph) {
4859 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4860 service->setProperty(gIOServiceMatchDeferredKey, kOSBooleanTrue);
4861 service->detach(this);
4862 OSSafeReleaseNULL(serverName);
4863 OSSafeReleaseNULL(obj);
4864 OSSafeReleaseNULL(serverTag);
4865 return false;
4866 }
4867
4868 prop = service->copyProperty(gIOUserClassKey);
4869 str = OSDynamicCast(OSString, prop);
4870 if (str) {
4871 service->setName(str);
4872 }
4873 OSSafeReleaseNULL(prop);
4874
4875 sym = OSSymbol::withString(serverName);
4876 bool reuse = service->propertyExists(gIOUserServerOneProcessKey);
4877 bool reuseRequired = false;
4878 if (propertyExists(gIOUserServerNameKey)) {
4879 reuseRequired = reuse && service->propertyHasValue(gIOUserServerNameKey, getProperty(gIOUserServerNameKey));
4880 }
4881 serverDUI = OSDynamicCast(OSData, service->getProperty(kOSBundleDextUniqueIdentifierKey));
4882 userServer = IOUserServer::launchUserServer(bundleID, sym, serverTag, reuse, &token, serverDUI);
4883 OSSafeReleaseNULL(sym);
4884 OSSafeReleaseNULL(serverTag);
4885 OSSafeReleaseNULL(serverName);
4886 if (userServer) {
4887 DKLOG(DKS " using existing server " DKS "\n", DKN(service), DKN(userServer));
4888 } else if (!reuseRequired && token != NULL) {
4889 const OSSymbol * tokenServerName = token->copyServerName();
4890 OSNumber * tokenServerTag = token->copyServerTag();
4891 assert(tokenServerName && tokenServerTag);
4892 DKLOG(DKS " waiting for server %s-%llx\n", DKN(service), tokenServerName->getCStringNoCopy(), tokenServerTag->unsigned64BitValue());
4893 userServer = __WAITING_FOR_USER_SERVER__(token);
4894 OSSafeReleaseNULL(tokenServerName);
4895 OSSafeReleaseNULL(tokenServerTag);
4896 } else if (reuseRequired) {
4897 DKLOG(DKS " failed to reuse server\n", DKN(service));
4898 } else {
4899 DKLOG(DKS " failed to launch server\n", DKN(service));
4900 }
4901
4902
4903 if (!userServer) {
4904 service->detach(this);
4905 IOServicePH::matchingEnd(this);
4906 OSSafeReleaseNULL(obj);
4907
4908 if (token != NULL) {
4909 DKLOG(DKS " user server timeout\n", DKN(service));
4910 #if DEVELOPMENT || DEBUG
4911 driverkit_checkin_timed_out = mach_absolute_time();
4912 #endif
4913 }
4914
4915 OSSafeReleaseNULL(token);
4916 return false;
4917 }
4918
4919 if (token && !(kIODKDisableCheckInTokenVerification & gIODKDebug)) {
4920 if (!userServer->serviceMatchesCheckInToken(token)) {
4921 OSSafeReleaseNULL(token);
4922 service->detach(this);
4923 IOServicePH::matchingEnd(this);
4924 OSSafeReleaseNULL(obj);
4925 userServer->exit("Check In Token verification failed");
4926 userServer->release();
4927 return false;
4928 }
4929 }
4930 OSSafeReleaseNULL(token);
4931 OSSafeReleaseNULL(obj);
4932
4933 if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4934 if (!userServer->checkEntitlements(this, service)) {
4935 service->detach(this);
4936 IOServicePH::matchingEnd(this);
4937 userServer->exit("Entitlements check failed");
4938 userServer->release();
4939 return false;
4940 }
4941 }
4942 #if !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG)
4943 // Prevent third party drivers from matching IOUserResources/IOResources, except when signed for development
4944 if (!userServer->isPlatformDriver() &&
4945 userServer->getCSValidationCategory() != CS_VALIDATION_CATEGORY_DEVELOPMENT
4946 && (this == gIOUserResources || this == gIOResources)) {
4947 service->detach(this);
4948 IOServicePH::matchingEnd(this);
4949 userServer->exit("Third party driver may only match real hardware");
4950 userServer->release();
4951 return false;
4952 }
4953 #endif /* !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG) */
4954
4955 userServer->serviceAttach(service, this);
4956 } else {
4957 OSSafeReleaseNULL(obj);
4958 }
4959 }
4960
4961 AbsoluteTime startTime;
4962 AbsoluteTime endTime;
4963 UInt64 nano;
4964 bool recordTime = (kIOLogStart & gIOKitDebug) != 0;
4965
4966 if (recordTime) {
4967 clock_get_uptime(&startTime);
4968 }
4969
4970 ok = service->start(this);
4971
4972 if (recordTime) {
4973 clock_get_uptime(&endTime);
4974
4975 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4976 SUB_ABSOLUTETIME(&endTime, &startTime);
4977 absolutetime_to_nanoseconds(endTime, &nano);
4978 if (nano > 500000000ULL) {
4979 IOLog("%s[0x%qx]::start took %ld ms\n", service->getName(), service->getRegistryEntryID(), (long)(UInt32)(nano / 1000000ULL));
4980 }
4981 }
4982 }
4983 if (userServer) {
4984 userServer->serviceStarted(service, this, ok);
4985 userServer->release();
4986 }
4987
4988 if (ok) {
4989 IOInstallServiceSleepPlatformActions(service);
4990 #if 00
4991 if (!strcmp("XHC1", getName())) {
4992 service->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
4993 }
4994 #endif
4995 }
4996
4997 if (!ok) {
4998 service->detach( this );
4999 }
5000
5001 if (ph) {
5002 IOServicePH::matchingEnd(this);
5003 }
5004
5005 return ok;
5006 }
5007
5008 void
publishResource(const char * key,OSObject * value)5009 IOService::publishResource( const char * key, OSObject * value )
5010 {
5011 const OSSymbol * sym;
5012
5013 if ((sym = OSSymbol::withCString( key))) {
5014 publishResource( sym, value);
5015 sym->release();
5016 }
5017 }
5018
5019 void
publishResource(const OSSymbol * key,OSObject * value)5020 IOService::publishResource( const OSSymbol * key, OSObject * value )
5021 {
5022 if (NULL == value) {
5023 value = (OSObject *) gIOServiceKey;
5024 }
5025
5026 gIOResources->setProperty( key, value);
5027
5028 if (IORecursiveLockHaveLock( gNotificationLock)) {
5029 return;
5030 }
5031
5032 gIOResourceGenerationCount++;
5033 gIOResources->registerService();
5034 }
5035
5036 void
publishUserResource(const OSSymbol * key,OSObject * value)5037 IOService::publishUserResource( const OSSymbol * key, OSObject * value )
5038 {
5039 if (NULL == value) {
5040 value = (OSObject *) gIOServiceKey;
5041 }
5042
5043 gIOUserResources->setProperty( key, value);
5044
5045 if (IORecursiveLockHaveLock( gNotificationLock)) {
5046 return;
5047 }
5048
5049 gIOResourceGenerationCount++;
5050 gIOUserResources->registerService();
5051 }
5052
5053 bool
addNeededResource(const char * key)5054 IOService::addNeededResource( const char * key )
5055 {
5056 OSObject * resourcesProp;
5057 OSSet * set;
5058 OSString * newKey;
5059 bool ret;
5060
5061 resourcesProp = copyProperty( gIOResourceMatchKey );
5062 if (!resourcesProp) {
5063 return false;
5064 }
5065
5066 newKey = OSString::withCString( key );
5067 if (!newKey) {
5068 resourcesProp->release();
5069 return false;
5070 }
5071
5072 set = OSDynamicCast( OSSet, resourcesProp );
5073 if (!set) {
5074 set = OSSet::withCapacity( 1 );
5075 set->setObject( resourcesProp );
5076 } else {
5077 set->retain();
5078 }
5079
5080 set->setObject( newKey );
5081 newKey->release();
5082 ret = setProperty( gIOResourceMatchKey, set );
5083 set->release();
5084 resourcesProp->release();
5085
5086 return ret;
5087 }
5088
5089 bool
checkResource(OSObject * matching)5090 IOService::checkResource( OSObject * matching )
5091 {
5092 OSString * str;
5093 OSDictionary * table;
5094
5095 if ((str = OSDynamicCast( OSString, matching ))) {
5096 if (gIOResources->getProperty( str )) {
5097 return true;
5098 }
5099 }
5100
5101 if (str) {
5102 table = resourceMatching( str );
5103 } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
5104 table->retain();
5105 } else {
5106 IOLog("%s: Can't match using: %s\n", getName(),
5107 matching->getMetaClass()->getClassName());
5108 /* false would stall forever */
5109 return true;
5110 }
5111
5112 if (gIOKitDebug & kIOLogConfig) {
5113 LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
5114 }
5115
5116 waitForService( table );
5117
5118 if (gIOKitDebug & kIOLogConfig) {
5119 LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5120 }
5121
5122 return true;
5123 }
5124
5125 bool
checkResources(void)5126 IOService::checkResources( void )
5127 {
5128 OSObject * resourcesProp;
5129 OSSet * set;
5130 OSObject * obj;
5131 OSIterator * iter;
5132 bool ok;
5133
5134 resourcesProp = copyProperty( gIOResourceMatchKey );
5135 if (NULL == resourcesProp) {
5136 return true;
5137 }
5138
5139 if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
5140 iter = OSCollectionIterator::withCollection( set );
5141 ok = (NULL != iter);
5142 while (ok && (obj = iter->getNextObject())) {
5143 ok = checkResource( obj );
5144 }
5145 if (iter) {
5146 iter->release();
5147 }
5148 } else {
5149 ok = checkResource( resourcesProp );
5150 }
5151
5152 OSSafeReleaseNULL(resourcesProp);
5153
5154 return ok;
5155 }
5156
5157
5158 void
configThread(const char * name)5159 _IOConfigThread::configThread( const char * name )
5160 {
5161 _IOConfigThread * inst;
5162
5163 do {
5164 if (!(inst = new _IOConfigThread)) {
5165 continue;
5166 }
5167 if (!inst->init()) {
5168 continue;
5169 }
5170 thread_t thread;
5171 if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &thread)) {
5172 continue;
5173 }
5174
5175 char threadName[MAXTHREADNAMESIZE];
5176 snprintf(threadName, sizeof(threadName), "IOConfigThread_'%s'", name);
5177 thread_set_thread_name(thread, threadName);
5178 thread_deallocate(thread);
5179
5180 return;
5181 } while (false);
5182
5183 if (inst) {
5184 inst->release();
5185 }
5186
5187 return;
5188 }
5189
5190 /*
5191 * To support driver replacement of boot matched drivers later in boot, drivers can
5192 * opt-in to be being terminated if a non-boot driver matches their provider, by
5193 * setting the gIOPrimaryDriverTerminateOptionsKey property. The driver providing the
5194 * root disk media may not be terminated.
5195 * IOMedia objects are hidden from user space until all drivers are available, but any
5196 * associated with the root disk must be published immediately.
5197 */
5198
5199 struct FindRootMediaContext {
5200 OSArray * services;
5201 IOService * parent;
5202 };
5203
5204 bool
hasParent(IOService * parent)5205 IOService::hasParent(IOService * parent)
5206 {
5207 IOService * service;
5208
5209 for (service = this;
5210 service && (service != parent);
5211 service = service->getProvider()) {
5212 }
5213
5214 return service != NULL;
5215 }
5216
5217 bool
publishHiddenMediaApplier(const OSObject * entry,void * context)5218 IOService::publishHiddenMediaApplier(const OSObject * entry, void * context)
5219 {
5220 FindRootMediaContext * ctx = (typeof(ctx))context;
5221 IOService * service = (typeof(service))entry;
5222
5223 do {
5224 if (ctx->parent && !service->hasParent(ctx->parent)) {
5225 break;
5226 }
5227 if (ctx->services) {
5228 ctx->services->setObject(service);
5229 } else {
5230 ctx->services = OSArray::withObjects((const OSObject **) &service, 1);
5231 assert(ctx->services);
5232 }
5233 } while (false);
5234
5235 return false;
5236 }
5237
5238 // publish to user space any hidden IOMedia under the 'parent' object, or all
5239 // if 'parent' is NULL
5240
5241 void
publishHiddenMedia(IOService * parent)5242 IOService::publishHiddenMedia(IOService * parent)
5243 {
5244 const OSMetaClass * iomediaClass;
5245 bool wasHiding;
5246
5247 iomediaClass = OSMetaClass::getMetaClassWithName(gIOMediaKey);
5248 assert(iomediaClass);
5249
5250 LOCKWRITENOTIFY();
5251 wasHiding = gIOServiceHideIOMedia;
5252 if (wasHiding && !parent) {
5253 gIOServiceHideIOMedia = false;
5254 }
5255 UNLOCKNOTIFY();
5256
5257 FindRootMediaContext ctx = { .services = NULL, .parent = parent };
5258
5259 if (wasHiding) {
5260 iomediaClass->applyToInstances(publishHiddenMediaApplier, &ctx);
5261 }
5262 if (ctx.services) {
5263 unsigned int idx, notiIdx;
5264 IOService * service;
5265 OSArray * notifiers[3] = {};
5266
5267 for (idx = 0; (service = (IOService *) ctx.services->getObject(idx)); idx++) {
5268 service->lockForArbitration(true);
5269 if (!(kIOServiceUserInvisibleMatchState & service->__state[0])) {
5270 service->unlockForArbitration();
5271 continue;
5272 }
5273 service->__state[0] &= ~kIOServiceUserInvisibleMatchState;
5274 service->__state[1] |= kIOServiceUserUnhidden;
5275 notifiers[0] = service->copyNotifiers(gIOFirstPublishNotification, 0, 0xffffffff);
5276 if (kIOServiceMatchedState & service->__state[0]) {
5277 notifiers[1] = service->copyNotifiers(gIOMatchedNotification, 0, 0xffffffff);
5278 }
5279 if (kIOServiceFirstMatchState & service->__state[0]) {
5280 notifiers[2] = service->copyNotifiers(gIOFirstMatchNotification, 0, 0xffffffff);
5281 }
5282 service->unlockForArbitration();
5283 for (notiIdx = 0; notiIdx < 3; notiIdx++) {
5284 service->invokeNotifiers(¬ifiers[notiIdx]);
5285 }
5286 }
5287 OSSafeReleaseNULL(ctx.services);
5288 }
5289 }
5290
5291 // Find the block storage driver providing the root disk, or NULL if not booting from
5292 // a block device
5293
5294 void
setRootMedia(IOService * root)5295 IOService::setRootMedia(IOService * root)
5296 {
5297 const OSMetaClass * ioblockstoragedriverClass;
5298 bool unhide;
5299
5300 ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey);
5301 assert(ioblockstoragedriverClass);
5302
5303 while (root) {
5304 if (root->metaCast(ioblockstoragedriverClass)) {
5305 break;
5306 }
5307 root = root->getProvider();
5308 }
5309
5310 LOCKWRITENOTIFY();
5311 unhide = (kIOServiceRootMediaParentInvalid == gIOServiceRootMediaParent);
5312 if (unhide) {
5313 gIOServiceRootMediaParent = root;
5314 }
5315 UNLOCKNOTIFY();
5316
5317 if (unhide) {
5318 publishHiddenMedia(root);
5319 }
5320 }
5321
5322 // Check if the driver may be terminated when a later driver could be used instead
5323
5324 bool
canTerminateForReplacement(IOService * client)5325 IOService::canTerminateForReplacement(IOService * client)
5326 {
5327 IOService * parent;
5328
5329 assert(kIOServiceRootMediaParentInvalid != gIOServiceRootMediaParent);
5330
5331 if (!client->propertyExists(gIOPrimaryDriverTerminateOptionsKey)) {
5332 return false;
5333 }
5334 if (!gIOServiceRootMediaParent) {
5335 return false;
5336 }
5337 parent = gIOServiceRootMediaParent;
5338 while (parent && (parent != client)) {
5339 parent = parent->getProvider();
5340 }
5341 if (parent) {
5342 IOLog("Can't replace primary matched driver on root media %s-0x%qx\n",
5343 client->getName(), client->getRegistryEntryID());
5344 return false;
5345 }
5346 return true;
5347 }
5348
5349 void
doServiceMatch(IOOptionBits options)5350 IOService::doServiceMatch( IOOptionBits options )
5351 {
5352 _IOServiceNotifier * notify;
5353 OSIterator * iter;
5354 OSOrderedSet * matches;
5355 OSArray * resourceKeys = NULL;
5356 SInt32 catalogGeneration;
5357 bool keepGuessing = true;
5358 bool reRegistered = true;
5359 bool didRegister;
5360 OSArray * notifiers[2] = {NULL};
5361
5362 // job->nub->deliverNotification( gIOPublishNotification,
5363 // kIOServiceRegisteredState, 0xffffffff );
5364
5365 while (keepGuessing) {
5366 matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
5367 // the matches list should always be created by findDrivers()
5368 if (matches) {
5369 lockForArbitration();
5370 if (0 == (__state[0] & kIOServiceFirstPublishState)) {
5371 getMetaClass()->addInstance(this);
5372 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
5373 kIOServiceFirstPublishState, 0xffffffff );
5374 }
5375 LOCKREADNOTIFY();
5376 __state[1] &= ~kIOServiceNeedConfigState;
5377 __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
5378 didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
5379 __state[0] |= kIOServiceRegisteredState;
5380
5381 if (gIOServiceHideIOMedia
5382 && metaCast(gIOMediaKey)
5383 && !(kIOServiceUserUnhidden & __state[1])
5384 && gIOServiceRootMediaParent
5385 && !hasParent(gIOServiceRootMediaParent)
5386 && propertyExists(gIOPrimaryDriverTerminateOptionsKey, gIOServicePlane)) {
5387 __state[0] |= kIOServiceUserInvisibleMatchState;
5388 }
5389
5390 keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
5391 if (reRegistered && keepGuessing) {
5392 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
5393 gNotifications->getObject( gIOPublishNotification ));
5394 if (iter) {
5395 while ((notify = (_IOServiceNotifier *)
5396 iter->getNextObject())) {
5397 if (matchPassive(notify->matching, 0)
5398 && (kIOServiceNotifyEnable & notify->state)) {
5399 matches->setObject( notify );
5400 }
5401 }
5402 iter->release();
5403 }
5404 }
5405
5406 UNLOCKNOTIFY();
5407 unlockForArbitration();
5408 invokeNotifiers(¬ifiers[0]);
5409 if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
5410 if ((this == gIOResources) || (this == gIOUserResources)) {
5411 if (resourceKeys) {
5412 resourceKeys->release();
5413 }
5414 resourceKeys = copyPropertyKeys();
5415 }
5416 probeCandidates( matches );
5417 } else {
5418 matches->release();
5419 }
5420 }
5421
5422 lockForArbitration();
5423 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
5424 keepGuessing =
5425 (reRegistered || (catalogGeneration !=
5426 gIOCatalogue->getGenerationCount()))
5427 && (0 == (__state[0] & kIOServiceInactiveState));
5428
5429 if (keepGuessing) {
5430 unlockForArbitration();
5431 }
5432 }
5433
5434 if ((0 == (__state[0] & kIOServiceInactiveState))
5435 && (0 == (__state[1] & kIOServiceModuleStallState))) {
5436 if (resourceKeys) {
5437 setProperty(gIOResourceMatchedKey, resourceKeys);
5438 }
5439
5440 notifiers[0] = copyNotifiers(gIOMatchedNotification,
5441 kIOServiceMatchedState, 0xffffffff);
5442 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
5443 notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
5444 kIOServiceFirstMatchState, 0xffffffff);
5445 }
5446 }
5447
5448 __state[1] &= ~kIOServiceConfigRunning;
5449 unlockForArbitration();
5450
5451 if (resourceKeys) {
5452 resourceKeys->release();
5453 }
5454
5455 invokeNotifiers(¬ifiers[0]);
5456 invokeNotifiers(¬ifiers[1]);
5457
5458 lockForArbitration();
5459 __state[1] &= ~kIOServiceConfigState;
5460 scheduleTerminatePhase2();
5461
5462 _adjustBusy(-1, /*unlock*/ true);
5463 // does unlockForArbitration();
5464 }
5465
5466 UInt32
_adjustBusy(SInt32 delta)5467 IOService::_adjustBusy(SInt32 delta)
5468 {
5469 return _adjustBusy(delta, false);
5470 }
5471
5472 UInt32
_adjustBusy(SInt32 delta,bool unlock)5473 IOService::_adjustBusy(SInt32 delta, bool unlock)
5474 {
5475 IOService * next;
5476 IOService * nextProvider;
5477 UInt32 count;
5478 UInt32 result;
5479 bool wasQuiet, nowQuiet, needWake;
5480
5481 next = this;
5482 result = __state[1] & kIOServiceBusyStateMask;
5483
5484 if (delta) {
5485 do {
5486 if (next != this) {
5487 next->lockForArbitration();
5488 }
5489 count = next->__state[1] & kIOServiceBusyStateMask;
5490 wasQuiet = (0 == count);
5491 if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
5492 OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), (uint32_t)count, (int)delta);
5493 } else {
5494 count += delta;
5495 }
5496 next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
5497 nowQuiet = (0 == count);
5498 needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
5499
5500 if (needWake) {
5501 next->__state[1] &= ~kIOServiceBusyWaiterState;
5502 IOLockLock( gIOServiceBusyLock );
5503 thread_wakeup((event_t) next);
5504 IOLockUnlock( gIOServiceBusyLock );
5505 }
5506 if (wasQuiet || nowQuiet) {
5507 nextProvider = next->getProvider();
5508 if (nextProvider) {
5509 nextProvider->retain();
5510 }
5511 }
5512 if ((next != this) || unlock) {
5513 next->unlockForArbitration();
5514 }
5515
5516 if ((wasQuiet || nowQuiet)) {
5517 uint64_t regID = next->getRegistryEntryID();
5518 IOServiceTrace(
5519 ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
5520 (uintptr_t) regID,
5521 (uintptr_t) (regID >> 32),
5522 (uintptr_t) next,
5523 0);
5524
5525 if (wasQuiet) {
5526 next->__timeBusy = mach_absolute_time();
5527 } else {
5528 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
5529 next->__timeBusy = 0;
5530 }
5531
5532 MessageClientsContext context;
5533
5534 context.service = next;
5535 context.type = kIOMessageServiceBusyStateChange;
5536 context.argument = (void *) wasQuiet; /*nowBusy*/
5537 context.argSize = 0;
5538
5539 applyToInterestNotifiers( next, gIOBusyInterest,
5540 &messageClientsApplier, &context );
5541
5542 #if !NO_KEXTD
5543 if (nowQuiet && (next == gIOServiceRoot)) {
5544 if (gIOServiceHideIOMedia) {
5545 publishHiddenMedia(NULL);
5546 }
5547
5548 OSKext::considerUnloads();
5549 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
5550 }
5551 #endif
5552 }
5553
5554 delta = nowQuiet ? -1 : +1;
5555 if (next != this) {
5556 next->release();
5557 }
5558 next = nextProvider;
5559 nextProvider = NULL;
5560 } while ((wasQuiet || nowQuiet) && next);
5561 }
5562
5563 return result;
5564 }
5565
5566 void
adjustBusy(SInt32 delta)5567 IOService::adjustBusy( SInt32 delta )
5568 {
5569 lockForArbitration();
5570 _adjustBusy( delta );
5571 unlockForArbitration();
5572 }
5573
5574 uint64_t
getAccumulatedBusyTime(void)5575 IOService::getAccumulatedBusyTime( void )
5576 {
5577 uint64_t accumBusy = __accumBusy;
5578 uint64_t timeBusy = __timeBusy;
5579 uint64_t nano;
5580
5581 do{
5582 accumBusy = __accumBusy;
5583 timeBusy = __timeBusy;
5584 if (timeBusy) {
5585 accumBusy += mach_absolute_time() - timeBusy;
5586 }
5587 }while (timeBusy != __timeBusy);
5588
5589 absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
5590
5591 return nano;
5592 }
5593
5594 UInt32
getBusyState(void)5595 IOService::getBusyState( void )
5596 {
5597 return __state[1] & kIOServiceBusyStateMask;
5598 }
5599
5600 IOReturn
waitForState(UInt32 mask,UInt32 value,mach_timespec_t * timeout)5601 IOService::waitForState( UInt32 mask, UInt32 value,
5602 mach_timespec_t * timeout )
5603 {
5604 panic("waitForState");
5605 return kIOReturnUnsupported;
5606 }
5607
5608 IOReturn
waitForState(UInt32 mask,UInt32 value,uint64_t timeout)5609 IOService::waitForState( UInt32 mask, UInt32 value,
5610 uint64_t timeout )
5611 {
5612 bool wait, done;
5613 int waitResult = THREAD_AWAKENED;
5614 bool computeDeadline = true;
5615 AbsoluteTime abstime;
5616
5617 do {
5618 lockForArbitration();
5619 IOLockLock( gIOServiceBusyLock );
5620 done = (value == (__state[1] & mask));
5621 wait = (waitResult != THREAD_TIMED_OUT) && !done;
5622 if (wait) {
5623 __state[1] |= kIOServiceBusyWaiterState;
5624 unlockForArbitration();
5625 if (timeout != UINT64_MAX) {
5626 if (computeDeadline) {
5627 AbsoluteTime nsinterval;
5628 nanoseconds_to_absolutetime(timeout, &nsinterval );
5629 clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
5630 computeDeadline = false;
5631 }
5632 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
5633 } else {
5634 assert_wait((event_t)this, THREAD_UNINT );
5635 }
5636 } else {
5637 unlockForArbitration();
5638 }
5639 IOLockUnlock( gIOServiceBusyLock );
5640 if (wait) {
5641 waitResult = thread_block(THREAD_CONTINUE_NULL);
5642 }
5643 } while (wait);
5644
5645 if (done) {
5646 return kIOReturnSuccess;
5647 } else {
5648 return kIOReturnTimeout;
5649 }
5650 }
5651
5652 IOReturn
waitQuietWithOptions(uint64_t timeout,IOOptionBits options)5653 IOService::waitQuietWithOptions( uint64_t timeout, IOOptionBits options )
5654 {
5655 IOReturn ret;
5656 uint32_t loops, timeoutExtensions;
5657 char * busyEntriesString = NULL;
5658 char * panicString = NULL;
5659 size_t busyEntriesStringLen;
5660 size_t panicStringLen;
5661 uint64_t time;
5662 uint64_t nano;
5663 bool pendingRequests;
5664 bool registryRootBusy;
5665 bool multipleEntries;
5666 bool dopanic = false;
5667
5668 enum { kIOServiceBusyTimeoutExtensionsMax = 8 };
5669 #if KASAN
5670 /*
5671 * On kasan kernels, everything takes longer, so double the number of
5672 * timeout extensions. This should help with issues like 41259215
5673 * where WindowServer was timing out waiting for kextd to get all the
5674 * kasan kexts loaded and started.
5675 *
5676 * On legacy/x86 systems give a bit more time since we may be
5677 * booting from a HDD.
5678 */
5679 enum { kTimeoutExtensions = 8 };
5680 #define WITH_IOWAITQUIET_EXTENSIONS 1
5681 #elif defined(__x86_64__)
5682 enum { kTimeoutExtensions = 4 };
5683 #define WITH_IOWAITQUIET_EXTENSIONS 1
5684 #elif defined(XNU_TARGET_OS_OSX)
5685 enum { kTimeoutExtensions = 1 };
5686 #define WITH_IOWAITQUIET_EXTENSIONS 1
5687 #else
5688 enum { kTimeoutExtensions = 1 };
5689 #define WITH_IOWAITQUIET_EXTENSIONS 0
5690 #endif
5691
5692 timeoutExtensions = kTimeoutExtensions;
5693 time = mach_absolute_time();
5694 pendingRequests = false;
5695 for (loops = 0; loops < timeoutExtensions; loops++) {
5696 ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
5697
5698 if (loops && (kIOReturnSuccess == ret)) {
5699 time = mach_absolute_time() - time;
5700 absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
5701 IOLog("busy extended ok[%d], (%llds, %llds)\n",
5702 loops, timeout / 1000000000ULL, nano / 1000000000ULL);
5703 break;
5704 } else if (kIOReturnTimeout != ret) {
5705 break;
5706 } else if (timeout < (41ull * NSEC_PER_SEC)) {
5707 break;
5708 }
5709
5710 {
5711 IORegistryIterator * iter;
5712 OSOrderedSet * set;
5713 OSOrderedSet * leaves;
5714 IOService * next;
5715 IOService * nextParent;
5716 char * s;
5717 size_t l;
5718 size_t busyEntriesStringRemaining;
5719
5720 busyEntriesStringLen = 256;
5721 panicStringLen = 256;
5722 if (!busyEntriesString) {
5723 busyEntriesString = IONewZeroData(char, busyEntriesStringLen);
5724 assert(busyEntriesString != NULL);
5725 }
5726 if (!panicString) {
5727 panicString = IONewZeroData(char, panicStringLen);
5728 assert(panicString != NULL);
5729 }
5730
5731 set = NULL;
5732 pendingRequests = OSKext::pendingIOKitDaemonRequests();
5733 iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
5734 leaves = OSOrderedSet::withCapacity(4);
5735 if (iter) {
5736 set = iter->iterateAll();
5737 }
5738 if (leaves && set) {
5739 busyEntriesString[0] = panicString[0] = 0;
5740 set->setObject(this);
5741 while ((next = (IOService *) set->getLastObject())) {
5742 if (next->getBusyState()) {
5743 if (kIOServiceModuleStallState & next->__state[1]) {
5744 pendingRequests = true;
5745 }
5746 #if defined(XNU_TARGET_OS_OSX)
5747 OSObject * prop;
5748 if ((prop = next->copyProperty(kIOServiceBusyTimeoutExtensionsKey, gIOServicePlane))) {
5749 OSNumber * num;
5750 uint32_t value;
5751 if ((num = OSDynamicCast(OSNumber, prop))) {
5752 value = num->unsigned32BitValue();
5753 if (value
5754 && (value <= kIOServiceBusyTimeoutExtensionsMax)
5755 && (value > timeoutExtensions)) {
5756 timeoutExtensions = value;
5757 }
5758 }
5759 OSSafeReleaseNULL(prop);
5760 }
5761 #endif /* defined(XNU_TARGET_OS_OSX) */
5762 leaves->setObject(next);
5763 nextParent = next;
5764 while ((nextParent = nextParent->getProvider())) {
5765 set->removeObject(nextParent);
5766 leaves->removeObject(nextParent);
5767 }
5768 }
5769 set->removeObject(next);
5770 }
5771 registryRootBusy = leaves->getCount() == 1 && leaves->getObject(0) == getServiceRoot();
5772 multipleEntries = leaves->getCount() > 1;
5773 s = busyEntriesString;
5774 busyEntriesStringRemaining = busyEntriesStringLen;
5775
5776 if (registryRootBusy) {
5777 snprintf(s, busyEntriesStringRemaining, "registry root held busy, " kIOKitDaemonName " %s checked in", OSKext::iokitDaemonActive() ? "has" : "has not");
5778 } else {
5779 while ((next = (IOService *) leaves->getLastObject())) {
5780 l = snprintf(s, busyEntriesStringRemaining, "%s'%s' (%x,%x)", ((s == busyEntriesString) ? "" : ", "), next->getName(), (uint32_t)next->__state[0], (uint32_t)next->__state[1]);
5781 if (l >= busyEntriesStringRemaining) {
5782 break;
5783 }
5784 s += l;
5785 busyEntriesStringRemaining -= l;
5786 leaves->removeObject(next);
5787 }
5788 }
5789 }
5790 OSSafeReleaseNULL(leaves);
5791 OSSafeReleaseNULL(set);
5792 OSSafeReleaseNULL(iter);
5793 }
5794
5795 dopanic = (kIOWaitQuietPanics & gIOKitDebug) && (options & kIOWaitQuietPanicOnFailure) && !gIOKitWillTerminate;
5796 #if WITH_IOWAITQUIET_EXTENSIONS
5797 dopanic = (dopanic && (loops >= (timeoutExtensions - 1)));
5798 #endif
5799 assert(panicString != NULL);
5800 if (multipleEntries) {
5801 snprintf(panicString, panicStringLen,
5802 "%s[%d], (%llds): multiple entries holding the registry busy, IOKit termination queue depth %u: %s",
5803 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5804 loops, timeout / 1000000000ULL,
5805 (uint32_t)gIOTerminateWork,
5806 busyEntriesString ? busyEntriesString : "");
5807 } else {
5808 snprintf(panicString, panicStringLen,
5809 "%s[%d], (%llds): %s",
5810 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5811 loops, timeout / 1000000000ULL,
5812 busyEntriesString ? busyEntriesString : "");
5813 }
5814
5815 IOLog("%s\n", panicString);
5816 if (dopanic) {
5817 panic("%s", panicString);
5818 } else if (!loops) {
5819 getPMRootDomain()->startSpinDump(1);
5820 }
5821 }
5822
5823 if (busyEntriesString) {
5824 IODeleteData(busyEntriesString, char, busyEntriesStringLen);
5825 }
5826 if (panicString) {
5827 IODeleteData(panicString, char, panicStringLen);
5828 }
5829
5830 return ret;
5831 }
5832
5833 IOReturn
waitQuiet(uint64_t timeout)5834 IOService::waitQuiet( uint64_t timeout )
5835 {
5836 return waitQuietWithOptions(timeout);
5837 }
5838
5839 IOReturn
waitQuiet(mach_timespec_t * timeout)5840 IOService::waitQuiet( mach_timespec_t * timeout )
5841 {
5842 uint64_t timeoutNS;
5843
5844 if (timeout) {
5845 timeoutNS = timeout->tv_sec;
5846 timeoutNS *= kSecondScale;
5847 timeoutNS += timeout->tv_nsec;
5848 } else {
5849 timeoutNS = UINT64_MAX;
5850 }
5851
5852 return waitQuiet(timeoutNS);
5853 }
5854
5855 bool
serializeProperties(OSSerialize * s) const5856 IOService::serializeProperties( OSSerialize * s ) const
5857 {
5858 #if 0
5859 ((IOService *)this)->setProperty(((IOService *)this)->__state,
5860 sizeof(__state), "__state");
5861 #endif
5862 return super::serializeProperties(s);
5863 }
5864
5865 void
resetRematchProperties()5866 IOService::resetRematchProperties()
5867 {
5868 removeProperty(gIORematchCountKey);
5869 removeProperty(gIORematchPersonalityKey);
5870 }
5871
5872
5873 void
main(void * arg,wait_result_t result)5874 _IOConfigThread::main(void * arg, wait_result_t result)
5875 {
5876 _IOConfigThread * self = (_IOConfigThread *) arg;
5877 _IOServiceJob * job;
5878 IOService * nub;
5879 bool alive = true;
5880 kern_return_t kr;
5881 thread_precedence_policy_data_t precedence = { -1 };
5882
5883 kr = thread_policy_set(current_thread(),
5884 THREAD_PRECEDENCE_POLICY,
5885 (thread_policy_t) &precedence,
5886 THREAD_PRECEDENCE_POLICY_COUNT);
5887 if (KERN_SUCCESS != kr) {
5888 IOLog("thread_policy_set(%d)\n", kr);
5889 }
5890
5891 do {
5892 // randomDelay();
5893
5894 semaphore_wait( gJobsSemaphore );
5895
5896 IOTakeLock( gJobsLock );
5897 job = (_IOServiceJob *) gJobs->getFirstObject();
5898 job->retain();
5899 gJobs->removeObject(job);
5900 if (job) {
5901 gOutstandingJobs--;
5902 // gNumConfigThreads--; // we're out of service
5903 gNumWaitingThreads--; // we're out of service
5904 }
5905 IOUnlock( gJobsLock );
5906
5907 if (job) {
5908 nub = job->nub;
5909
5910 if (gIOKitDebug & kIOLogConfig) {
5911 LOG("config(%p): starting on %s, %d\n",
5912 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5913 }
5914
5915 switch (job->type) {
5916 case kMatchNubJob:
5917 nub->doServiceMatch( job->options );
5918 break;
5919
5920 default:
5921 LOG("config(%p): strange type (%d)\n",
5922 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5923 break;
5924 }
5925
5926 if (job->options & kIOServiceDextRequirePowerForMatching) {
5927 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
5928 }
5929
5930 OSSafeReleaseNULL(nub);
5931 OSSafeReleaseNULL(job);
5932
5933 IOTakeLock( gJobsLock );
5934 alive = (gOutstandingJobs > gNumWaitingThreads);
5935 if (alive) {
5936 gNumWaitingThreads++; // back in service
5937 }
5938 // gNumConfigThreads++;
5939 else {
5940 if (0 == --gNumConfigThreads) {
5941 // IOLog("MATCH IDLE\n");
5942 IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
5943 }
5944 }
5945 IOUnlock( gJobsLock );
5946 }
5947 } while (alive);
5948
5949 if (gIOKitDebug & kIOLogConfig) {
5950 LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5951 }
5952
5953 self->release();
5954 }
5955
5956 IOReturn
waitMatchIdle(UInt32 msToWait)5957 IOService::waitMatchIdle( UInt32 msToWait )
5958 {
5959 bool wait;
5960 int waitResult = THREAD_AWAKENED;
5961 bool computeDeadline = true;
5962 AbsoluteTime deadline;
5963
5964 IOLockLock( gJobsLock );
5965 do {
5966 wait = (0 != gNumConfigThreads);
5967 if (wait) {
5968 if (msToWait) {
5969 if (computeDeadline) {
5970 clock_interval_to_deadline(
5971 msToWait, kMillisecondScale, &deadline );
5972 computeDeadline = false;
5973 }
5974 waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
5975 deadline, THREAD_UNINT );
5976 } else {
5977 waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
5978 THREAD_UNINT );
5979 }
5980 }
5981 } while (wait && (waitResult != THREAD_TIMED_OUT));
5982 IOLockUnlock( gJobsLock );
5983
5984 if (waitResult == THREAD_TIMED_OUT) {
5985 return kIOReturnTimeout;
5986 } else {
5987 return kIOReturnSuccess;
5988 }
5989 }
5990
5991 void
cpusRunning(void)5992 IOService::cpusRunning(void)
5993 {
5994 gCPUsRunning = true;
5995 }
5996
5997 void
pingConfig(_IOServiceJob * job)5998 _IOServiceJob::pingConfig( _IOServiceJob * job )
5999 {
6000 int count;
6001 bool create;
6002 IOService * nub;
6003
6004 assert( job );
6005 nub = job->nub;
6006
6007 IOTakeLock( gJobsLock );
6008
6009 gOutstandingJobs++;
6010 if (nub == gIOResources) {
6011 gJobs->setFirstObject( job );
6012 } else {
6013 gJobs->setLastObject( job );
6014 }
6015
6016 count = gNumWaitingThreads;
6017 // if( gNumConfigThreads) count++;// assume we're called from a config thread
6018
6019 create = ((gOutstandingJobs > count)
6020 && ((gNumConfigThreads < gMaxConfigThreads)
6021 || (nub == gIOResources)
6022 || !gCPUsRunning));
6023 if (create) {
6024 gNumConfigThreads++;
6025 gNumWaitingThreads++;
6026 if (gNumConfigThreads > gHighNumConfigThreads) {
6027 gHighNumConfigThreads = gNumConfigThreads;
6028 }
6029 }
6030
6031 IOUnlock( gJobsLock );
6032
6033 job->release();
6034
6035 if (create) {
6036 if (gIOKitDebug & kIOLogConfig) {
6037 LOG("config(%d): creating\n", gNumConfigThreads - 1);
6038 }
6039 _IOConfigThread::configThread(nub->getName());
6040 }
6041
6042 semaphore_signal( gJobsSemaphore );
6043 }
6044
6045 struct IOServiceMatchContext {
6046 OSDictionary * table;
6047 OSObject * result;
6048 uint32_t options;
6049 uint32_t state;
6050 uint32_t count;
6051 uint32_t done;
6052 };
6053
6054 bool
instanceMatch(const OSObject * entry,void * context)6055 IOService::instanceMatch(const OSObject * entry, void * context)
6056 {
6057 IOServiceMatchContext * ctx = (typeof(ctx))context;
6058 IOService * service = (typeof(service))entry;
6059 OSDictionary * table = ctx->table;
6060 uint32_t options = ctx->options;
6061 uint32_t state = ctx->state;
6062 uint32_t done;
6063 bool match;
6064
6065 done = 0;
6066 do{
6067 match = ((state == (state & service->__state[0]))
6068 && (0 == (service->__state[0] & kIOServiceInactiveState)));
6069 if (!match) {
6070 break;
6071 }
6072
6073 match = service->matchInternal(table, options, &done);
6074 if (match) {
6075 ctx->count += table->getCount();
6076 ctx->done += done;
6077 }
6078 }while (false);
6079 if (!match) {
6080 return false;
6081 }
6082
6083 if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
6084 service->retain();
6085 ctx->result = service;
6086 return true;
6087 } else if (!ctx->result) {
6088 ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
6089 } else {
6090 ((OSSet *)ctx->result)->setObject(service);
6091 }
6092 return false;
6093 }
6094
6095 // internal - call with gNotificationLock
6096 OSObject *
copyExistingServices(OSDictionary * matching,IOOptionBits inState,IOOptionBits options)6097 IOService::copyExistingServices( OSDictionary * matching,
6098 IOOptionBits inState, IOOptionBits options )
6099 {
6100 OSObject * current = NULL;
6101 OSIterator * iter;
6102 IOService * service;
6103 OSObject * obj;
6104 OSString * str;
6105
6106 if (!matching) {
6107 return NULL;
6108 }
6109
6110 #if MATCH_DEBUG
6111 OSSerialize * s = OSSerialize::withCapacity(128);
6112 matching->serialize(s);
6113 #endif
6114
6115 if ((obj = matching->getObject(gIOProviderClassKey))
6116 && gIOResourcesKey
6117 && gIOResourcesKey->isEqualTo(obj)
6118 && (service = gIOResources)) {
6119 if ((inState == (service->__state[0] & inState))
6120 && (0 == (service->__state[0] & kIOServiceInactiveState))
6121 && service->matchPassive(matching, options)) {
6122 if (options & kIONotifyOnce) {
6123 service->retain();
6124 current = service;
6125 } else {
6126 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
6127 }
6128 }
6129 } else {
6130 IOServiceMatchContext ctx;
6131
6132 options |= kIOServiceClassDone;
6133 ctx.table = matching;
6134 ctx.state = inState;
6135 ctx.count = 0;
6136 ctx.done = 0;
6137 ctx.options = options;
6138 ctx.result = NULL;
6139
6140 if ((str = OSDynamicCast(OSString, obj))) {
6141 const OSSymbol * sym = OSSymbol::withString(str);
6142 OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
6143 sym->release();
6144 } else {
6145 IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
6146 }
6147
6148 if (((!(options & kIONotifyOnce) || !ctx.result))
6149 && matching->getObject(gIOCompatibilityMatchKey)) {
6150 IOServiceCompatibility::gMetaClass.applyToInstances(instanceMatch, &ctx);
6151 }
6152
6153 current = ctx.result;
6154 options |= kIOServiceInternalDone;
6155 if (current && (ctx.done != ctx.count)) {
6156 OSSet * source = OSDynamicCast(OSSet, current);
6157 current = NULL;
6158 while ((service = (IOService *) source->getAnyObject())) {
6159 if (service->matchPassive(matching, options)) {
6160 if (options & kIONotifyOnce) {
6161 service->retain();
6162 current = service;
6163 break;
6164 }
6165 if (current) {
6166 ((OSSet *)current)->setObject( service );
6167 } else {
6168 current = OSSet::withObjects(
6169 (const OSObject **) &service, 1, 1 );
6170 }
6171 }
6172 source->removeObject(service);
6173 }
6174 source->release();
6175 }
6176 }
6177
6178 #if MATCH_DEBUG
6179 {
6180 OSObject * _current = 0;
6181
6182 iter = IORegistryIterator::iterateOver( gIOServicePlane,
6183 kIORegistryIterateRecursively );
6184 if (iter) {
6185 do {
6186 iter->reset();
6187 while ((service = (IOService *) iter->getNextObject())) {
6188 if ((inState == (service->__state[0] & inState))
6189 && (0 == (service->__state[0] & kIOServiceInactiveState))
6190 && service->matchPassive(matching, 0)) {
6191 if (options & kIONotifyOnce) {
6192 service->retain();
6193 _current = service;
6194 break;
6195 }
6196 if (_current) {
6197 ((OSSet *)_current)->setObject( service );
6198 } else {
6199 _current = OSSet::withObjects(
6200 (const OSObject **) &service, 1, 1 );
6201 }
6202 }
6203 }
6204 } while (!service && !iter->isValid());
6205 iter->release();
6206 }
6207
6208 if (((current != 0) != (_current != 0))
6209 || (current && _current && !current->isEqualTo(_current))) {
6210 OSSerialize * s1 = OSSerialize::withCapacity(128);
6211 OSSerialize * s2 = OSSerialize::withCapacity(128);
6212 current->serialize(s1);
6213 _current->serialize(s2);
6214 kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
6215 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
6216 s1->release();
6217 s2->release();
6218 }
6219
6220 if (_current) {
6221 _current->release();
6222 }
6223 }
6224
6225 s->release();
6226 #endif
6227
6228 if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
6229 iter = OSCollectionIterator::withCollection((OSSet *)current );
6230 current->release();
6231 current = iter;
6232 }
6233
6234 return current;
6235 }
6236
6237 // public version
6238 OSIterator *
getMatchingServices(OSDictionary * matching)6239 IOService::getMatchingServices( OSDictionary * matching )
6240 {
6241 OSIterator * iter;
6242
6243 // is a lock even needed?
6244 LOCKWRITENOTIFY();
6245
6246 iter = (OSIterator *) copyExistingServices( matching,
6247 kIOServiceMatchedState );
6248
6249 UNLOCKNOTIFY();
6250
6251 return iter;
6252 }
6253
6254 IOService *
copyMatchingService(OSDictionary * matching)6255 IOService::copyMatchingService( OSDictionary * matching )
6256 {
6257 IOService * service;
6258
6259 // is a lock even needed?
6260 LOCKWRITENOTIFY();
6261
6262 service = (IOService *) copyExistingServices( matching,
6263 kIOServiceMatchedState, kIONotifyOnce );
6264
6265 UNLOCKNOTIFY();
6266
6267 return service;
6268 }
6269
6270 struct _IOServiceMatchingNotificationHandlerRef {
6271 IOServiceNotificationHandler handler;
6272 void * ref;
6273 };
6274
6275 static bool
_IOServiceMatchingNotificationHandler(void * target,void * refCon,IOService * newService,IONotifier * notifier)6276 _IOServiceMatchingNotificationHandler( void * target, void * refCon,
6277 IOService * newService,
6278 IONotifier * notifier )
6279 {
6280 return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
6281 }
6282
6283 // internal - call with gNotificationLock
6284 IONotifier *
setNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6285 IOService::setNotification(
6286 const OSSymbol * type, OSDictionary * matching,
6287 IOServiceMatchingNotificationHandler handler, void * target, void * ref,
6288 SInt32 priority )
6289 {
6290 _IOServiceNotifier * notify = NULL;
6291 OSOrderedSet * set;
6292
6293 if (!matching) {
6294 return NULL;
6295 }
6296
6297 notify = new _IOServiceNotifier;
6298 if (notify && !notify->init()) {
6299 notify->release();
6300 notify = NULL;
6301 }
6302
6303 if (notify) {
6304 notify->handler = handler;
6305 notify->target = target;
6306 notify->type = type;
6307 notify->matching = matching;
6308 matching->retain();
6309 if (handler == &_IOServiceMatchingNotificationHandler) {
6310 notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
6311 notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
6312 } else {
6313 notify->ref = ref;
6314 }
6315 notify->priority = priority;
6316 notify->state = kIOServiceNotifyEnable;
6317 queue_init( ¬ify->handlerInvocations );
6318
6319 ////// queue
6320
6321 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
6322 set = OSOrderedSet::withCapacity( 1,
6323 IONotifyOrdering, NULL );
6324 if (set) {
6325 gNotifications->setObject( type, set );
6326 set->release();
6327 }
6328 }
6329 notify->whence = set;
6330 if (set) {
6331 set->setObject( notify );
6332 }
6333 }
6334
6335 return notify;
6336 }
6337
6338 // internal - call with gNotificationLock
6339 IONotifier *
doInstallNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6340 IOService::doInstallNotification(
6341 const OSSymbol * type, OSDictionary * matching,
6342 IOServiceMatchingNotificationHandler handler,
6343 void * target, void * ref,
6344 SInt32 priority, OSIterator ** existing )
6345 {
6346 OSIterator * exist;
6347 IONotifier * notify;
6348 IOOptionBits inState;
6349
6350 if (!matching) {
6351 return NULL;
6352 }
6353
6354 if (type == gIOPublishNotification) {
6355 inState = kIOServiceRegisteredState;
6356 } else if (type == gIOFirstPublishNotification) {
6357 inState = kIOServiceFirstPublishState;
6358 } else if (type == gIOMatchedNotification) {
6359 inState = kIOServiceMatchedState;
6360 } else if (type == gIOFirstMatchNotification) {
6361 inState = kIOServiceFirstMatchState;
6362 } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
6363 inState = 0;
6364 } else {
6365 return NULL;
6366 }
6367
6368 notify = setNotification( type, matching, handler, target, ref, priority );
6369
6370 if (inState) {
6371 // get the current set
6372 exist = (OSIterator *) copyExistingServices( matching, inState );
6373 } else {
6374 exist = NULL;
6375 }
6376
6377 *existing = exist;
6378
6379 return notify;
6380 }
6381
6382 #if !defined(__LP64__)
6383 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority,OSIterator ** existing)6384 IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
6385 IOServiceNotificationHandler handler,
6386 void * target, void * refCon,
6387 SInt32 priority, OSIterator ** existing )
6388 {
6389 IONotifier * result;
6390 _IOServiceMatchingNotificationHandlerRef ref;
6391 ref.handler = handler;
6392 ref.ref = refCon;
6393
6394 result = (_IOServiceNotifier *) installNotification( type, matching,
6395 &_IOServiceMatchingNotificationHandler,
6396 target, &ref, priority, existing );
6397 if (result) {
6398 matching->release();
6399 }
6400
6401 return result;
6402 }
6403
6404 #endif /* !defined(__LP64__) */
6405
6406
6407 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6408 IOService::installNotification(
6409 const OSSymbol * type, OSDictionary * matching,
6410 IOServiceMatchingNotificationHandler handler,
6411 void * target, void * ref,
6412 SInt32 priority, OSIterator ** existing )
6413 {
6414 IONotifier * notify;
6415
6416 LOCKWRITENOTIFY();
6417
6418 notify = doInstallNotification( type, matching, handler, target, ref,
6419 priority, existing );
6420
6421 // in case handler remove()s
6422 if (notify) {
6423 notify->retain();
6424 }
6425
6426 UNLOCKNOTIFY();
6427
6428 return notify;
6429 }
6430
6431 IONotifier *
addNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority)6432 IOService::addNotification(
6433 const OSSymbol * type, OSDictionary * matching,
6434 IOServiceNotificationHandler handler,
6435 void * target, void * refCon,
6436 SInt32 priority )
6437 {
6438 IONotifier * result;
6439 _IOServiceMatchingNotificationHandlerRef ref;
6440
6441 ref.handler = handler;
6442 ref.ref = refCon;
6443
6444 result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
6445 target, &ref, priority);
6446
6447 if (result) {
6448 matching->release();
6449 }
6450
6451 return result;
6452 }
6453
6454 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6455 IOService::addMatchingNotification(
6456 const OSSymbol * type, OSDictionary * matching,
6457 IOServiceMatchingNotificationHandler handler,
6458 void * target, void * ref,
6459 SInt32 priority )
6460 {
6461 OSIterator * existing = NULL;
6462 IONotifier * ret;
6463 _IOServiceNotifier * notify;
6464 IOService * next;
6465
6466 ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
6467 handler, target, ref, priority, &existing );
6468 if (!ret) {
6469 OSSafeReleaseNULL(existing);
6470 return NULL;
6471 }
6472
6473 // send notifications for existing set
6474 if (existing) {
6475 while ((next = (IOService *) existing->getNextObject())) {
6476 if (0 == (next->__state[0] & kIOServiceInactiveState)) {
6477 next->invokeNotifier( notify );
6478 }
6479 }
6480 existing->release();
6481 }
6482
6483 LOCKWRITENOTIFY();
6484 bool removed = (NULL == notify->whence);
6485 notify->release();
6486 if (removed) {
6487 ret = gIOServiceNullNotifier;
6488 }
6489 UNLOCKNOTIFY();
6490
6491 return ret;
6492 }
6493
6494 static bool
IOServiceMatchingNotificationHandlerToBlock(void * target __unused,void * refCon,IOService * newService,IONotifier * notifier)6495 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
6496 IOService * newService,
6497 IONotifier * notifier )
6498 {
6499 return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
6500 }
6501
6502 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,SInt32 priority,IOServiceMatchingNotificationHandlerBlock handler)6503 IOService::addMatchingNotification(
6504 const OSSymbol * type, OSDictionary * matching,
6505 SInt32 priority,
6506 IOServiceMatchingNotificationHandlerBlock handler)
6507 {
6508 IONotifier * notify;
6509 void * block;
6510
6511 block = Block_copy(handler);
6512 if (!block) {
6513 return NULL;
6514 }
6515
6516 notify = addMatchingNotification(type, matching,
6517 &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
6518
6519 if (!notify) {
6520 Block_release(block);
6521 }
6522
6523 return notify;
6524 }
6525
6526 struct IOUserServerCancellationHandlerArgs {
6527 IOService ** ref;
6528 bool canceled;
6529 };
6530
6531 void
userServerCheckInTokenCancellationHandler(__unused IOUserServerCheckInToken * token,void * ref)6532 IOService::userServerCheckInTokenCancellationHandler(
6533 __unused IOUserServerCheckInToken *token,
6534 void *ref)
6535 {
6536 IOUserServerCancellationHandlerArgs * args = (typeof(args))ref;
6537 LOCKWRITENOTIFY();
6538 WAKEUPNOTIFY(args->ref);
6539 args->canceled = true;
6540 UNLOCKNOTIFY();
6541 }
6542
6543 bool
syncNotificationHandler(void *,void * ref,IOService * newService,IONotifier * notifier)6544 IOService::syncNotificationHandler(
6545 void * /* target */, void * ref,
6546 IOService * newService,
6547 IONotifier * notifier )
6548 {
6549 LOCKWRITENOTIFY();
6550 if (!*((IOService **) ref)) {
6551 newService->retain();
6552 (*(IOService **) ref) = newService;
6553 WAKEUPNOTIFY(ref);
6554 }
6555 UNLOCKNOTIFY();
6556
6557 return false;
6558 }
6559
6560 IOService *
waitForMatchingServiceWithToken(OSDictionary * matching,uint64_t timeout,IOUserServerCheckInToken * checkInToken)6561 IOService::waitForMatchingServiceWithToken( OSDictionary * matching,
6562 uint64_t timeout,
6563 IOUserServerCheckInToken * checkInToken)
6564 {
6565 IONotifier * notify = NULL;
6566 // priority doesn't help us much since we need a thread wakeup
6567 SInt32 priority = 0;
6568 IOService * result;
6569 IOUserServerCancellationHandlerArgs cancelArgs;
6570 _IOUserServerCheckInCancellationHandler * cancellationHandler = NULL;
6571
6572 if (!matching) {
6573 return NULL;
6574 }
6575
6576 result = NULL;
6577 cancelArgs.ref = &result;
6578 cancelArgs.canceled = false;
6579
6580 #if DEBUG || DEVELOPMENT
6581 char currentName[MAXTHREADNAMESIZE];
6582 char newName[MAXTHREADNAMESIZE];
6583 OSObject * obj;
6584 OSString * str;
6585 OSDictionary * dict;
6586
6587 currentName[0] = '\0';
6588 if (thread_has_thread_name(current_thread())) {
6589 dict = matching;
6590 obj = matching->getObject(gIOPropertyMatchKey);
6591 if ((dict = OSDynamicCast(OSDictionary, obj))) {
6592 OSObject * result __block = NULL;
6593 dict->iterateObjects(^bool (const OSSymbol * sym, OSObject * value) {
6594 result = __DECONST(OSObject *, sym);
6595 return true;
6596 });
6597 obj = result;
6598 }
6599 if (!obj) {
6600 obj = matching->getObject(gIOResourceMatchKey);
6601 }
6602 if (!obj) {
6603 obj = matching->getObject(gIONameMatchKey);
6604 }
6605 if (!obj) {
6606 obj = matching->getObject(gIOProviderClassKey);
6607 }
6608 if ((str = OSDynamicCast(OSString, obj))) {
6609 thread_get_thread_name(current_thread(), currentName);
6610 snprintf(newName, sizeof(newName), "Waiting_'%s'", str->getCStringNoCopy());
6611 thread_set_thread_name(current_thread(), newName);
6612 }
6613 }
6614 #endif /* DEBUG || DEVELOPMENT */
6615
6616 if (checkInToken) {
6617 cancellationHandler = checkInToken->setCancellationHandler(&IOService::userServerCheckInTokenCancellationHandler,
6618 &cancelArgs);
6619 }
6620
6621 LOCKWRITENOTIFY();
6622 do{
6623 if (cancelArgs.canceled) {
6624 // token was already canceled, no need to wait or find services
6625 break;
6626 }
6627 result = (IOService *) copyExistingServices( matching,
6628 kIOServiceMatchedState, kIONotifyOnce );
6629 if (result) {
6630 break;
6631 }
6632 notify = IOService::setNotification( gIOMatchedNotification, matching,
6633 &IOService::syncNotificationHandler, (void *) NULL,
6634 &result, priority );
6635 if (!notify) {
6636 break;
6637 }
6638 if (UINT64_MAX != timeout) {
6639 AbsoluteTime deadline;
6640 nanoseconds_to_absolutetime(timeout, &deadline);
6641 clock_absolutetime_interval_to_deadline(deadline, &deadline);
6642 SLEEPNOTIFYTO(&result, deadline);
6643 } else {
6644 SLEEPNOTIFY(&result);
6645 }
6646 }while (false);
6647
6648 UNLOCKNOTIFY();
6649
6650 if (checkInToken && cancellationHandler) {
6651 checkInToken->removeCancellationHandler(cancellationHandler);
6652 }
6653
6654 #if DEBUG || DEVELOPMENT
6655 if (currentName[0]) {
6656 thread_set_thread_name(current_thread(), currentName);
6657 }
6658 #endif /* DEBUG || DEVELOPMENT */
6659
6660 if (notify) {
6661 notify->remove(); // dequeues
6662 }
6663
6664 OSSafeReleaseNULL(cancellationHandler);
6665
6666 return result;
6667 }
6668
6669 IOService *
waitForMatchingService(OSDictionary * matching,uint64_t timeout)6670 IOService::waitForMatchingService( OSDictionary * matching,
6671 uint64_t timeout)
6672 {
6673 return IOService::waitForMatchingServiceWithToken(matching, timeout, NULL);
6674 }
6675
6676 IOService *
waitForService(OSDictionary * matching,mach_timespec_t * timeout)6677 IOService::waitForService( OSDictionary * matching,
6678 mach_timespec_t * timeout )
6679 {
6680 IOService * result;
6681 uint64_t timeoutNS;
6682
6683 if (timeout) {
6684 timeoutNS = timeout->tv_sec;
6685 timeoutNS *= kSecondScale;
6686 timeoutNS += timeout->tv_nsec;
6687 } else {
6688 timeoutNS = UINT64_MAX;
6689 }
6690
6691 result = waitForMatchingService(matching, timeoutNS);
6692
6693 matching->release();
6694 if (result) {
6695 result->release();
6696 }
6697
6698 return result;
6699 }
6700
6701 __dead2
6702 void
deliverNotification(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6703 IOService::deliverNotification( const OSSymbol * type,
6704 IOOptionBits orNewState, IOOptionBits andNewState )
6705 {
6706 panic("deliverNotification");
6707 }
6708
6709 OSArray *
copyNotifiers(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6710 IOService::copyNotifiers(const OSSymbol * type,
6711 IOOptionBits orNewState, IOOptionBits andNewState )
6712 {
6713 _IOServiceNotifier * notify;
6714 OSIterator * iter;
6715 OSArray * willSend = NULL;
6716
6717 lockForArbitration();
6718
6719 if ((0 == (__state[0] & kIOServiceInactiveState))
6720 || (type == gIOTerminatedNotification)
6721 || (type == gIOWillTerminateNotification)) {
6722 LOCKREADNOTIFY();
6723
6724 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
6725 gNotifications->getObject( type ));
6726
6727 if (iter) {
6728 while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
6729 if (matchPassive(notify->matching, 0)
6730 && (kIOServiceNotifyEnable & notify->state)) {
6731 if (NULL == willSend) {
6732 willSend = OSArray::withCapacity(8);
6733 }
6734 if (willSend) {
6735 willSend->setObject( notify );
6736 }
6737 }
6738 }
6739 iter->release();
6740 }
6741 __state[0] = (__state[0] | orNewState) & andNewState;
6742 UNLOCKNOTIFY();
6743 }
6744
6745 unlockForArbitration();
6746
6747 return willSend;
6748 }
6749
6750 IOOptionBits
getState(void) const6751 IOService::getState( void ) const
6752 {
6753 return __state[0];
6754 }
6755
6756 /*
6757 * Helpers to make matching objects for simple cases
6758 */
6759
6760 OSDictionary *
serviceMatching(const OSString * name,OSDictionary * table)6761 IOService::serviceMatching( const OSString * name,
6762 OSDictionary * table )
6763 {
6764 const OSString * str;
6765
6766 str = OSSymbol::withString(name);
6767 if (!str) {
6768 return NULL;
6769 }
6770
6771 if (!table) {
6772 table = OSDictionary::withCapacity( 2 );
6773 }
6774 if (table) {
6775 table->setObject(gIOProviderClassKey, (OSObject *)str );
6776 }
6777 str->release();
6778
6779 return table;
6780 }
6781
6782
6783 OSSharedPtr<OSDictionary>
serviceMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6784 IOService::serviceMatching( const OSString * name,
6785 OSSharedPtr<OSDictionary> table)
6786 {
6787 OSDictionary * result = serviceMatching(name, table.get());
6788 if (table) {
6789 return OSSharedPtr<OSDictionary>(result, OSRetain);
6790 } else {
6791 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6792 }
6793 }
6794
6795
6796 OSDictionary *
serviceMatching(const char * name,OSDictionary * table)6797 IOService::serviceMatching( const char * name,
6798 OSDictionary * table )
6799 {
6800 const OSString * str;
6801
6802 str = OSSymbol::withCString( name );
6803 if (!str) {
6804 return NULL;
6805 }
6806
6807 table = serviceMatching( str, table );
6808 str->release();
6809 return table;
6810 }
6811
6812
6813 OSSharedPtr<OSDictionary>
serviceMatching(const char * className,OSSharedPtr<OSDictionary> table)6814 IOService::serviceMatching( const char * className,
6815 OSSharedPtr<OSDictionary> table)
6816 {
6817 OSDictionary * result = serviceMatching(className, table.get());
6818 if (table) {
6819 return OSSharedPtr<OSDictionary>(result, OSRetain);
6820 } else {
6821 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6822 }
6823 }
6824
6825
6826 OSDictionary *
nameMatching(const OSString * name,OSDictionary * table)6827 IOService::nameMatching( const OSString * name,
6828 OSDictionary * table )
6829 {
6830 if (!table) {
6831 table = OSDictionary::withCapacity( 2 );
6832 }
6833 if (table) {
6834 table->setObject( gIONameMatchKey, (OSObject *)name );
6835 }
6836
6837 return table;
6838 }
6839
6840
6841 OSSharedPtr<OSDictionary>
nameMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6842 IOService::nameMatching( const OSString * name,
6843 OSSharedPtr<OSDictionary> table)
6844 {
6845 OSDictionary * result = nameMatching(name, table.get());
6846 if (table) {
6847 return OSSharedPtr<OSDictionary>(result, OSRetain);
6848 } else {
6849 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6850 }
6851 }
6852
6853
6854 OSDictionary *
nameMatching(const char * name,OSDictionary * table)6855 IOService::nameMatching( const char * name,
6856 OSDictionary * table )
6857 {
6858 const OSString * str;
6859
6860 str = OSSymbol::withCString( name );
6861 if (!str) {
6862 return NULL;
6863 }
6864
6865 table = nameMatching( str, table );
6866 str->release();
6867 return table;
6868 }
6869
6870
6871 OSSharedPtr<OSDictionary>
nameMatching(const char * name,OSSharedPtr<OSDictionary> table)6872 IOService::nameMatching( const char * name,
6873 OSSharedPtr<OSDictionary> table)
6874 {
6875 OSDictionary * result = nameMatching(name, table.get());
6876 if (table) {
6877 return OSSharedPtr<OSDictionary>(result, OSRetain);
6878 } else {
6879 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6880 }
6881 }
6882
6883
6884 OSDictionary *
resourceMatching(const OSString * str,OSDictionary * table)6885 IOService::resourceMatching( const OSString * str,
6886 OSDictionary * table )
6887 {
6888 table = serviceMatching( gIOResourcesKey, table );
6889 if (table) {
6890 table->setObject( gIOResourceMatchKey, (OSObject *) str );
6891 }
6892
6893 return table;
6894 }
6895
6896
6897 OSSharedPtr<OSDictionary>
resourceMatching(const OSString * str,OSSharedPtr<OSDictionary> table)6898 IOService::resourceMatching( const OSString * str,
6899 OSSharedPtr<OSDictionary> table)
6900 {
6901 OSDictionary * result = resourceMatching(str, table.get());
6902 if (table) {
6903 return OSSharedPtr<OSDictionary>(result, OSRetain);
6904 } else {
6905 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6906 }
6907 }
6908
6909
6910 OSDictionary *
resourceMatching(const char * name,OSDictionary * table)6911 IOService::resourceMatching( const char * name,
6912 OSDictionary * table )
6913 {
6914 const OSSymbol * str;
6915
6916 str = OSSymbol::withCString( name );
6917 if (!str) {
6918 return NULL;
6919 }
6920
6921 table = resourceMatching( str, table );
6922 str->release();
6923
6924 return table;
6925 }
6926
6927
6928 OSSharedPtr<OSDictionary>
resourceMatching(const char * name,OSSharedPtr<OSDictionary> table)6929 IOService::resourceMatching( const char * name,
6930 OSSharedPtr<OSDictionary> table)
6931 {
6932 OSDictionary * result = resourceMatching(name, table.get());
6933 if (table) {
6934 return OSSharedPtr<OSDictionary>(result, OSRetain);
6935 } else {
6936 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6937 }
6938 }
6939
6940
6941 OSDictionary *
propertyMatching(const OSSymbol * key,const OSObject * value,OSDictionary * table)6942 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6943 OSDictionary * table )
6944 {
6945 OSDictionary * properties;
6946
6947 properties = OSDictionary::withCapacity( 2 );
6948 if (!properties) {
6949 return NULL;
6950 }
6951 properties->setObject( key, value );
6952
6953 if (!table) {
6954 table = OSDictionary::withCapacity( 2 );
6955 }
6956 if (table) {
6957 table->setObject( gIOPropertyMatchKey, properties );
6958 }
6959
6960 properties->release();
6961
6962 return table;
6963 }
6964
6965
6966 OSSharedPtr<OSDictionary>
propertyMatching(const OSSymbol * key,const OSObject * value,OSSharedPtr<OSDictionary> table)6967 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6968 OSSharedPtr<OSDictionary> table)
6969 {
6970 OSDictionary * result = propertyMatching(key, value, table.get());
6971 if (table) {
6972 return OSSharedPtr<OSDictionary>(result, OSRetain);
6973 } else {
6974 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6975 }
6976 }
6977
6978
6979 OSDictionary *
registryEntryIDMatching(uint64_t entryID,OSDictionary * table)6980 IOService::registryEntryIDMatching( uint64_t entryID,
6981 OSDictionary * table )
6982 {
6983 OSNumber * num;
6984
6985 num = OSNumber::withNumber( entryID, 64 );
6986 if (!num) {
6987 return NULL;
6988 }
6989
6990 if (!table) {
6991 table = OSDictionary::withCapacity( 2 );
6992 }
6993 if (table) {
6994 table->setObject( gIORegistryEntryIDKey, num );
6995 }
6996
6997 if (num) {
6998 num->release();
6999 }
7000
7001 return table;
7002 }
7003
7004
7005 OSSharedPtr<OSDictionary>
registryEntryIDMatching(uint64_t entryID,OSSharedPtr<OSDictionary> table)7006 IOService::registryEntryIDMatching( uint64_t entryID,
7007 OSSharedPtr<OSDictionary> table)
7008 {
7009 OSDictionary * result = registryEntryIDMatching(entryID, table.get());
7010 if (table) {
7011 return OSSharedPtr<OSDictionary>(result, OSRetain);
7012 } else {
7013 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
7014 }
7015 }
7016
7017
7018
7019 /*
7020 * _IOServiceNotifier
7021 */
7022
7023 // wait for all threads, other than the current one,
7024 // to exit the handler
7025
7026 void
wait()7027 _IOServiceNotifier::wait()
7028 {
7029 _IOServiceNotifierInvocation * next;
7030 bool doWait;
7031
7032 do {
7033 doWait = false;
7034 queue_iterate( &handlerInvocations, next,
7035 _IOServiceNotifierInvocation *, link) {
7036 if (next->thread != current_thread()) {
7037 doWait = true;
7038 break;
7039 }
7040 }
7041 if (doWait) {
7042 state |= kIOServiceNotifyWaiter;
7043 SLEEPNOTIFY(this);
7044 }
7045 } while (doWait);
7046 }
7047
7048 void
free()7049 _IOServiceNotifier::free()
7050 {
7051 assert( queue_empty( &handlerInvocations ));
7052
7053 if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
7054 Block_release(ref);
7055 }
7056
7057 OSObject::free();
7058 }
7059
7060 void
remove()7061 _IOServiceNotifier::remove()
7062 {
7063 LOCKWRITENOTIFY();
7064
7065 if (whence) {
7066 whence->removeObject((OSObject *) this );
7067 whence = NULL;
7068 }
7069 if (matching) {
7070 matching->release();
7071 matching = NULL;
7072 }
7073
7074 state &= ~kIOServiceNotifyEnable;
7075
7076 wait();
7077
7078 UNLOCKNOTIFY();
7079
7080 release();
7081 }
7082
7083 bool
disable()7084 _IOServiceNotifier::disable()
7085 {
7086 bool ret;
7087
7088 LOCKWRITENOTIFY();
7089
7090 ret = (0 != (kIOServiceNotifyEnable & state));
7091 state &= ~kIOServiceNotifyEnable;
7092 if (ret) {
7093 wait();
7094 }
7095
7096 UNLOCKNOTIFY();
7097
7098 return ret;
7099 }
7100
7101 void
enable(bool was)7102 _IOServiceNotifier::enable( bool was )
7103 {
7104 LOCKWRITENOTIFY();
7105 if (was) {
7106 state |= kIOServiceNotifyEnable;
7107 } else {
7108 state &= ~kIOServiceNotifyEnable;
7109 }
7110 UNLOCKNOTIFY();
7111 }
7112
7113
7114 /*
7115 * _IOServiceNullNotifier
7116 */
7117
7118 void
taggedRetain(const void * tag) const7119 _IOServiceNullNotifier::taggedRetain(const void *tag) const
7120 {
7121 }
7122 void
taggedRelease(const void * tag,const int when) const7123 _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
7124 {
7125 }
7126 void
free()7127 _IOServiceNullNotifier::free()
7128 {
7129 }
7130 void
wait()7131 _IOServiceNullNotifier::wait()
7132 {
7133 }
7134 void
remove()7135 _IOServiceNullNotifier::remove()
7136 {
7137 }
7138 void
enable(bool was)7139 _IOServiceNullNotifier::enable(bool was)
7140 {
7141 }
7142 bool
disable()7143 _IOServiceNullNotifier::disable()
7144 {
7145 return false;
7146 }
7147
7148 /*
7149 * IOResources
7150 */
7151
7152 IOService *
resources(void)7153 IOResources::resources( void )
7154 {
7155 IOResources * inst;
7156
7157 inst = new IOResources;
7158 if (inst && !inst->init()) {
7159 inst->release();
7160 inst = NULL;
7161 }
7162
7163 return inst;
7164 }
7165
7166 bool
init(OSDictionary * dictionary)7167 IOResources::init( OSDictionary * dictionary )
7168 {
7169 // Do super init first
7170 if (!IOService::init()) {
7171 return false;
7172 }
7173
7174 // Allow PAL layer to publish a value
7175 const char *property_name;
7176 int property_value;
7177
7178 pal_get_resource_property( &property_name, &property_value );
7179
7180 if (property_name) {
7181 OSNumber *num;
7182 const OSSymbol * sym;
7183
7184 if ((num = OSNumber::withNumber(property_value, 32)) != NULL) {
7185 if ((sym = OSSymbol::withCString( property_name)) != NULL) {
7186 this->setProperty( sym, num );
7187 sym->release();
7188 }
7189 num->release();
7190 }
7191 }
7192
7193 return true;
7194 }
7195
7196 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7197 IOResources::newUserClient(task_t owningTask, void * securityID,
7198 UInt32 type, OSDictionary * properties,
7199 IOUserClient ** handler)
7200 {
7201 return kIOReturnUnsupported;
7202 }
7203
7204 IOWorkLoop *
getWorkLoop() const7205 IOResources::getWorkLoop() const
7206 {
7207 // If we are the resource root
7208 // then use the platform's workloop
7209 if (this == (IOResources *) gIOResources) {
7210 return getPlatform()->getWorkLoop();
7211 } else {
7212 return IOService::getWorkLoop();
7213 }
7214 }
7215
7216 static bool
IOResourcesMatchPropertyTable(IOService * resources,OSDictionary * table)7217 IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
7218 {
7219 OSObject * prop;
7220 bool __block ok = true;
7221
7222 prop = table->getObject( gIOResourceMatchKey );
7223 if (prop) {
7224 prop->iterateObjects(^bool (OSObject * obj)
7225 {
7226 OSString *
7227 str = OSDynamicCast(OSString, obj);
7228 ok = (NULL != resources->getProperty(str));
7229 return !ok;
7230 });
7231 } else if ((prop = table->getObject(gIOResourceMatchedKey))) {
7232 OSObject * obj;
7233 OSArray * keys;
7234
7235 obj = resources->copyProperty(gIOResourceMatchedKey);
7236 keys = OSDynamicCast(OSArray, obj);
7237 ok = false;
7238 if (keys) {
7239 // assuming OSSymbol
7240 ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
7241 }
7242 OSSafeReleaseNULL(obj);
7243 }
7244
7245 return ok;
7246 }
7247
7248 bool
matchPropertyTable(OSDictionary * table)7249 IOResources::matchPropertyTable( OSDictionary * table )
7250 {
7251 return IOResourcesMatchPropertyTable(this, table);
7252 }
7253
7254 /*
7255 * IOUserResources
7256 */
7257
7258 IOService *
resources(void)7259 IOUserResources::resources( void )
7260 {
7261 IOUserResources * inst;
7262
7263 inst = OSTypeAlloc(IOUserResources);
7264 if (inst && !inst->init()) {
7265 inst->release();
7266 inst = NULL;
7267 }
7268
7269 return inst;
7270 }
7271
7272 bool
init(OSDictionary * dictionary)7273 IOUserResources::init( OSDictionary * dictionary )
7274 {
7275 // Do super init first
7276 if (!IOService::init()) {
7277 return false;
7278 }
7279 return true;
7280 }
7281
7282 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7283 IOUserResources::newUserClient(task_t owningTask, void * securityID,
7284 UInt32 type, OSDictionary * properties,
7285 IOUserClient ** handler)
7286 {
7287 return kIOReturnUnsupported;
7288 }
7289
7290 IOWorkLoop *
getWorkLoop() const7291 IOUserResources::getWorkLoop() const
7292 {
7293 return getPlatform()->getWorkLoop();
7294 }
7295
7296 bool
matchPropertyTable(OSDictionary * table)7297 IOUserResources::matchPropertyTable( OSDictionary * table )
7298 {
7299 return IOResourcesMatchPropertyTable(this, table);
7300 }
7301
7302 // --
7303
7304 void
consoleLockTimer(thread_call_param_t p0,thread_call_param_t p1)7305 IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
7306 {
7307 IOService::updateConsoleUsers(NULL, 0);
7308 }
7309
7310 void
updateConsoleUsers(OSArray * consoleUsers,IOMessage systemMessage,bool afterUserspaceReboot)7311 IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage, bool afterUserspaceReboot)
7312 {
7313 IORegistryEntry * regEntry;
7314 OSObject * locked = kOSBooleanFalse;
7315 uint32_t idx;
7316 bool publish;
7317 OSDictionary * user;
7318 clock_sec_t now = 0;
7319 clock_usec_t microsecs;
7320
7321 regEntry = IORegistryEntry::getRegistryRoot();
7322
7323 if (!gIOChosenEntry) {
7324 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
7325 }
7326
7327 IOLockLock(gIOConsoleUsersLock);
7328
7329 if (systemMessage) {
7330 sSystemPower = systemMessage;
7331 #if HIBERNATION
7332 if (kIOMessageSystemHasPoweredOn == systemMessage) {
7333 uint32_t lockState = IOHibernateWasScreenLocked();
7334 switch (lockState) {
7335 case 0:
7336 break;
7337 case kIOScreenLockLocked:
7338 case kIOScreenLockFileVaultDialog:
7339 gIOConsoleBooterLockState = kOSBooleanTrue;
7340 break;
7341 case kIOScreenLockNoLock:
7342 gIOConsoleBooterLockState = NULL;
7343 break;
7344 case kIOScreenLockUnlocked:
7345 default:
7346 gIOConsoleBooterLockState = kOSBooleanFalse;
7347 break;
7348 }
7349 }
7350 #endif /* HIBERNATION */
7351 }
7352
7353 if (consoleUsers) {
7354 OSNumber * num = NULL;
7355 bool loginLocked = true;
7356
7357 gIOConsoleLoggedIn = false;
7358 for (idx = 0;
7359 (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
7360 idx++) {
7361 gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
7362 && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
7363
7364 loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
7365 if (!num) {
7366 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
7367 }
7368 }
7369 #if HIBERNATION
7370 if (!loginLocked || afterUserspaceReboot) {
7371 gIOConsoleBooterLockState = NULL;
7372 }
7373 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
7374 (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
7375 gIOConsoleLoggedIn, loginLocked);
7376 #endif /* HIBERNATION */
7377 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
7378 }
7379
7380 if (!gIOConsoleLoggedIn
7381 || (kIOMessageSystemWillSleep == sSystemPower)
7382 || (kIOMessageSystemPagingOff == sSystemPower)) {
7383 if (afterUserspaceReboot) {
7384 // set "locked" to false after a user space reboot
7385 // because the reboot happens directly after a user
7386 // logs into the machine via fvunlock mode.
7387 locked = kOSBooleanFalse;
7388 } else {
7389 locked = kOSBooleanTrue;
7390 }
7391 }
7392 #if HIBERNATION
7393 else if (gIOConsoleBooterLockState) {
7394 locked = gIOConsoleBooterLockState;
7395 }
7396 #endif /* HIBERNATION */
7397 else if (gIOConsoleLockTime) {
7398 clock_get_calendar_microtime(&now, µsecs);
7399 if (gIOConsoleLockTime > now) {
7400 AbsoluteTime deadline;
7401 clock_sec_t interval;
7402 uint32_t interval32;
7403
7404 interval = (gIOConsoleLockTime - now);
7405 interval32 = (uint32_t) interval;
7406 if (interval32 != interval) {
7407 interval32 = UINT_MAX;
7408 }
7409 clock_interval_to_deadline(interval32, kSecondScale, &deadline);
7410 thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
7411 } else {
7412 locked = kOSBooleanTrue;
7413 }
7414 }
7415
7416 publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
7417 if (publish) {
7418 regEntry->setProperty(gIOConsoleLockedKey, locked);
7419 if (consoleUsers) {
7420 regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
7421 }
7422 OSIncrementAtomic( &gIOConsoleUsersSeed );
7423 }
7424
7425 #if HIBERNATION
7426 if (gIOChosenEntry) {
7427 if (locked == kOSBooleanTrue) {
7428 gIOScreenLockState = kIOScreenLockLocked;
7429 } else if (gIOConsoleLockTime) {
7430 gIOScreenLockState = kIOScreenLockUnlocked;
7431 } else {
7432 gIOScreenLockState = kIOScreenLockNoLock;
7433 }
7434 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
7435
7436 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
7437 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
7438 }
7439 #endif /* HIBERNATION */
7440
7441 IOLockUnlock(gIOConsoleUsersLock);
7442
7443 if (publish) {
7444 publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
7445
7446 MessageClientsContext context;
7447
7448 context.service = getServiceRoot();
7449 context.type = kIOMessageConsoleSecurityChange;
7450 context.argument = (void *) regEntry;
7451 context.argSize = 0;
7452
7453 applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
7454 &messageClientsApplier, &context );
7455 }
7456 }
7457
7458 IOReturn
setProperties(OSObject * properties)7459 IOResources::setProperties( OSObject * properties )
7460 {
7461 IOReturn err;
7462 const OSSymbol * key;
7463 OSDictionary * dict;
7464 OSCollectionIterator * iter;
7465
7466 if (!IOCurrentTaskHasEntitlement(kIOResourcesSetPropertyKey)) {
7467 err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
7468 if (kIOReturnSuccess != err) {
7469 return err;
7470 }
7471 }
7472
7473 dict = OSDynamicCast(OSDictionary, properties);
7474 if (NULL == dict) {
7475 return kIOReturnBadArgument;
7476 }
7477
7478 iter = OSCollectionIterator::withCollection( dict);
7479 if (NULL == iter) {
7480 return kIOReturnBadArgument;
7481 }
7482
7483 while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
7484 if (gIOConsoleUsersKey == key) {
7485 do{
7486 OSArray * consoleUsers;
7487 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
7488 if (!consoleUsers) {
7489 continue;
7490 }
7491 IOService::updateConsoleUsers(consoleUsers, 0);
7492 }while (false);
7493 }
7494
7495 publishResource( key, dict->getObject(key));
7496 }
7497
7498 iter->release();
7499
7500 return kIOReturnSuccess;
7501 }
7502
7503 /*
7504 * Helpers for matching dictionaries.
7505 * Keys existing in matching are checked in properties.
7506 * Keys may be a string or OSCollection of IOStrings
7507 */
7508
7509 bool
compareProperty(OSDictionary * matching,const char * key)7510 IOService::compareProperty( OSDictionary * matching,
7511 const char * key )
7512 {
7513 OSObject * value;
7514 OSObject * prop;
7515 bool ok;
7516
7517 value = matching->getObject( key );
7518 if (value) {
7519 prop = copyProperty(key);
7520 ok = value->isEqualTo(prop);
7521 if (prop) {
7522 prop->release();
7523 }
7524 } else {
7525 ok = true;
7526 }
7527
7528 return ok;
7529 }
7530
7531
7532 bool
compareProperty(OSDictionary * matching,const OSString * key)7533 IOService::compareProperty( OSDictionary * matching,
7534 const OSString * key )
7535 {
7536 OSObject * value;
7537 OSObject * prop;
7538 bool ok;
7539
7540 value = matching->getObject( key );
7541 if (value) {
7542 prop = copyProperty(key);
7543 ok = value->isEqualTo(prop);
7544 if (prop) {
7545 prop->release();
7546 }
7547 } else {
7548 ok = true;
7549 }
7550
7551 return ok;
7552 }
7553
7554 #ifndef __clang_analyzer__
7555 // Implementation of this function is hidden from the static analyzer.
7556 // The analyzer was worried about this function's confusing contract over
7557 // the 'keys' parameter. The contract is to either release it or not release it
7558 // depending on whether 'matching' is non-null. Such contracts are discouraged
7559 // but changing it now would break compatibility.
7560 bool
compareProperties(OSDictionary * matching,OSCollection * keys)7561 IOService::compareProperties( OSDictionary * matching,
7562 OSCollection * keys )
7563 {
7564 OSCollectionIterator * iter;
7565 const OSString * key;
7566 bool ok = true;
7567
7568 if (!matching || !keys) {
7569 return false;
7570 }
7571
7572 iter = OSCollectionIterator::withCollection( keys );
7573
7574 if (iter) {
7575 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
7576 ok = compareProperty( matching, key );
7577 }
7578
7579 iter->release();
7580 }
7581 keys->release(); // !! consume a ref !!
7582
7583 return ok;
7584 }
7585 #endif // __clang_analyzer__
7586
7587 /* Helper to add a location matching dict to the table */
7588
7589 OSDictionary *
addLocation(OSDictionary * table)7590 IOService::addLocation( OSDictionary * table )
7591 {
7592 OSDictionary * dict;
7593
7594 if (!table) {
7595 return NULL;
7596 }
7597
7598 dict = OSDictionary::withCapacity( 1 );
7599 if (dict) {
7600 bool ok = table->setObject( gIOLocationMatchKey, dict );
7601 dict->release();
7602 if (!ok) {
7603 dict = NULL;
7604 }
7605 }
7606
7607 return dict;
7608 }
7609
7610 /*
7611 * Go looking for a provider to match a location dict.
7612 */
7613
7614 IOService *
matchLocation(IOService *)7615 IOService::matchLocation( IOService * /* client */ )
7616 {
7617 IOService * parent;
7618
7619 parent = getProvider();
7620
7621 if (parent) {
7622 parent = parent->matchLocation( this );
7623 }
7624
7625 return parent;
7626 }
7627
7628 OSDictionary *
_copyPropertiesForMatching(void)7629 IOService::_copyPropertiesForMatching(void)
7630 {
7631 OSDictionary * matchProps;
7632
7633 matchProps = dictionaryWithProperties();
7634 if (matchProps) {
7635 // merge will check the OSDynamicCast
7636 matchProps->merge((const OSDictionary *)matchProps->getObject(gIOUserServicePropertiesKey));
7637 }
7638 return matchProps;
7639 }
7640
7641 bool
matchInternal(OSDictionary * table,uint32_t options,uint32_t * did)7642 IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
7643 {
7644 OSString * matched;
7645 OSObject * obj;
7646 OSString * str;
7647 OSDictionary * matchProps;
7648 IORegistryEntry * entry;
7649 OSNumber * num;
7650 bool match = true;
7651 bool changesOK = (0 != (kIOServiceChangesOK & options));
7652 uint32_t count;
7653 uint32_t done;
7654
7655 do{
7656 count = table->getCount();
7657 done = 0;
7658 matchProps = NULL;
7659 bool isUser;
7660
7661 isUser = (NULL != table->getObject(gIOServiceNotificationUserKey));
7662 if (isUser) {
7663 done++;
7664 match = (0 == (kIOServiceUserInvisibleMatchState & __state[0]));
7665 if ((!match) || (done == count)) {
7666 break;
7667 }
7668 }
7669
7670 if (propertyExists(gIOExclaveAssignedKey)) {
7671 if (!table->getObject(gIOExclaveProxyKey) && !isUser) {
7672 match = false;
7673 break;
7674 }
7675 } else if (table->getObject(gIOExclaveProxyKey)) {
7676 match = false;
7677 break;
7678 }
7679
7680 if (table->getObject(gIOCompatibilityMatchKey)) {
7681 done++;
7682 obj = copyProperty(gIOCompatibilityPropertiesKey);
7683 matchProps = OSDynamicCast(OSDictionary, obj);
7684 if (!matchProps) {
7685 OSSafeReleaseNULL(obj);
7686 }
7687 }
7688
7689 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
7690 if (str) {
7691 done++;
7692 if (matchProps && (obj = matchProps->getObject(gIOClassKey))) {
7693 match = str->isEqualTo(obj);
7694 } else {
7695 match = ((kIOServiceClassDone & options) || (NULL != metaCast(str)));
7696 }
7697
7698 #if MATCH_DEBUG
7699 match = (0 != metaCast( str ));
7700 if ((kIOServiceClassDone & options) && !match) {
7701 panic("classDone");
7702 }
7703 #endif
7704 if ((!match) || (done == count)) {
7705 break;
7706 }
7707 }
7708
7709 obj = table->getObject( gIONameMatchKey );
7710 if (obj) {
7711 done++;
7712 match = compareNames( obj, changesOK ? &matched : NULL );
7713 if (!match) {
7714 break;
7715 }
7716 if (changesOK && matched) {
7717 // leave a hint as to which name matched
7718 table->setObject( gIONameMatchedKey, matched );
7719 matched->release();
7720 }
7721 if (done == count) {
7722 break;
7723 }
7724 }
7725
7726 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
7727 if (str) {
7728 const OSSymbol * sym;
7729 done++;
7730 match = false;
7731 sym = copyLocation();
7732 if (sym) {
7733 match = sym->isEqualTo( str );
7734 sym->release();
7735 }
7736 if ((!match) || (done == count)) {
7737 break;
7738 }
7739 }
7740
7741 obj = table->getObject( gIOPropertyMatchKey );
7742 if (obj) {
7743 OSDictionary * nextDict;
7744 OSIterator * iter;
7745 done++;
7746 match = false;
7747 if (!matchProps) {
7748 matchProps = _copyPropertiesForMatching();
7749 }
7750 if (matchProps) {
7751 nextDict = OSDynamicCast( OSDictionary, obj);
7752 if (nextDict) {
7753 iter = NULL;
7754 } else {
7755 iter = OSCollectionIterator::withCollection(
7756 OSDynamicCast(OSCollection, obj));
7757 }
7758
7759 while (nextDict
7760 || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
7761 iter->getNextObject()))))) {
7762 match = matchProps->isEqualTo( nextDict, nextDict);
7763 if (match) {
7764 break;
7765 }
7766 nextDict = NULL;
7767 }
7768 if (iter) {
7769 iter->release();
7770 }
7771 }
7772 if ((!match) || (done == count)) {
7773 break;
7774 }
7775 }
7776
7777 obj = table->getObject( gIOPropertyExistsMatchKey );
7778 if (obj) {
7779 OSString * nextKey;
7780 OSIterator * iter;
7781 done++;
7782 match = false;
7783 if (!matchProps) {
7784 matchProps = _copyPropertiesForMatching();
7785 }
7786 if (matchProps) {
7787 nextKey = OSDynamicCast( OSString, obj);
7788 if (nextKey) {
7789 iter = NULL;
7790 } else {
7791 iter = OSCollectionIterator::withCollection(
7792 OSDynamicCast(OSCollection, obj));
7793 }
7794
7795 while (nextKey
7796 || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
7797 iter->getNextObject()))))) {
7798 match = (NULL != matchProps->getObject(nextKey));
7799 if (match) {
7800 break;
7801 }
7802 nextKey = NULL;
7803 }
7804 if (iter) {
7805 iter->release();
7806 }
7807 }
7808 if ((!match) || (done == count)) {
7809 break;
7810 }
7811 }
7812
7813 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
7814 if (str) {
7815 done++;
7816 entry = IORegistryEntry::fromPath( str->getCStringNoCopy());
7817 match = (this == entry);
7818 if (entry) {
7819 entry->release();
7820 }
7821 if (!match && matchProps && (obj = matchProps->getObject(gIOPathKey))) {
7822 match = str->isEqualTo(obj);
7823 }
7824 if ((!match) || (done == count)) {
7825 break;
7826 }
7827 }
7828
7829 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
7830 if (num) {
7831 done++;
7832 match = (getRegistryEntryID() == num->unsigned64BitValue());
7833 if ((!match) || (done == count)) {
7834 break;
7835 }
7836 }
7837
7838 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
7839 if (num) {
7840 OSIterator * iter;
7841 IOService * service = NULL;
7842 UInt32 serviceCount = 0;
7843
7844 done++;
7845 iter = getClientIterator();
7846 if (iter) {
7847 while ((service = (IOService *) iter->getNextObject())) {
7848 if (kIOServiceInactiveState & service->__state[0]) {
7849 continue;
7850 }
7851 if (NULL == service->getProperty( gIOMatchCategoryKey )) {
7852 continue;
7853 }
7854 ++serviceCount;
7855 }
7856 iter->release();
7857 }
7858 match = (serviceCount == num->unsigned32BitValue());
7859 if ((!match) || (done == count)) {
7860 break;
7861 }
7862 }
7863
7864 #define propMatch(key) \
7865 obj = table->getObject(key); \
7866 if (obj) \
7867 { \
7868 OSObject * prop; \
7869 done++; \
7870 prop = copyProperty(key); \
7871 match = obj->isEqualTo(prop); \
7872 if (prop) prop->release(); \
7873 if ((!match) || (done == count)) break; \
7874 }
7875 propMatch(gIOBSDNameKey)
7876 propMatch(gIOBSDMajorKey)
7877 propMatch(gIOBSDMinorKey)
7878 propMatch(gIOBSDUnitKey)
7879 #undef propMatch
7880 }while (false);
7881
7882 OSSafeReleaseNULL(matchProps);
7883
7884 if (did) {
7885 *did = done;
7886 }
7887 return match;
7888 }
7889
7890 bool
passiveMatch(OSDictionary * table,bool changesOK)7891 IOService::passiveMatch( OSDictionary * table, bool changesOK )
7892 {
7893 return matchPassive(table, changesOK ? kIOServiceChangesOK : 0);
7894 }
7895
7896 bool
matchPassive(OSDictionary * table,uint32_t options)7897 IOService::matchPassive(OSDictionary * table, uint32_t options)
7898 {
7899 IOService * where;
7900 OSDictionary * nextTable;
7901 SInt32 score;
7902 OSNumber * newPri;
7903 bool match = true;
7904 bool matchParent = false;
7905 uint32_t count;
7906 uint32_t done;
7907
7908 assert( table );
7909
7910 #if defined(XNU_TARGET_OS_OSX)
7911 OSArray* aliasServiceRegIds = NULL;
7912 IOService* foundAlternateService = NULL;
7913 #endif /* defined(XNU_TARGET_OS_OSX) */
7914
7915 #if MATCH_DEBUG
7916 OSDictionary * root = table;
7917 #endif
7918
7919 where = this;
7920 do{
7921 do{
7922 count = table->getCount();
7923 if (!(kIOServiceInternalDone & options)) {
7924 match = where->matchInternal(table, options, &done);
7925 // don't call family if we've done all the entries in the table
7926 if ((!match) || (done == count)) {
7927 break;
7928 }
7929 }
7930
7931 // pass in score from property table
7932 score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
7933
7934 // do family specific matching
7935 match = where->matchPropertyTable( table, &score );
7936
7937 if (!match) {
7938 #if IOMATCHDEBUG
7939 if (kIOLogMatch & getDebugFlags( table )) {
7940 LOG("%s: family specific matching fails\n", where->getName());
7941 }
7942 #endif
7943 break;
7944 }
7945
7946 if (kIOServiceChangesOK & options) {
7947 // save the score
7948 newPri = OSNumber::withNumber( score, 32 );
7949 if (newPri) {
7950 table->setObject( gIOProbeScoreKey, newPri );
7951 newPri->release();
7952 }
7953 }
7954
7955 options = 0;
7956 matchParent = false;
7957
7958 nextTable = OSDynamicCast(OSDictionary,
7959 table->getObject( gIOParentMatchKey ));
7960 if (nextTable) {
7961 // look for a matching entry anywhere up to root
7962 match = false;
7963 matchParent = true;
7964 table = nextTable;
7965 break;
7966 }
7967
7968 table = OSDynamicCast(OSDictionary,
7969 table->getObject( gIOLocationMatchKey ));
7970 if (table) {
7971 // look for a matching entry at matchLocation()
7972 match = false;
7973 where = where->getProvider();
7974 if (where && (where = where->matchLocation(where))) {
7975 continue;
7976 }
7977 }
7978 break;
7979 }while (true);
7980
7981 if (match == true) {
7982 break;
7983 }
7984
7985 if (matchParent == true) {
7986 #if defined(XNU_TARGET_OS_OSX)
7987 // check if service has an alias to search its other "parents" if a parent match isn't found
7988 OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
7989 OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
7990 if (alternateRegistryID != NULL) {
7991 if (aliasServiceRegIds == NULL) {
7992 aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
7993 }
7994 aliasServiceRegIds->setObject(alternateRegistryID);
7995 }
7996 OSSafeReleaseNULL(prop);
7997 #endif /* defined(XNU_TARGET_OS_OSX) */
7998 } else {
7999 break;
8000 }
8001
8002 where = where->getProvider();
8003 #if defined(XNU_TARGET_OS_OSX)
8004 if (where == NULL) {
8005 // there were no matching parent services, check to see if there are aliased services that have a matching parent
8006 if (aliasServiceRegIds != NULL) {
8007 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
8008 if (numAliasedServices != 0) {
8009 OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
8010 if (alternateRegistryID != NULL) {
8011 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
8012 aliasServiceRegIds->removeObject(numAliasedServices - 1);
8013 if (alternateMatchingDict != NULL) {
8014 OSSafeReleaseNULL(foundAlternateService);
8015 foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
8016 alternateMatchingDict->release();
8017 if (foundAlternateService != NULL) {
8018 where = foundAlternateService;
8019 }
8020 }
8021 }
8022 }
8023 }
8024 }
8025 #endif /* defined(XNU_TARGET_OS_OSX) */
8026 }while (where != NULL);
8027
8028 #if defined(XNU_TARGET_OS_OSX)
8029 OSSafeReleaseNULL(foundAlternateService);
8030 OSSafeReleaseNULL(aliasServiceRegIds);
8031 #endif /* defined(XNU_TARGET_OS_OSX) */
8032
8033 #if MATCH_DEBUG
8034 if (where != this) {
8035 OSSerialize * s = OSSerialize::withCapacity(128);
8036 root->serialize(s);
8037 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
8038 s->release();
8039 }
8040 #endif
8041
8042 return match;
8043 }
8044
8045
8046 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)8047 IOService::newUserClient( task_t owningTask, void * securityID,
8048 UInt32 type, OSDictionary * properties,
8049 IOUserClient ** handler )
8050 {
8051 const OSSymbol *userClientClass = NULL;
8052 IOUserClient *client;
8053 OSObject *prop;
8054 OSObject *temp;
8055
8056 if (reserved && reserved->uvars && reserved->uvars->userServer) {
8057 return reserved->uvars->userServer->serviceNewUserClient(this, owningTask, securityID, type, properties, handler);
8058 }
8059
8060 if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
8061 return kIOReturnSuccess;
8062 }
8063
8064 // First try my own properties for a user client class name
8065 prop = copyProperty(gIOUserClientClassKey);
8066 if (prop) {
8067 if (OSDynamicCast(OSSymbol, prop)) {
8068 userClientClass = (const OSSymbol *) prop;
8069 prop = NULL;
8070 } else if (OSDynamicCast(OSString, prop)) {
8071 userClientClass = OSSymbol::withString((OSString *) prop);
8072 OSSafeReleaseNULL(prop);
8073 if (userClientClass) {
8074 setProperty(gIOUserClientClassKey,
8075 (OSObject *) userClientClass);
8076 }
8077 } else {
8078 OSSafeReleaseNULL(prop);
8079 }
8080 }
8081
8082 // Didn't find one so lets just bomb out now without further ado.
8083 if (!userClientClass) {
8084 return kIOReturnUnsupported;
8085 }
8086
8087 // This reference is consumed by the IOServiceOpen call
8088 temp = OSMetaClass::allocClassWithName(userClientClass);
8089 OSSafeReleaseNULL(userClientClass);
8090 if (!temp) {
8091 return kIOReturnNoMemory;
8092 }
8093
8094 if (OSDynamicCast(IOUserClient, temp)) {
8095 client = (IOUserClient *) temp;
8096 } else {
8097 temp->release();
8098 return kIOReturnUnsupported;
8099 }
8100
8101 if (!client->initWithTask(owningTask, securityID, type, properties)) {
8102 client->release();
8103 return kIOReturnBadArgument;
8104 }
8105
8106 if (!client->attach(this)) {
8107 client->release();
8108 return kIOReturnUnsupported;
8109 }
8110
8111 if (!client->start(this)) {
8112 client->detach(this);
8113 client->release();
8114 return kIOReturnUnsupported;
8115 }
8116
8117 *handler = client;
8118 return kIOReturnSuccess;
8119 }
8120
8121 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,OSSharedPtr<IOUserClient> & handler)8122 IOService::newUserClient( task_t owningTask, void * securityID,
8123 UInt32 type, OSDictionary * properties,
8124 OSSharedPtr<IOUserClient>& handler )
8125 {
8126 IOUserClient* handlerRaw = NULL;
8127 IOReturn result = newUserClient(owningTask, securityID, type, properties, &handlerRaw);
8128 handler.reset(handlerRaw, OSNoRetain);
8129 return result;
8130 }
8131
8132 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,IOUserClient ** handler)8133 IOService::newUserClient( task_t owningTask, void * securityID,
8134 UInt32 type, IOUserClient ** handler )
8135 {
8136 return kIOReturnUnsupported;
8137 }
8138
8139 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSSharedPtr<IOUserClient> & handler)8140 IOService::newUserClient( task_t owningTask, void * securityID,
8141 UInt32 type, OSSharedPtr<IOUserClient>& handler )
8142 {
8143 IOUserClient* handlerRaw = nullptr;
8144 IOReturn result = IOService::newUserClient(owningTask, securityID, type, &handlerRaw);
8145 handler.reset(handlerRaw, OSNoRetain);
8146 return result;
8147 }
8148
8149
8150 IOReturn
requestProbe(IOOptionBits options)8151 IOService::requestProbe( IOOptionBits options )
8152 {
8153 return kIOReturnUnsupported;
8154 }
8155
8156 bool
hasUserServer() const8157 IOService::hasUserServer() const
8158 {
8159 return reserved && reserved->uvars && reserved->uvars->userServer;
8160 }
8161
8162 /*
8163 * Convert an IOReturn to text. Subclasses which add additional
8164 * IOReturn's should override this method and call
8165 * super::stringFromReturn if the desired value is not found.
8166 */
8167
8168 const char *
stringFromReturn(IOReturn rtn)8169 IOService::stringFromReturn( IOReturn rtn )
8170 {
8171 static const IONamedValue IOReturn_values[] = {
8172 {kIOReturnSuccess, "success" },
8173 {kIOReturnError, "general error" },
8174 {kIOReturnNoMemory, "memory allocation error" },
8175 {kIOReturnNoResources, "resource shortage" },
8176 {kIOReturnIPCError, "Mach IPC failure" },
8177 {kIOReturnNoDevice, "no such device" },
8178 {kIOReturnNotPrivileged, "privilege violation" },
8179 {kIOReturnBadArgument, "invalid argument" },
8180 {kIOReturnLockedRead, "device is read locked" },
8181 {kIOReturnLockedWrite, "device is write locked" },
8182 {kIOReturnExclusiveAccess, "device is exclusive access" },
8183 {kIOReturnBadMessageID, "bad IPC message ID" },
8184 {kIOReturnUnsupported, "unsupported function" },
8185 {kIOReturnVMError, "virtual memory error" },
8186 {kIOReturnInternalError, "internal driver error" },
8187 {kIOReturnIOError, "I/O error" },
8188 {kIOReturnCannotLock, "cannot acquire lock" },
8189 {kIOReturnNotOpen, "device is not open" },
8190 {kIOReturnNotReadable, "device is not readable" },
8191 {kIOReturnNotWritable, "device is not writeable" },
8192 {kIOReturnNotAligned, "alignment error" },
8193 {kIOReturnBadMedia, "media error" },
8194 {kIOReturnStillOpen, "device is still open" },
8195 {kIOReturnRLDError, "rld failure" },
8196 {kIOReturnDMAError, "DMA failure" },
8197 {kIOReturnBusy, "device is busy" },
8198 {kIOReturnTimeout, "I/O timeout" },
8199 {kIOReturnOffline, "device is offline" },
8200 {kIOReturnNotReady, "device is not ready" },
8201 {kIOReturnNotAttached, "device/channel is not attached" },
8202 {kIOReturnNoChannels, "no DMA channels available" },
8203 {kIOReturnNoSpace, "no space for data" },
8204 {kIOReturnPortExists, "device port already exists" },
8205 {kIOReturnCannotWire, "cannot wire physical memory" },
8206 {kIOReturnNoInterrupt, "no interrupt attached" },
8207 {kIOReturnNoFrames, "no DMA frames enqueued" },
8208 {kIOReturnMessageTooLarge, "message is too large" },
8209 {kIOReturnNotPermitted, "operation is not permitted" },
8210 {kIOReturnNoPower, "device is without power" },
8211 {kIOReturnNoMedia, "media is not present" },
8212 {kIOReturnUnformattedMedia, "media is not formatted" },
8213 {kIOReturnUnsupportedMode, "unsupported mode" },
8214 {kIOReturnUnderrun, "data underrun" },
8215 {kIOReturnOverrun, "data overrun" },
8216 {kIOReturnDeviceError, "device error" },
8217 {kIOReturnNoCompletion, "no completion routine" },
8218 {kIOReturnAborted, "operation was aborted" },
8219 {kIOReturnNoBandwidth, "bus bandwidth would be exceeded" },
8220 {kIOReturnNotResponding, "device is not responding" },
8221 {kIOReturnInvalid, "unanticipated driver error" },
8222 {0, NULL }
8223 };
8224
8225 return IOFindNameForValue(rtn, IOReturn_values);
8226 }
8227
8228 /*
8229 * Convert an IOReturn to an errno.
8230 */
8231 int
errnoFromReturn(IOReturn rtn)8232 IOService::errnoFromReturn( IOReturn rtn )
8233 {
8234 if (unix_err(err_get_code(rtn)) == rtn) {
8235 return err_get_code(rtn);
8236 }
8237
8238 switch (rtn) {
8239 // (obvious match)
8240 case kIOReturnSuccess:
8241 return 0;
8242 case kIOReturnNoMemory:
8243 return ENOMEM;
8244 case kIOReturnNoDevice:
8245 return ENXIO;
8246 case kIOReturnVMError:
8247 return EFAULT;
8248 case kIOReturnNotPermitted:
8249 return EPERM;
8250 case kIOReturnNotPrivileged:
8251 return EACCES;
8252 case kIOReturnIOError:
8253 return EIO;
8254 case kIOReturnNotWritable:
8255 return EROFS;
8256 case kIOReturnBadArgument:
8257 return EINVAL;
8258 case kIOReturnUnsupported:
8259 return ENOTSUP;
8260 case kIOReturnBusy:
8261 return EBUSY;
8262 case kIOReturnNoPower:
8263 return EPWROFF;
8264 case kIOReturnDeviceError:
8265 return EDEVERR;
8266 case kIOReturnTimeout:
8267 return ETIMEDOUT;
8268 case kIOReturnMessageTooLarge:
8269 return EMSGSIZE;
8270 case kIOReturnNoSpace:
8271 return ENOSPC;
8272 case kIOReturnCannotLock:
8273 return ENOLCK;
8274
8275 // (best match)
8276 case kIOReturnBadMessageID:
8277 case kIOReturnNoCompletion:
8278 case kIOReturnNotAligned:
8279 return EINVAL;
8280 case kIOReturnNotReady:
8281 return EBUSY;
8282 case kIOReturnRLDError:
8283 return EBADMACHO;
8284 case kIOReturnPortExists:
8285 case kIOReturnStillOpen:
8286 return EEXIST;
8287 case kIOReturnExclusiveAccess:
8288 case kIOReturnLockedRead:
8289 case kIOReturnLockedWrite:
8290 case kIOReturnNotOpen:
8291 case kIOReturnNotReadable:
8292 return EACCES;
8293 case kIOReturnCannotWire:
8294 case kIOReturnNoResources:
8295 return ENOMEM;
8296 case kIOReturnAborted:
8297 case kIOReturnOffline:
8298 case kIOReturnNotResponding:
8299 return EBUSY;
8300 case kIOReturnBadMedia:
8301 case kIOReturnNoMedia:
8302 case kIOReturnNotAttached:
8303 case kIOReturnUnformattedMedia:
8304 return ENXIO; // (media error)
8305 case kIOReturnDMAError:
8306 case kIOReturnOverrun:
8307 case kIOReturnUnderrun:
8308 return EIO; // (transfer error)
8309 case kIOReturnNoBandwidth:
8310 case kIOReturnNoChannels:
8311 case kIOReturnNoFrames:
8312 case kIOReturnNoInterrupt:
8313 return EIO; // (hardware error)
8314 case kIOReturnError:
8315 case kIOReturnInternalError:
8316 case kIOReturnInvalid:
8317 return EIO; // (generic error)
8318 case kIOReturnIPCError:
8319 return EIO; // (ipc error)
8320 default:
8321 return EIO; // (all other errors)
8322 }
8323 }
8324
8325 IOReturn
message(UInt32 type,IOService * provider,void * argument)8326 IOService::message( UInt32 type, IOService * provider,
8327 void * argument )
8328 {
8329 /*
8330 * Generic entry point for calls from the provider. A return value of
8331 * kIOReturnSuccess indicates that the message was received, and where
8332 * applicable, that it was successful.
8333 */
8334
8335 return kIOReturnUnsupported;
8336 }
8337
8338 /*
8339 * Device memory
8340 */
8341
8342 IOItemCount
getDeviceMemoryCount(void)8343 IOService::getDeviceMemoryCount( void )
8344 {
8345 OSArray * array;
8346 IOItemCount count;
8347
8348 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8349 if (array) {
8350 count = array->getCount();
8351 } else {
8352 count = 0;
8353 }
8354
8355 return count;
8356 }
8357
8358 IODeviceMemory *
getDeviceMemoryWithIndex(unsigned int index)8359 IOService::getDeviceMemoryWithIndex( unsigned int index )
8360 {
8361 OSArray * array;
8362 IODeviceMemory * range;
8363
8364 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8365 if (array) {
8366 range = (IODeviceMemory *) array->getObject( index );
8367 } else {
8368 range = NULL;
8369 }
8370
8371 return range;
8372 }
8373
8374 IOMemoryMap *
mapDeviceMemoryWithIndex(unsigned int index,IOOptionBits options)8375 IOService::mapDeviceMemoryWithIndex( unsigned int index,
8376 IOOptionBits options )
8377 {
8378 IODeviceMemory * range;
8379 IOMemoryMap * map;
8380
8381 range = getDeviceMemoryWithIndex( index );
8382 if (range) {
8383 map = range->map( options );
8384 } else {
8385 map = NULL;
8386 }
8387
8388 return map;
8389 }
8390
8391 OSArray *
getDeviceMemory(void)8392 IOService::getDeviceMemory( void )
8393 {
8394 return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8395 }
8396
8397
8398 void
setDeviceMemory(OSArray * array)8399 IOService::setDeviceMemory( OSArray * array )
8400 {
8401 setProperty( gIODeviceMemoryKey, array);
8402 }
8403
8404 static void
requireMaxCpuDelay(IOService * service,UInt32 ns,UInt32 delayType)8405 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
8406 {
8407 static const UInt kNoReplace = -1U; // Must be an illegal index
8408 UInt replace = kNoReplace;
8409 bool setCpuDelay = false;
8410
8411 IORecursiveLockLock(sCpuDelayLock);
8412
8413 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8414 __typed_allocators_ignore_push
8415 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8416 __typed_allocators_ignore_pop
8417 IOService * holder = NULL;
8418
8419 if (ns) {
8420 const CpuDelayEntry ne = {service, ns, delayType};
8421 holder = service;
8422 // Set maximum delay.
8423 for (UInt i = 0; i < count; i++) {
8424 IOService *thisService = entries[i].fService;
8425 bool sameType = (delayType == entries[i].fDelayType);
8426 if ((service == thisService) && sameType) {
8427 replace = i;
8428 } else if (!thisService) {
8429 if (kNoReplace == replace) {
8430 replace = i;
8431 }
8432 } else if (sameType) {
8433 const UInt32 thisMax = entries[i].fMaxDelay;
8434 if (thisMax < ns) {
8435 ns = thisMax;
8436 holder = thisService;
8437 }
8438 }
8439 }
8440
8441 setCpuDelay = true;
8442 if (kNoReplace == replace) {
8443 __typed_allocators_ignore_push
8444 sCpuDelayData->appendBytes(&ne, sizeof(ne));
8445 __typed_allocators_ignore_pop
8446 } else {
8447 entries[replace] = ne;
8448 }
8449 } else {
8450 ns = -1U; // Set to max unsigned, i.e. no restriction
8451
8452 for (UInt i = 0; i < count; i++) {
8453 // Clear a maximum delay.
8454 IOService *thisService = entries[i].fService;
8455 if (thisService && (delayType == entries[i].fDelayType)) {
8456 UInt32 thisMax = entries[i].fMaxDelay;
8457 if (service == thisService) {
8458 replace = i;
8459 } else if (thisMax < ns) {
8460 ns = thisMax;
8461 holder = thisService;
8462 }
8463 }
8464 }
8465
8466 // Check if entry found
8467 if (kNoReplace != replace) {
8468 entries[replace].fService = NULL; // Null the entry
8469 setCpuDelay = true;
8470 }
8471 }
8472
8473 if (setCpuDelay) {
8474 if (holder && debug_boot_arg) {
8475 strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
8476 }
8477
8478 // Must be safe to call from locked context
8479 if (delayType == kCpuDelayBusStall) {
8480 #if defined(__x86_64__)
8481 ml_set_maxbusdelay(ns);
8482 #endif /* defined(__x86_64__) */
8483 }
8484 #if defined(__x86_64__)
8485 else if (delayType == kCpuDelayInterrupt) {
8486 ml_set_maxintdelay(ns);
8487 }
8488 #endif /* defined(__x86_64__) */
8489 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
8490 sCPULatencySet[delayType]->setValue(ns);
8491
8492 OSArray * handlers = sCpuLatencyHandlers[delayType];
8493 IOService * target;
8494 if (handlers) {
8495 for (unsigned int idx = 0;
8496 (target = (IOService *) handlers->getObject(idx));
8497 idx++) {
8498 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8499 (void *) (uintptr_t) ns, holder,
8500 NULL, NULL);
8501 }
8502 }
8503 }
8504
8505 IORecursiveLockUnlock(sCpuDelayLock);
8506 }
8507
8508 static IOReturn
setLatencyHandler(UInt32 delayType,IOService * target,bool enable)8509 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
8510 {
8511 IOReturn result = kIOReturnNotFound;
8512 OSArray * array;
8513 unsigned int idx;
8514
8515 IORecursiveLockLock(sCpuDelayLock);
8516
8517 do{
8518 if (enable && !sCpuLatencyHandlers[delayType]) {
8519 sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
8520 }
8521 array = sCpuLatencyHandlers[delayType];
8522 if (!array) {
8523 break;
8524 }
8525 idx = array->getNextIndexOfObject(target, 0);
8526 if (!enable) {
8527 if (-1U != idx) {
8528 array->removeObject(idx);
8529 result = kIOReturnSuccess;
8530 }
8531 } else {
8532 if (-1U != idx) {
8533 result = kIOReturnExclusiveAccess;
8534 break;
8535 }
8536 array->setObject(target);
8537
8538 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8539 __typed_allocators_ignore_push
8540 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8541 __typed_allocators_ignore_pop
8542 UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
8543 IOService * holder = NULL;
8544
8545 for (UInt i = 0; i < count; i++) {
8546 if (entries[i].fService
8547 && (delayType == entries[i].fDelayType)
8548 && (entries[i].fMaxDelay < ns)) {
8549 ns = entries[i].fMaxDelay;
8550 holder = entries[i].fService;
8551 }
8552 }
8553 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8554 (void *) (uintptr_t) ns, holder,
8555 NULL, NULL);
8556 result = kIOReturnSuccess;
8557 }
8558 }while (false);
8559
8560 IORecursiveLockUnlock(sCpuDelayLock);
8561
8562 return result;
8563 }
8564
8565 IOReturn
requireMaxBusStall(UInt32 ns)8566 IOService::requireMaxBusStall(UInt32 ns)
8567 {
8568 #if !defined(__x86_64__)
8569 switch (ns) {
8570 case kIOMaxBusStall40usec:
8571 case kIOMaxBusStall30usec:
8572 case kIOMaxBusStall25usec:
8573 case kIOMaxBusStall20usec:
8574 case kIOMaxBusStall10usec:
8575 case kIOMaxBusStall5usec:
8576 case kIOMaxBusStallNone:
8577 break;
8578 default:
8579 return kIOReturnBadArgument;
8580 }
8581 #endif /* !defined(__x86_64__) */
8582 requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
8583 return kIOReturnSuccess;
8584 }
8585
8586 IOReturn
requireMaxInterruptDelay(uint32_t ns)8587 IOService::requireMaxInterruptDelay(uint32_t ns)
8588 {
8589 #if defined(__x86_64__)
8590 requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
8591 return kIOReturnSuccess;
8592 #else /* defined(__x86_64__) */
8593 return kIOReturnUnsupported;
8594 #endif /* defined(__x86_64__) */
8595 }
8596
8597 /*
8598 * Device interrupts
8599 */
8600
8601 IOReturn
resolveInterrupt(IOService * nub,int source)8602 IOService::resolveInterrupt(IOService *nub, int source)
8603 {
8604 IOInterruptController *interruptController;
8605 OSArray *array;
8606 OSData *data;
8607 OSSymbol *interruptControllerName;
8608 unsigned int numSources;
8609 IOInterruptSource *interruptSources;
8610 IOInterruptSourcePrivate *interruptSourcesPrivate;
8611
8612 // Get the parents list from the nub.
8613 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
8614 if (array == NULL) {
8615 return kIOReturnNoResources;
8616 }
8617
8618 // Allocate space for the IOInterruptSources if needed... then return early.
8619 if (nub->_interruptSources == NULL) {
8620 numSources = array->getCount();
8621 interruptSources = IONewZero(IOInterruptSource, numSources);
8622 interruptSourcesPrivate = IONewZero(IOInterruptSourcePrivate, numSources);
8623
8624 if (interruptSources == NULL || interruptSourcesPrivate == NULL) {
8625 IODelete(interruptSources, IOInterruptSource, numSources);
8626 IODelete(interruptSourcesPrivate, IOInterruptSourcePrivate, numSources);
8627 return kIOReturnNoMemory;
8628 }
8629
8630 nub->_numInterruptSources = numSources;
8631 nub->_interruptSources = interruptSources;
8632 nub->reserved->interruptSourcesPrivate = interruptSourcesPrivate;
8633 return kIOReturnSuccess;
8634 }
8635
8636 interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
8637 if (interruptControllerName == NULL) {
8638 return kIOReturnNoResources;
8639 }
8640
8641 interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
8642 if (interruptController == NULL) {
8643 return kIOReturnNoResources;
8644 }
8645
8646 // Get the interrupt numbers from the nub.
8647 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
8648 if (array == NULL) {
8649 return kIOReturnNoResources;
8650 }
8651 data = OSDynamicCast(OSData, array->getObject(source));
8652 if (data == NULL) {
8653 return kIOReturnNoResources;
8654 }
8655
8656 // Set the interruptController and interruptSource in the nub's table.
8657 interruptSources = nub->_interruptSources;
8658 interruptSources[source].interruptController = interruptController;
8659 interruptSources[source].vectorData = data;
8660
8661 return kIOReturnSuccess;
8662 }
8663
8664 IOReturn
lookupInterrupt(int source,bool resolve,IOInterruptController ** interruptController)8665 IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
8666 {
8667 IOReturn ret;
8668
8669 /* Make sure the _interruptSources are set */
8670 if (_interruptSources == NULL) {
8671 ret = resolveInterrupt(this, source);
8672 if (ret != kIOReturnSuccess) {
8673 return ret;
8674 }
8675 }
8676
8677 /* Make sure the local source number is valid */
8678 if ((source < 0) || (source >= _numInterruptSources)) {
8679 return kIOReturnNoInterrupt;
8680 }
8681
8682 /* Look up the contoller for the local source */
8683 *interruptController = _interruptSources[source].interruptController;
8684
8685 if (*interruptController == NULL) {
8686 if (!resolve) {
8687 return kIOReturnNoInterrupt;
8688 }
8689
8690 /* Try to resolve the interrupt */
8691 ret = resolveInterrupt(this, source);
8692 if (ret != kIOReturnSuccess) {
8693 return ret;
8694 }
8695
8696 *interruptController = _interruptSources[source].interruptController;
8697 }
8698
8699 return kIOReturnSuccess;
8700 }
8701
8702 IOReturn
registerInterrupt(int source,OSObject * target,IOInterruptAction handler,void * refCon)8703 IOService::registerInterrupt(int source, OSObject *target,
8704 IOInterruptAction handler,
8705 void *refCon)
8706 {
8707 IOInterruptController *interruptController;
8708 IOReturn ret;
8709
8710 ret = lookupInterrupt(source, true, &interruptController);
8711 if (ret != kIOReturnSuccess) {
8712 return ret;
8713 }
8714
8715 /* Register the source */
8716 return interruptController->registerInterrupt(this, source, target,
8717 (IOInterruptHandler)handler,
8718 refCon);
8719 }
8720
8721 static void
IOServiceInterruptActionToBlock(OSObject * target,void * refCon,IOService * nub,int source)8722 IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
8723 IOService * nub, int source )
8724 {
8725 ((IOInterruptActionBlock)(refCon))(nub, source);
8726 }
8727
8728 IOReturn
registerInterruptBlock(int source,OSObject * target,IOInterruptActionBlock handler)8729 IOService::registerInterruptBlock(int source, OSObject *target,
8730 IOInterruptActionBlock handler)
8731 {
8732 IOReturn ret;
8733 void * block;
8734
8735 block = Block_copy(handler);
8736 if (!block) {
8737 return kIOReturnNoMemory;
8738 }
8739
8740 ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
8741 if (kIOReturnSuccess != ret) {
8742 Block_release(block);
8743 return ret;
8744 }
8745
8746 reserved->interruptSourcesPrivate[source].vectorBlock = block;
8747
8748 return ret;
8749 }
8750
8751 IOReturn
unregisterInterrupt(int source)8752 IOService::unregisterInterrupt(int source)
8753 {
8754 IOReturn ret;
8755 IOInterruptController *interruptController;
8756 IOInterruptSourcePrivate *priv;
8757 void *block;
8758
8759 ret = lookupInterrupt(source, false, &interruptController);
8760 if (ret != kIOReturnSuccess) {
8761 return ret;
8762 }
8763
8764 /* Unregister the source */
8765 priv = &reserved->interruptSourcesPrivate[source];
8766 block = priv->vectorBlock;
8767 ret = interruptController->unregisterInterrupt(this, source);
8768 if ((kIOReturnSuccess == ret) && (block = priv->vectorBlock)) {
8769 priv->vectorBlock = NULL;
8770 Block_release(block);
8771 }
8772
8773 return ret;
8774 }
8775
8776 void
unregisterAllInterrupts(void)8777 IOService::unregisterAllInterrupts(void)
8778 {
8779 for (int source = 0; source < _numInterruptSources; source++) {
8780 (void) unregisterInterrupt(source);
8781 }
8782 }
8783
8784 IOReturn
addInterruptStatistics(IOInterruptAccountingData * statistics,int source)8785 IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
8786 {
8787 IOReportLegend * legend = NULL;
8788 IOInterruptAccountingData * oldValue = NULL;
8789 IOInterruptAccountingReporter * newArray = NULL;
8790 char subgroupName[64];
8791 int newArraySize = 0;
8792 int i = 0;
8793
8794 if (source < 0) {
8795 return kIOReturnBadArgument;
8796 }
8797
8798 /*
8799 * We support statistics on a maximum of 256 interrupts per nub; if a nub
8800 * has more than 256 interrupt specifiers associated with it, and tries
8801 * to register a high interrupt index with interrupt accounting, panic.
8802 * Having more than 256 interrupts associated with a single nub is
8803 * probably a sign that something fishy is going on.
8804 */
8805 if (source > IA_INDEX_MAX) {
8806 panic("addInterruptStatistics called for an excessively large index (%d)", source);
8807 }
8808
8809 /*
8810 * TODO: This is ugly (wrapping a lock around an allocation). I'm only
8811 * leaving it as is because the likelihood of contention where we are
8812 * actually growing the array is minimal (we would realistically need
8813 * to be starting a driver for the first time, with an IOReporting
8814 * client already in place). Nonetheless, cleanup that can be done
8815 * to adhere to best practices; it'll make the code more complicated,
8816 * unfortunately.
8817 */
8818 IOLockLock(&reserved->interruptStatisticsLock);
8819
8820 /*
8821 * Lazily allocate the statistics array.
8822 */
8823 if (!reserved->interruptStatisticsArray) {
8824 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
8825 assert(reserved->interruptStatisticsArray);
8826 reserved->interruptStatisticsArrayCount = 1;
8827 bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
8828 }
8829
8830 if (source >= reserved->interruptStatisticsArrayCount) {
8831 /*
8832 * We're still within the range of supported indices, but we are out
8833 * of space in the current array. Do a nasty realloc (because
8834 * IORealloc isn't a thing) here. We'll double the size with each
8835 * reallocation.
8836 *
8837 * Yes, the "next power of 2" could be more efficient; but this will
8838 * be invoked incredibly rarely. Who cares.
8839 */
8840 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
8841
8842 while (newArraySize <= source) {
8843 newArraySize = (newArraySize << 1);
8844 }
8845 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
8846
8847 assert(newArray);
8848
8849 /*
8850 * TODO: This even zeroes the memory it is about to overwrite.
8851 * Shameful; fix it. Not particularly high impact, however.
8852 */
8853 bzero(newArray, newArraySize * sizeof(*newArray));
8854 memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
8855 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
8856 reserved->interruptStatisticsArray = newArray;
8857 reserved->interruptStatisticsArrayCount = newArraySize;
8858 }
8859
8860 if (!reserved->interruptStatisticsArray[source].reporter) {
8861 /*
8862 * We don't have a reporter associated with this index yet, so we
8863 * need to create one.
8864 */
8865 /*
8866 * TODO: Some statistics do in fact have common units (time); should this be
8867 * split into separate reporters to communicate this?
8868 */
8869 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
8870
8871 /*
8872 * Each statistic is given an identifier based on the interrupt index (which
8873 * should be unique relative to any single nub) and the statistic involved.
8874 * We should now have a sane (small and positive) index, so start
8875 * constructing the channels for statistics.
8876 */
8877 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
8878 /*
8879 * TODO: Currently, this does not add channels for disabled statistics.
8880 * Will this be confusing for clients? If so, we should just add the
8881 * channels; we can avoid updating the channels even if they exist.
8882 */
8883 if (IA_GET_STATISTIC_ENABLED(i)) {
8884 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
8885 }
8886 }
8887
8888 /*
8889 * We now need to add the legend for this reporter to the registry.
8890 */
8891 OSObject * prop = copyProperty(kIOReportLegendKey);
8892 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
8893 OSSafeReleaseNULL(prop);
8894
8895 /*
8896 * Note that while we compose the subgroup name, we do not need to
8897 * manage its lifecycle (the reporter will handle this).
8898 */
8899 snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
8900 subgroupName[sizeof(subgroupName) - 1] = 0;
8901 legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
8902 setProperty(kIOReportLegendKey, legend->getLegend());
8903 legend->release();
8904
8905 /*
8906 * TODO: Is this a good idea? Probably not; my assumption is it opts
8907 * all entities who register interrupts into public disclosure of all
8908 * IOReporting channels. Unfortunately, this appears to be as fine
8909 * grain as it gets.
8910 */
8911 setProperty(kIOReportLegendPublicKey, true);
8912 }
8913
8914 /*
8915 * Don't stomp existing entries. If we are about to, panic; this
8916 * probably means we failed to tear down our old interrupt source
8917 * correctly.
8918 */
8919 oldValue = reserved->interruptStatisticsArray[source].statistics;
8920
8921 if (oldValue) {
8922 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
8923 }
8924
8925 reserved->interruptStatisticsArray[source].statistics = statistics;
8926
8927 /*
8928 * Inherit the reporter values for each statistic. The target may
8929 * be torn down as part of the runtime of the service (especially
8930 * for sleep/wake), so we inherit in order to avoid having values
8931 * reset for no apparent reason. Our statistics are ultimately
8932 * tied to the index and the sevice, not to an individual target,
8933 * so we should maintain them accordingly.
8934 */
8935 interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8936
8937 IOLockUnlock(&reserved->interruptStatisticsLock);
8938
8939 return kIOReturnSuccess;
8940 }
8941
8942 IOReturn
removeInterruptStatistics(int source)8943 IOService::removeInterruptStatistics(int source)
8944 {
8945 IOInterruptAccountingData * value = NULL;
8946
8947 if (source < 0) {
8948 return kIOReturnBadArgument;
8949 }
8950
8951 IOLockLock(&reserved->interruptStatisticsLock);
8952
8953 /*
8954 * We dynamically grow the statistics array, so an excessively
8955 * large index value has NEVER been registered. This either
8956 * means our cap on the array size is too small (unlikely), or
8957 * that we have been passed a corrupt index (this must be passed
8958 * the plain index into the interrupt specifier list).
8959 */
8960 if (source >= reserved->interruptStatisticsArrayCount) {
8961 panic("removeInterruptStatistics called for index %d, which was never registered", source);
8962 }
8963
8964 assert(reserved->interruptStatisticsArray);
8965
8966 /*
8967 * If there is no existing entry, we are most likely trying to
8968 * free an interrupt owner twice, or we have corrupted the
8969 * index value.
8970 */
8971 value = reserved->interruptStatisticsArray[source].statistics;
8972
8973 if (!value) {
8974 panic("removeInterruptStatistics called for empty index %d", source);
8975 }
8976
8977 /*
8978 * We update the statistics, so that any delta with the reporter
8979 * state is not lost.
8980 */
8981 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8982 reserved->interruptStatisticsArray[source].statistics = NULL;
8983 IOLockUnlock(&reserved->interruptStatisticsLock);
8984
8985 return kIOReturnSuccess;
8986 }
8987
8988 IOReturn
getInterruptType(int source,int * interruptType)8989 IOService::getInterruptType(int source, int *interruptType)
8990 {
8991 IOInterruptController *interruptController;
8992 IOReturn ret;
8993
8994 ret = lookupInterrupt(source, true, &interruptController);
8995 if (ret != kIOReturnSuccess) {
8996 return ret;
8997 }
8998
8999 /* Return the type */
9000 return interruptController->getInterruptType(this, source, interruptType);
9001 }
9002
9003 IOReturn
enableInterrupt(int source)9004 IOService::enableInterrupt(int source)
9005 {
9006 IOInterruptController *interruptController;
9007 IOReturn ret;
9008
9009 ret = lookupInterrupt(source, false, &interruptController);
9010 if (ret != kIOReturnSuccess) {
9011 return ret;
9012 }
9013
9014 /* Enable the source */
9015 return interruptController->enableInterrupt(this, source);
9016 }
9017
9018 IOReturn
disableInterrupt(int source)9019 IOService::disableInterrupt(int source)
9020 {
9021 IOInterruptController *interruptController;
9022 IOReturn ret;
9023
9024 ret = lookupInterrupt(source, false, &interruptController);
9025 if (ret != kIOReturnSuccess) {
9026 return ret;
9027 }
9028
9029 /* Disable the source */
9030 return interruptController->disableInterrupt(this, source);
9031 }
9032
9033 IOReturn
causeInterrupt(int source)9034 IOService::causeInterrupt(int source)
9035 {
9036 IOInterruptController *interruptController;
9037 IOReturn ret;
9038
9039 ret = lookupInterrupt(source, false, &interruptController);
9040 if (ret != kIOReturnSuccess) {
9041 return ret;
9042 }
9043
9044 /* Cause an interrupt for the source */
9045 return interruptController->causeInterrupt(this, source);
9046 }
9047
9048 IOReturn
configureReport(IOReportChannelList * channelList,IOReportConfigureAction action,void * result,void * destination)9049 IOService::configureReport(IOReportChannelList *channelList,
9050 IOReportConfigureAction action,
9051 void *result,
9052 void *destination)
9053 {
9054 unsigned cnt;
9055
9056 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9057 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9058 if (pwrMgt) {
9059 configurePowerStatesReport(action, result);
9060 } else {
9061 return kIOReturnUnsupported;
9062 }
9063 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9064 if (pwrMgt) {
9065 configureSimplePowerReport(action, result);
9066 } else {
9067 return kIOReturnUnsupported;
9068 }
9069 }
9070 }
9071
9072 IOLockLock(&reserved->interruptStatisticsLock);
9073
9074 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9075 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9076 if (reserved->interruptStatisticsArray[cnt].reporter) {
9077 /*
9078 * If the reporter is currently associated with the statistics
9079 * for an event source, we may need to update the reporter.
9080 */
9081 if (reserved->interruptStatisticsArray[cnt].statistics) {
9082 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9083 }
9084
9085 reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
9086 }
9087 }
9088
9089 IOLockUnlock(&reserved->interruptStatisticsLock);
9090
9091 if (hasUserServer()) {
9092 return _ConfigureReport(channelList, action, result, destination);
9093 } else {
9094 return kIOReturnSuccess;
9095 }
9096 }
9097
9098 IOReturn
updateReport(IOReportChannelList * channelList,IOReportUpdateAction action,void * result,void * destination)9099 IOService::updateReport(IOReportChannelList *channelList,
9100 IOReportUpdateAction action,
9101 void *result,
9102 void *destination)
9103 {
9104 unsigned cnt;
9105
9106 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9107 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9108 if (pwrMgt) {
9109 updatePowerStatesReport(action, result, destination);
9110 } else {
9111 return kIOReturnUnsupported;
9112 }
9113 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9114 if (pwrMgt) {
9115 updateSimplePowerReport(action, result, destination);
9116 } else {
9117 return kIOReturnUnsupported;
9118 }
9119 }
9120 }
9121
9122 IOLockLock(&reserved->interruptStatisticsLock);
9123
9124 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9125 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9126 if (reserved->interruptStatisticsArray[cnt].reporter) {
9127 /*
9128 * If the reporter is currently associated with the statistics
9129 * for an event source, we need to update the reporter.
9130 */
9131 if (reserved->interruptStatisticsArray[cnt].statistics) {
9132 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9133 }
9134
9135 reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
9136 }
9137 }
9138
9139 IOLockUnlock(&reserved->interruptStatisticsLock);
9140
9141
9142 if (hasUserServer()) {
9143 return _UpdateReport(channelList, action, result, destination);
9144 } else {
9145 return kIOReturnSuccess;
9146 }
9147 }
9148
9149 uint64_t
getAuthorizationID(void)9150 IOService::getAuthorizationID( void )
9151 {
9152 return reserved->authorizationID;
9153 }
9154
9155 IOReturn
setAuthorizationID(uint64_t authorizationID)9156 IOService::setAuthorizationID( uint64_t authorizationID )
9157 {
9158 OSObject * entitlement;
9159 IOReturn status;
9160
9161 entitlement = IOUserClient::copyClientEntitlement( current_task(), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
9162
9163 if (entitlement) {
9164 if (entitlement == kOSBooleanTrue) {
9165 reserved->authorizationID = authorizationID;
9166
9167 status = kIOReturnSuccess;
9168 } else {
9169 status = kIOReturnNotPrivileged;
9170 }
9171
9172 entitlement->release();
9173 } else {
9174 status = kIOReturnNotPrivileged;
9175 }
9176
9177 return status;
9178 }
9179
9180 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9181
9182 #if __LP64__
9183 OSMetaClassDefineReservedUsedX86(IOService, 0);
9184 OSMetaClassDefineReservedUsedX86(IOService, 1);
9185 OSMetaClassDefineReservedUnused(IOService, 2);
9186 OSMetaClassDefineReservedUnused(IOService, 3);
9187 OSMetaClassDefineReservedUnused(IOService, 4);
9188 OSMetaClassDefineReservedUnused(IOService, 5);
9189 OSMetaClassDefineReservedUnused(IOService, 6);
9190 OSMetaClassDefineReservedUnused(IOService, 7);
9191 #else
9192 OSMetaClassDefineReservedUsedX86(IOService, 0);
9193 OSMetaClassDefineReservedUsedX86(IOService, 1);
9194 OSMetaClassDefineReservedUsedX86(IOService, 2);
9195 OSMetaClassDefineReservedUsedX86(IOService, 3);
9196 OSMetaClassDefineReservedUsedX86(IOService, 4);
9197 OSMetaClassDefineReservedUsedX86(IOService, 5);
9198 OSMetaClassDefineReservedUsedX86(IOService, 6);
9199 OSMetaClassDefineReservedUsedX86(IOService, 7);
9200 #endif
9201 OSMetaClassDefineReservedUnused(IOService, 8);
9202 OSMetaClassDefineReservedUnused(IOService, 9);
9203 OSMetaClassDefineReservedUnused(IOService, 10);
9204 OSMetaClassDefineReservedUnused(IOService, 11);
9205 OSMetaClassDefineReservedUnused(IOService, 12);
9206 OSMetaClassDefineReservedUnused(IOService, 13);
9207 OSMetaClassDefineReservedUnused(IOService, 14);
9208 OSMetaClassDefineReservedUnused(IOService, 15);
9209 OSMetaClassDefineReservedUnused(IOService, 16);
9210 OSMetaClassDefineReservedUnused(IOService, 17);
9211 OSMetaClassDefineReservedUnused(IOService, 18);
9212 OSMetaClassDefineReservedUnused(IOService, 19);
9213 OSMetaClassDefineReservedUnused(IOService, 20);
9214 OSMetaClassDefineReservedUnused(IOService, 21);
9215 OSMetaClassDefineReservedUnused(IOService, 22);
9216 OSMetaClassDefineReservedUnused(IOService, 23);
9217 OSMetaClassDefineReservedUnused(IOService, 24);
9218 OSMetaClassDefineReservedUnused(IOService, 25);
9219 OSMetaClassDefineReservedUnused(IOService, 26);
9220 OSMetaClassDefineReservedUnused(IOService, 27);
9221 OSMetaClassDefineReservedUnused(IOService, 28);
9222 OSMetaClassDefineReservedUnused(IOService, 29);
9223 OSMetaClassDefineReservedUnused(IOService, 30);
9224 OSMetaClassDefineReservedUnused(IOService, 31);
9225 OSMetaClassDefineReservedUnused(IOService, 32);
9226 OSMetaClassDefineReservedUnused(IOService, 33);
9227 OSMetaClassDefineReservedUnused(IOService, 34);
9228 OSMetaClassDefineReservedUnused(IOService, 35);
9229 OSMetaClassDefineReservedUnused(IOService, 36);
9230 OSMetaClassDefineReservedUnused(IOService, 37);
9231 OSMetaClassDefineReservedUnused(IOService, 38);
9232 OSMetaClassDefineReservedUnused(IOService, 39);
9233 OSMetaClassDefineReservedUnused(IOService, 40);
9234 OSMetaClassDefineReservedUnused(IOService, 41);
9235 OSMetaClassDefineReservedUnused(IOService, 42);
9236 OSMetaClassDefineReservedUnused(IOService, 43);
9237 OSMetaClassDefineReservedUnused(IOService, 44);
9238 OSMetaClassDefineReservedUnused(IOService, 45);
9239 OSMetaClassDefineReservedUnused(IOService, 46);
9240 OSMetaClassDefineReservedUnused(IOService, 47);
9241