1 /*
2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define IOKIT_ENABLE_SHARED_PTR
30
31 #include <ptrauth.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOKitDebug.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOService.h>
36 #include <IOKit/IOInterrupts.h>
37 #include <IOKit/IOTimeStamp.h>
38 #include <IOKit/IOWorkLoop.h>
39 #include <IOKit/IOInterruptAccountingPrivate.h>
40 #include <libkern/Block_private.h>
41
42 #if IOKITSTATS
43
44 #define IOStatisticsInitializeCounter() \
45 do { \
46 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
47 } while (0)
48
49 #define IOStatisticsCheckForWork() \
50 do { \
51 IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
52 } while (0)
53
54 #define IOStatisticsInterrupt() \
55 do { \
56 IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
57 } while (0)
58
59 #else
60
61 #define IOStatisticsInitializeCounter()
62 #define IOStatisticsCheckForWork()
63 #define IOStatisticsInterrupt()
64
65 #endif // IOKITSTATS
66
67 #define super IOEventSource
68
69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
78
79 bool
init(OSObject * inOwner,Action inAction,IOService * inProvider,int inIntIndex)80 IOInterruptEventSource::init(OSObject *inOwner,
81 Action inAction,
82 IOService *inProvider,
83 int inIntIndex)
84 {
85 bool res = true;
86
87 if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
88 return false;
89 }
90
91 reserved = IOMallocType(ExpansionData);
92
93 provider = inProvider;
94 producerCount = consumerCount = 0;
95 autoDisable = explicitDisable = false;
96 intIndex = ~inIntIndex;
97
98 // Assumes inOwner holds a reference(retain) on the provider
99 if (inProvider) {
100 if (IA_ANY_STATISTICS_ENABLED) {
101 /*
102 * We only treat this as an "interrupt" if it has a provider; if it does,
103 * set up the objects necessary to track interrupt statistics. Interrupt
104 * event sources without providers are most likely being used as simple
105 * event source in order to poke at workloops and kick off work.
106 *
107 * We also avoid try to avoid interrupt accounting overhead if none of
108 * the statistics are enabled.
109 */
110 reserved->statistics = IOMallocType(IOInterruptAccountingData);
111
112 reserved->statistics->owner = this;
113 }
114
115 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
116
117 if (res) {
118 intIndex = inIntIndex;
119 }
120 }
121
122 IOStatisticsInitializeCounter();
123
124 return res;
125 }
126
127 IOReturn
registerInterruptHandler(IOService * inProvider,int inIntIndex)128 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
129 int inIntIndex)
130 {
131 IOReturn ret;
132 int intType;
133 IOInterruptAction intHandler;
134
135 ret = inProvider->getInterruptType(inIntIndex, &intType);
136 if (kIOReturnSuccess != ret) {
137 return ret;
138 }
139
140 autoDisable = (intType == kIOInterruptTypeLevel);
141 if (autoDisable) {
142 intHandler = OSMemberFunctionCast(IOInterruptAction,
143 this, &IOInterruptEventSource::disableInterruptOccurred);
144 } else {
145 intHandler = OSMemberFunctionCast(IOInterruptAction,
146 this, &IOInterruptEventSource::normalInterruptOccurred);
147 }
148
149 ret = provider->registerInterrupt(inIntIndex, this, intHandler);
150
151 /*
152 * Add statistics to the provider. The setWorkLoop convention should ensure
153 * that we always go down the unregister path before we register (outside of
154 * init()), so we don't have to worry that we will invoke addInterruptStatistics
155 * erroneously.
156 */
157 if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
158 /*
159 * Stash the normal index value, for the sake of debugging.
160 */
161 reserved->statistics->interruptIndex = inIntIndex;
162
163 /*
164 * We need to hook the interrupt information up to the provider so that it
165 * can find the statistics for this interrupt when desired. The provider is
166 * responsible for maintaining the reporter for a particular interrupt, and
167 * needs a handle on the statistics so that it can request that the reporter
168 * be updated as needed. Errors are considered "soft" for the moment (it
169 * will either panic, or fail in a way such that we can still service the
170 * interrupt).
171 */
172 provider->addInterruptStatistics(reserved->statistics, inIntIndex);
173
174 /*
175 * Add the statistics object to the global list of statistics objects; this
176 * is an aid to debugging (we can trivially find statistics for all eligible
177 * interrupts, and dump them; potentially helpful if the system is wedged
178 * due to interrupt activity).
179 */
180 interruptAccountingDataAddToList(reserved->statistics);
181 }
182
183 return ret;
184 }
185
186 void
unregisterInterruptHandler(IOService * inProvider,int inIntIndex)187 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
188 int inIntIndex)
189 {
190 if (reserved->statistics) {
191 interruptAccountingDataRemoveFromList(reserved->statistics);
192 provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
193 }
194
195 provider->unregisterInterrupt(inIntIndex);
196 }
197
198
199 OSSharedPtr<IOInterruptEventSource>
interruptEventSource(OSObject * inOwner,Action inAction,IOService * inProvider,int inIntIndex)200 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
201 Action inAction,
202 IOService *inProvider,
203 int inIntIndex)
204 {
205 OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>();
206
207 if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
208 return nullptr;
209 }
210
211 return me;
212 }
213
214 OSSharedPtr<IOInterruptEventSource>
interruptEventSource(OSObject * inOwner,IOService * inProvider,int inIntIndex,ActionBlock inAction)215 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
216 IOService *inProvider,
217 int inIntIndex,
218 ActionBlock inAction)
219 {
220 OSSharedPtr<IOInterruptEventSource> ies;
221 ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
222 if (ies) {
223 ies->setActionBlock((IOEventSource::ActionBlock) inAction);
224 }
225
226 return ies;
227 }
228
229 void
free()230 IOInterruptEventSource::free()
231 {
232 if (provider && intIndex >= 0) {
233 unregisterInterruptHandler(provider, intIndex);
234 }
235
236 if (reserved) {
237 if (reserved->statistics) {
238 IOFreeType(reserved->statistics, IOInterruptAccountingData);
239 }
240
241 IOFreeType(reserved, ExpansionData);
242 }
243
244 super::free();
245 }
246
247 void
enable()248 IOInterruptEventSource::enable()
249 {
250 if (provider && intIndex >= 0) {
251 provider->enableInterrupt(intIndex);
252 explicitDisable = false;
253 enabled = true;
254 }
255 }
256
257 void
disable()258 IOInterruptEventSource::disable()
259 {
260 if (provider && intIndex >= 0) {
261 provider->disableInterrupt(intIndex);
262 explicitDisable = true;
263 enabled = false;
264 }
265 }
266
267 void
setWorkLoop(IOWorkLoop * inWorkLoop)268 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
269 {
270 if (inWorkLoop) {
271 super::setWorkLoop(inWorkLoop);
272 }
273
274 if (provider) {
275 if (!inWorkLoop) {
276 if (intIndex >= 0) {
277 /*
278 * It isn't necessarily safe to wait until free() to unregister the interrupt;
279 * our provider may disappear.
280 */
281 unregisterInterruptHandler(provider, intIndex);
282 intIndex = ~intIndex;
283 }
284 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
285 intIndex = ~intIndex;
286 }
287 }
288
289 if (!inWorkLoop) {
290 super::setWorkLoop(inWorkLoop);
291 }
292 }
293
294 const IOService *
getProvider() const295 IOInterruptEventSource::getProvider() const
296 {
297 return provider;
298 }
299
300 int
getIntIndex() const301 IOInterruptEventSource::getIntIndex() const
302 {
303 return intIndex;
304 }
305
306 bool
getAutoDisable() const307 IOInterruptEventSource::getAutoDisable() const
308 {
309 return autoDisable;
310 }
311
312 bool
checkForWork()313 IOInterruptEventSource::checkForWork()
314 {
315 uint64_t startSystemTime = 0;
316 uint64_t endSystemTime = 0;
317 uint64_t startCPUTime = 0;
318 uint64_t endCPUTime = 0;
319 unsigned int cacheProdCount = producerCount;
320 int numInts = cacheProdCount - consumerCount;
321 IOEventSource::Action intAction = action;
322 ActionBlock intActionBlock = (ActionBlock) actionBlock;
323 void *address;
324 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
325
326 if (kActionBlock & flags) {
327 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock));
328 } else {
329 address = ptrauth_nop_cast(void *, intAction);
330 }
331
332 IOStatisticsCheckForWork();
333
334 if (numInts > 0) {
335 if (trace) {
336 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
337 VM_KERNEL_ADDRHIDE(address),
338 VM_KERNEL_ADDRHIDE(owner),
339 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
340 }
341
342 if (reserved->statistics) {
343 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
344 startSystemTime = mach_absolute_time();
345 }
346
347 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
348 startCPUTime = thread_get_runtime_self();
349 }
350 }
351
352 // Call the handler
353 if (kActionBlock & flags) {
354 (intActionBlock)(this, numInts);
355 } else {
356 ((IOInterruptEventAction)intAction)(owner, this, numInts);
357 }
358
359 if (reserved->statistics) {
360 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
361 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
362 }
363
364 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
365 endCPUTime = thread_get_runtime_self();
366 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
367 }
368
369 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
370 endSystemTime = mach_absolute_time();
371 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
372 }
373 }
374
375 if (trace) {
376 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
377 VM_KERNEL_ADDRHIDE(address),
378 VM_KERNEL_ADDRHIDE(owner),
379 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
380 }
381
382 consumerCount = cacheProdCount;
383 if (autoDisable && !explicitDisable) {
384 enable();
385 }
386 } else if (numInts < 0) {
387 if (trace) {
388 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
389 VM_KERNEL_ADDRHIDE(address),
390 VM_KERNEL_ADDRHIDE(owner),
391 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
392 }
393
394 if (reserved->statistics) {
395 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
396 startSystemTime = mach_absolute_time();
397 }
398
399 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
400 startCPUTime = thread_get_runtime_self();
401 }
402 }
403
404 // Call the handler
405 if (kActionBlock & flags) {
406 (intActionBlock)(this, numInts);
407 } else {
408 ((IOInterruptEventAction)intAction)(owner, this, numInts);
409 }
410
411 if (reserved->statistics) {
412 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
413 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
414 }
415
416 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
417 endCPUTime = thread_get_runtime_self();
418 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
419 }
420
421 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
422 endSystemTime = mach_absolute_time();
423 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
424 }
425 }
426
427 if (trace) {
428 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
429 VM_KERNEL_ADDRHIDE(address),
430 VM_KERNEL_ADDRHIDE(owner),
431 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
432 }
433
434 consumerCount = cacheProdCount;
435 if (autoDisable && !explicitDisable) {
436 enable();
437 }
438 }
439
440 return false;
441 }
442
443 void
normalInterruptOccurred(void *,IOService *,int)444 IOInterruptEventSource::normalInterruptOccurred
445 (void */*refcon*/, IOService */*prov*/, int /*source*/)
446 {
447 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
448
449 IOStatisticsInterrupt();
450 producerCount++;
451
452 if (trace) {
453 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
454 }
455
456 if (reserved->statistics) {
457 if (reserved->statistics->enablePrimaryTimestamp) {
458 reserved->statistics->primaryTimestamp = mach_absolute_time();
459 }
460 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
461 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
462 }
463 }
464
465 signalWorkAvailable();
466
467 if (trace) {
468 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
469 }
470 }
471
472 void
disableInterruptOccurred(void *,IOService * prov,int source)473 IOInterruptEventSource::disableInterruptOccurred
474 (void */*refcon*/, IOService *prov, int source)
475 {
476 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
477
478 prov->disableInterrupt(source); /* disable the interrupt */
479
480 IOStatisticsInterrupt();
481 producerCount++;
482
483 if (trace) {
484 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
485 }
486
487 if (reserved->statistics) {
488 if (reserved->statistics->enablePrimaryTimestamp) {
489 reserved->statistics->primaryTimestamp = mach_absolute_time();
490 }
491 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
492 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
493 }
494 }
495
496 signalWorkAvailable();
497
498 if (trace) {
499 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
500 }
501 }
502
503 void
interruptOccurred(void * _refcon,IOService * prov,int source)504 IOInterruptEventSource::interruptOccurred
505 (void *_refcon, IOService *prov, int source)
506 {
507 if (autoDisable && prov) {
508 disableInterruptOccurred(_refcon, prov, source);
509 } else {
510 normalInterruptOccurred(_refcon, prov, source);
511 }
512 }
513
514 IOReturn
warmCPU(uint64_t abstime)515 IOInterruptEventSource::warmCPU
516 (uint64_t abstime)
517 {
518 return ml_interrupt_prewarm(abstime);
519 }
520
521 void
enablePrimaryInterruptTimestamp(bool enable)522 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
523 {
524 if (reserved->statistics) {
525 reserved->statistics->enablePrimaryTimestamp = enable;
526 }
527 }
528
529 uint64_t
getPrimaryInterruptTimestamp()530 IOInterruptEventSource::getPrimaryInterruptTimestamp()
531 {
532 if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
533 return reserved->statistics->primaryTimestamp;
534 }
535 return -1ULL;
536 }
537