1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 */
4
5 #include <IOKit/perfcontrol/IOPerfControl.h>
6
7 #include <stdatomic.h>
8
9 #include <kern/thread_group.h>
10 #include <kern/task.h>
11 #include <sys/coalition.h>
12 #include <kern/coalition.h>
13
14 #undef super
15 #define super OSObject
16 OSDefineMetaClassAndStructors(IOPerfControlClient, OSObject);
17
18 static IOPerfControlClient::IOPerfControlClientShared *_Atomic gIOPerfControlClientShared;
19
20 bool
init(IOService * driver,uint64_t maxWorkCapacity)21 IOPerfControlClient::init(IOService *driver, uint64_t maxWorkCapacity)
22 {
23 // TODO: Remove this limit and implement dynamic table growth if workloads are found that exceed this
24 if (maxWorkCapacity > kMaxWorkTableNumEntries) {
25 maxWorkCapacity = kMaxWorkTableNumEntries;
26 }
27
28 if (!super::init()) {
29 return false;
30 }
31
32 shared = atomic_load_explicit(&gIOPerfControlClientShared, memory_order_acquire);
33 if (shared == nullptr) {
34 IOPerfControlClient::IOPerfControlClientShared *expected = shared;
35 shared = kalloc_type(IOPerfControlClientShared, Z_WAITOK);
36 if (!shared) {
37 return false;
38 }
39
40 atomic_init(&shared->maxDriverIndex, 0);
41
42 shared->interface = PerfControllerInterface{
43 .version = PERFCONTROL_INTERFACE_VERSION_NONE,
44 .registerDevice =
45 [](IOService *device) {
46 return kIOReturnSuccess;
47 },
48 .unregisterDevice =
49 [](IOService *device) {
50 return kIOReturnSuccess;
51 },
52 .workCanSubmit =
53 [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
54 return false;
55 },
56 .workSubmit =
57 [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
58 },
59 .workBegin =
60 [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
61 },
62 .workEnd =
63 [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
64 },
65 .workUpdate =
66 [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkUpdateArgs *args) {
67 },
68 };
69
70 shared->interfaceLock = IOLockAlloc();
71 if (!shared->interfaceLock) {
72 goto shared_init_error;
73 }
74
75 shared->deviceRegistrationList = OSSet::withCapacity(4);
76 if (!shared->deviceRegistrationList) {
77 goto shared_init_error;
78 }
79
80 if (!atomic_compare_exchange_strong_explicit(&gIOPerfControlClientShared, &expected, shared, memory_order_acq_rel,
81 memory_order_acquire)) {
82 IOLockFree(shared->interfaceLock);
83 shared->deviceRegistrationList->release();
84 kfree_type(IOPerfControlClientShared, shared);
85 shared = expected;
86 }
87 }
88 workTable = NULL;
89 workTableLock = NULL;
90
91 // Note: driverIndex is not guaranteed to be unique if maxDriverIndex wraps around. It is intended for debugging only.
92 driverIndex = atomic_fetch_add_explicit(&shared->maxDriverIndex, 1, memory_order_relaxed) + 1;
93
94 // + 1 since index 0 is unused for kIOPerfControlClientWorkUntracked
95 workTableLength = maxWorkCapacity + 1;
96 assertf(workTableLength <= kWorkTableMaxSize, "%zu exceeds max allowed capacity of %zu", workTableLength, kWorkTableMaxSize);
97 if (maxWorkCapacity > 0) {
98 workTable = kalloc_type(WorkTableEntry, workTableLength, Z_WAITOK_ZERO);
99 if (!workTable) {
100 goto error;
101 }
102 workTableNextIndex = 1;
103
104 workTableLock = IOSimpleLockAlloc();
105 if (!workTableLock) {
106 goto error;
107 }
108 }
109
110 bzero(&clientData, sizeof(clientData));
111
112 return true;
113
114 error:
115 if (workTable) {
116 kfree_type(WorkTableEntry, workTableLength, workTable);
117 workTable = NULL;
118 }
119 if (workTableLock) {
120 IOSimpleLockFree(workTableLock);
121 workTableLock = NULL;
122 }
123 return false;
124 shared_init_error:
125 if (shared) {
126 if (shared->interfaceLock) {
127 IOLockFree(shared->interfaceLock);
128 }
129 if (shared->deviceRegistrationList) {
130 shared->deviceRegistrationList->release();
131 }
132 kfree_type(IOPerfControlClientShared, shared);
133 shared = nullptr;
134 }
135 return false;
136 }
137
138 void
free()139 IOPerfControlClient::free()
140 {
141 if (workTable) {
142 kfree_type(WorkTableEntry, workTableLength, workTable);
143 }
144 if (workTableLock) {
145 IOSimpleLockFree(workTableLock);
146 }
147 super::free();
148 }
149
150 void
setDeviceType(IOPCDeviceType newDeviceType)151 IOPerfControlClient::setDeviceType(IOPCDeviceType newDeviceType)
152 {
153 if (newDeviceType >= IOPCDeviceTypeMax) {
154 panic("unknown device type %d", newDeviceType);
155 }
156
157 if (deviceType != IOPCDeviceTypeUnknown) {
158 panic("deviceType already set to %d", deviceType);
159 }
160
161 deviceType = newDeviceType;
162 }
163
164
165 IOPerfControlClient *
copyClientForDeviceType(IOService * driver,uint64_t maxWorkCapacity,IOPCDeviceType deviceType)166 IOPerfControlClient::copyClientForDeviceType(IOService *driver, uint64_t maxWorkCapacity, IOPCDeviceType deviceType)
167 {
168 IOPerfControlClient *client = new IOPerfControlClient;
169 if (!client || !client->init(driver, maxWorkCapacity)) {
170 panic("could not create IOPerfControlClient");
171 }
172 client->setDeviceType(deviceType);
173 return client;
174 }
175
176 IOPerfControlClient *
copyClient(IOService * driver,uint64_t maxWorkCapacity)177 IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
178 {
179 return copyClientForDeviceType(driver, maxWorkCapacity, IOPCDeviceTypeUnknown);
180 }
181
182 /* Convert the per driver token into a globally unique token for the performance
183 * controller's consumption. This is achieved by setting the driver's unique
184 * index onto the high order bits. The performance controller is shared between
185 * all drivers and must track all instances separately, while each driver has
186 * its own token table, so this step is needed to avoid token collisions between
187 * drivers.
188 */
189 inline uint64_t
tokenToGlobalUniqueToken(uint64_t token)190 IOPerfControlClient::tokenToGlobalUniqueToken(uint64_t token)
191 {
192 return token | (static_cast<uint64_t>(driverIndex) << kWorkTableIndexBits);
193 }
194
195 /* Accounting resources used in a work item to the containing coalition.
196 * Contigent upon the PerfController signaling that it wants resource accounting
197 * in the registerDevice()/registerDriverDevice calls. More device types can
198 * be added here in the future.
199 */
200 void
accountResources(coalition_t coal,PerfControllerInterface::PerfDeviceID device_type,PerfControllerInterface::ResourceAccounting * resources)201 IOPerfControlClient::accountResources(coalition_t coal, PerfControllerInterface::PerfDeviceID device_type, PerfControllerInterface::ResourceAccounting *resources)
202 {
203 switch (device_type) {
204 case PerfControllerInterface::PerfDeviceID::kANE:
205 if (coal != nullptr) {
206 coalition_update_ane_stats(coal, resources->mach_time_delta, resources->energy_nj_delta);
207 }
208 break;
209
210 default:
211 assertf(false, "Unexpected device type for IOPerfControlClient::accountResources: %llu", static_cast<uint64_t>(device_type));
212 }
213 }
214
215 /* With this implementation, tokens returned to the driver differ from tokens
216 * passed to the performance controller. This implementation has the nice
217 * property that tokens returns to the driver will aways be between 1 and
218 * the value of maxWorkCapacity passed by the driver to copyClient. The tokens
219 * the performance controller sees will match on the lower order bits and have
220 * the driver index set on the high order bits.
221 */
222 uint64_t
allocateToken(thread_group * thread_group)223 IOPerfControlClient::allocateToken(thread_group *thread_group)
224 {
225 uint64_t token = kIOPerfControlClientWorkUntracked;
226
227 #if CONFIG_THREAD_GROUPS
228 auto s = IOSimpleLockLockDisableInterrupt(workTableLock);
229
230 uint64_t num_tries = 0;
231 size_t index = workTableNextIndex;
232 // - 1 since entry 0 is for kIOPerfControlClientWorkUntracked
233 while (num_tries < workTableLength - 1) {
234 if (workTable[index].thread_group == nullptr) {
235 thread_group_retain(thread_group);
236 workTable[index].thread_group = thread_group;
237 if (clientData.driverState.resource_accounting) {
238 auto *coalition = task_get_coalition(current_task(), COALITION_TYPE_RESOURCE);
239 assert(coalition != nullptr);
240 coalition_retain(coalition);
241 workTable[index].coal = coalition;
242 }
243 token = index;
244 // next integer between 1 and workTableLength - 1
245 workTableNextIndex = (index % (workTableLength - 1)) + 1;
246 break;
247 }
248 // next integer between 1 and workTableLength - 1
249 index = (index % (workTableLength - 1)) + 1;
250 num_tries += 1;
251 }
252 #if (DEVELOPMENT || DEBUG)
253 if (token == kIOPerfControlClientWorkUntracked) {
254 /* When investigating a panic here, first check that the driver is not leaking tokens.
255 * If the driver is not leaking tokens and maximum is less than kMaxWorkTableNumEntries,
256 * the driver should be modified to pass a larger value to copyClient.
257 * If the driver is not leaking tokens and maximum is equal to kMaxWorkTableNumEntries,
258 * this code will have to be modified to support dynamic table growth to support larger
259 * numbers of tokens.
260 */
261 panic("Tokens allocated for this device exceeded maximum of %zu.",
262 workTableLength - 1); // - 1 since entry 0 is for kIOPerfControlClientWorkUntracked
263 }
264 #endif
265
266 IOSimpleLockUnlockEnableInterrupt(workTableLock, s);
267 #endif
268
269 return token;
270 }
271
272 void
deallocateToken(uint64_t token)273 IOPerfControlClient::deallocateToken(uint64_t token)
274 {
275 #if CONFIG_THREAD_GROUPS
276 assertf(token != kIOPerfControlClientWorkUntracked, "Attempt to deallocate token kIOPerfControlClientWorkUntracked\n");
277 assertf(token <= workTableLength, "Attempt to deallocate token %llu which is greater than the table size of %zu\n", token, workTableLength);
278 auto s = IOSimpleLockLockDisableInterrupt(workTableLock);
279
280 auto &entry = workTable[token];
281 auto *thread_group = entry.thread_group;
282 auto *coal = entry.coal;
283 bzero(&entry, sizeof(entry));
284 workTableNextIndex = token;
285
286 IOSimpleLockUnlockEnableInterrupt(workTableLock, s);
287
288 // This can call into the performance controller if the last reference is dropped here. Are we sure
289 // the driver isn't holding any locks? If not, we may want to async this to another context.
290 thread_group_release(thread_group);
291 if (coal != nullptr) {
292 coalition_release(coal);
293 }
294 #endif
295 }
296
297 IOPerfControlClient::WorkTableEntry *
getEntryForToken(uint64_t token)298 IOPerfControlClient::getEntryForToken(uint64_t token)
299 {
300 if (token == kIOPerfControlClientWorkUntracked) {
301 return nullptr;
302 }
303
304 if (token >= workTableLength) {
305 panic("Invalid work token (%llu): index out of bounds.", token);
306 }
307
308 WorkTableEntry *entry = &workTable[token];
309 assertf(entry->thread_group, "Invalid work token: %llu", token);
310 return entry;
311 }
312
313 void
markEntryStarted(uint64_t token,bool started)314 IOPerfControlClient::markEntryStarted(uint64_t token, bool started)
315 {
316 if (token == kIOPerfControlClientWorkUntracked) {
317 return;
318 }
319
320 if (token >= workTableLength) {
321 panic("Invalid work token (%llu): index out of bounds.", token);
322 }
323
324 workTable[token].started = started;
325 }
326
327 #if CONFIG_THREAD_GROUPS
328
329 static struct thread_group *
threadGroupForDextService(IOService * device)330 threadGroupForDextService(IOService *device)
331 {
332 assert(device);
333
334 if (!device->hasUserServer()) {
335 return NULL;
336 }
337
338 // Devices associated with a dext driver, must be called from dext
339 // context to ensure that thread_group reference is valid.
340 thread_t thread = current_thread();
341 assert(get_threadtask(thread) != kernel_task);
342 struct thread_group * thread_group = thread_group_get(thread);
343 assert(thread_group != nullptr);
344 return thread_group;
345 }
346
347 #endif /* CONFIG_THREAD_GROUPS */
348
349 IOReturn
registerDevice(IOService * driver,IOService * device)350 IOPerfControlClient::registerDevice(IOService *driver, IOService *device)
351 {
352 IOReturn ret = kIOReturnSuccess;
353 #if CONFIG_THREAD_GROUPS
354 IOLockLock(shared->interfaceLock);
355
356 clientData.device = device;
357
358 if (device) {
359 struct thread_group *dext_thread_group = threadGroupForDextService(device);
360 if (dext_thread_group) {
361 if (clientData.driverState.has_target_thread_group) {
362 panic("driverState has already been initialized");
363 }
364 clientData.driverState.has_target_thread_group = true;
365 clientData.driverState.target_thread_group_id = thread_group_get_id(dext_thread_group);
366 clientData.driverState.target_thread_group_data = thread_group_get_machine_data(dext_thread_group);
367
368 clientData.target_thread_group = dext_thread_group;
369 thread_group_retain(dext_thread_group);
370 }
371 }
372
373 if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_3) {
374 ret = shared->interface.registerDriverDevice(driver, device, &clientData.driverState);
375 } else if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_1) {
376 ret = shared->interface.registerDevice(device);
377 } else {
378 shared->deviceRegistrationList->setObject(this);
379 }
380
381 IOLockUnlock(shared->interfaceLock);
382 #endif
383 return ret;
384 }
385
386 void
unregisterDevice(IOService * driver,IOService * device)387 IOPerfControlClient::unregisterDevice(IOService *driver, IOService *device)
388 {
389 #if CONFIG_THREAD_GROUPS
390 IOLockLock(shared->interfaceLock);
391
392 if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_3) {
393 shared->interface.unregisterDriverDevice(driver, device, &clientData.driverState);
394 } else if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_1) {
395 shared->interface.unregisterDevice(device);
396 } else {
397 shared->deviceRegistrationList->removeObject(this);
398 }
399
400 if (clientData.driverState.has_target_thread_group) {
401 thread_group_release(clientData.target_thread_group);
402 clientData.target_thread_group = nullptr;
403
404 clientData.driverState.has_target_thread_group = false;
405 clientData.driverState.target_thread_group_id = ~0ull;
406 clientData.driverState.target_thread_group_data = nullptr;
407 }
408
409 clientData.device = nullptr;
410
411 IOLockUnlock(shared->interfaceLock);
412 #endif
413 }
414
415 uint64_t
workSubmit(IOService * device,WorkSubmitArgs * args)416 IOPerfControlClient::workSubmit(IOService *device, WorkSubmitArgs *args)
417 {
418 #if CONFIG_THREAD_GROUPS
419 auto *thread_group = thread_group_get(current_thread());
420 if (!thread_group) {
421 return kIOPerfControlClientWorkUntracked;
422 }
423
424 PerfControllerInterface::WorkState state{
425 .thread_group_id = thread_group_get_id(thread_group),
426 .thread_group_data = thread_group_get_machine_data(thread_group),
427 .work_data = nullptr,
428 .work_data_size = 0,
429 .started = false,
430 .driver_state = &clientData.driverState
431 };
432 if (!shared->interface.workCanSubmit(device, &state, args)) {
433 return kIOPerfControlClientWorkUntracked;
434 }
435
436 uint64_t token = allocateToken(thread_group);
437 if (token != kIOPerfControlClientWorkUntracked) {
438 state.work_data = &workTable[token].perfcontrol_data;
439 state.work_data_size = sizeof(workTable[token].perfcontrol_data);
440 shared->interface.workSubmit(device, tokenToGlobalUniqueToken(token), &state, args);
441 }
442 return token;
443 #else
444 return kIOPerfControlClientWorkUntracked;
445 #endif
446 }
447
448 uint64_t
workSubmitAndBegin(IOService * device,WorkSubmitArgs * submitArgs,WorkBeginArgs * beginArgs)449 IOPerfControlClient::workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs, WorkBeginArgs *beginArgs)
450 {
451 #if CONFIG_THREAD_GROUPS
452 auto *thread_group = thread_group_get(current_thread());
453 if (!thread_group) {
454 return kIOPerfControlClientWorkUntracked;
455 }
456
457 PerfControllerInterface::WorkState state{
458 .thread_group_id = thread_group_get_id(thread_group),
459 .thread_group_data = thread_group_get_machine_data(thread_group),
460 .work_data = nullptr,
461 .work_data_size = 0,
462 .started = false,
463 .driver_state = &clientData.driverState
464 };
465 if (!shared->interface.workCanSubmit(device, &state, submitArgs)) {
466 return kIOPerfControlClientWorkUntracked;
467 }
468
469 uint64_t token = allocateToken(thread_group);
470 if (token != kIOPerfControlClientWorkUntracked) {
471 auto &entry = workTable[token];
472 state.work_data = &entry.perfcontrol_data;
473 state.work_data_size = sizeof(workTable[token].perfcontrol_data);
474 shared->interface.workSubmit(device, tokenToGlobalUniqueToken(token), &state, submitArgs);
475 state.started = true;
476 shared->interface.workBegin(device, tokenToGlobalUniqueToken(token), &state, beginArgs);
477 markEntryStarted(token, true);
478 }
479 return token;
480 #else
481 return kIOPerfControlClientWorkUntracked;
482 #endif
483 }
484
485 void
workBegin(IOService * device,uint64_t token,WorkBeginArgs * args)486 IOPerfControlClient::workBegin(IOService *device, uint64_t token, WorkBeginArgs *args)
487 {
488 #if CONFIG_THREAD_GROUPS
489 WorkTableEntry *entry = getEntryForToken(token);
490 if (entry == nullptr) {
491 return;
492 }
493
494 assertf(!entry->started, "Work for token %llu was already started", token);
495
496 PerfControllerInterface::WorkState state{
497 .thread_group_id = thread_group_get_id(entry->thread_group),
498 .thread_group_data = thread_group_get_machine_data(entry->thread_group),
499 .work_data = &entry->perfcontrol_data,
500 .work_data_size = sizeof(entry->perfcontrol_data),
501 .started = true,
502 .driver_state = &clientData.driverState
503 };
504 shared->interface.workBegin(device, tokenToGlobalUniqueToken(token), &state, args);
505 markEntryStarted(token, true);
506 #endif
507 }
508
509 void
workEnd(IOService * device,uint64_t token,WorkEndArgs * args,bool done)510 IOPerfControlClient::workEnd(IOService *device, uint64_t token, WorkEndArgs *args, bool done)
511 {
512 #if CONFIG_THREAD_GROUPS
513 WorkTableEntry *entry = getEntryForToken(token);
514 if (entry == nullptr) {
515 return;
516 }
517
518 PerfControllerInterface::WorkState state{
519 .thread_group_id = thread_group_get_id(entry->thread_group),
520 .thread_group_data = thread_group_get_machine_data(entry->thread_group),
521 .work_data = &entry->perfcontrol_data,
522 .work_data_size = sizeof(entry->perfcontrol_data),
523 .started = entry->started,
524 .driver_state = &clientData.driverState
525 };
526
527 if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_4) {
528 PerfControllerInterface::ResourceAccounting resources;
529 shared->interface.workEndWithResources(device, tokenToGlobalUniqueToken(token), &state, args, &resources, done);
530 if (clientData.driverState.resource_accounting) {
531 accountResources(workTable[token].coal, clientData.driverState.device_type, &resources);
532 }
533 } else {
534 shared->interface.workEnd(device, tokenToGlobalUniqueToken(token), &state, args, done);
535 }
536
537 if (done) {
538 deallocateToken(token);
539 } else {
540 markEntryStarted(token, false);
541 }
542 #endif
543 }
544
545 static _Atomic uint64_t unique_work_context_id = 1ull;
546
547 class IOPerfControlWorkContext : public OSObject
548 {
549 OSDeclareDefaultStructors(IOPerfControlWorkContext);
550
551 public:
552 uint64_t id;
553 struct thread_group *thread_group;
554 coalition_t coal;
555 bool started;
556 uint8_t perfcontrol_data[32];
557
558 bool init() override;
559 void reset();
560 void free() override;
561 };
562
563 OSDefineMetaClassAndStructors(IOPerfControlWorkContext, OSObject);
564
565 bool
init()566 IOPerfControlWorkContext::init()
567 {
568 if (!super::init()) {
569 return false;
570 }
571 id = atomic_fetch_add_explicit(&unique_work_context_id, 1, memory_order_relaxed) + 1;
572 reset();
573 return true;
574 }
575
576 void
reset()577 IOPerfControlWorkContext::reset()
578 {
579 thread_group = nullptr;
580 coal = nullptr;
581 started = false;
582 bzero(perfcontrol_data, sizeof(perfcontrol_data));
583 }
584
585 void
free()586 IOPerfControlWorkContext::free()
587 {
588 assertf(thread_group == nullptr, "IOPerfControlWorkContext ID %llu being released without calling workEnd!\n", id);
589 assertf(coal == nullptr, "IOPerfControlWorkContext ID %llu being released without calling workEnd!\n", id);
590 super::free();
591 }
592
593 OSObject *
copyWorkContext()594 IOPerfControlClient::copyWorkContext()
595 {
596 IOPerfControlWorkContext *context = new IOPerfControlWorkContext;
597
598 if (context == nullptr) {
599 return nullptr;
600 }
601
602 if (!context->init()) {
603 context->free();
604 return nullptr;
605 }
606
607 return context;
608 }
609
610 bool
workSubmitAndBeginWithContext(IOService * device,OSObject * context,WorkSubmitArgs * submitArgs,WorkBeginArgs * beginArgs)611 IOPerfControlClient::workSubmitAndBeginWithContext(IOService *device, OSObject *context, WorkSubmitArgs *submitArgs, WorkBeginArgs *beginArgs)
612 {
613 #if CONFIG_THREAD_GROUPS
614
615 if (workSubmitWithContext(device, context, submitArgs) == false) {
616 return false;
617 }
618
619 IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
620
621 PerfControllerInterface::WorkState state{
622 .thread_group_id = thread_group_get_id(work_context->thread_group),
623 .thread_group_data = thread_group_get_machine_data(work_context->thread_group),
624 .work_data = &work_context->perfcontrol_data,
625 .work_data_size = sizeof(work_context->perfcontrol_data),
626 .started = true,
627 .driver_state = &clientData.driverState
628 };
629
630 shared->interface.workBegin(device, work_context->id, &state, beginArgs);
631
632 work_context->started = true;
633
634 return true;
635 #else
636 return false;
637 #endif
638 }
639
640 bool
workSubmitWithContext(IOService * device,OSObject * context,WorkSubmitArgs * args)641 IOPerfControlClient::workSubmitWithContext(IOService *device, OSObject *context, WorkSubmitArgs *args)
642 {
643 #if CONFIG_THREAD_GROUPS
644 IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
645
646 if (work_context == nullptr) {
647 return false;
648 }
649
650 auto *thread_group = thread_group_get(current_thread());
651 assert(thread_group != nullptr);
652
653 assertf(!work_context->started, "IOPerfControlWorkContext ID %llu was already started", work_context->id);
654 assertf(work_context->thread_group == nullptr, "IOPerfControlWorkContext ID %llu has already taken a refcount on TG 0x%p \n", work_context->id, (void *)(work_context->thread_group));
655
656 PerfControllerInterface::WorkState state{
657 .thread_group_id = thread_group_get_id(thread_group),
658 .thread_group_data = thread_group_get_machine_data(thread_group),
659 .work_data = nullptr,
660 .work_data_size = 0,
661 .started = false,
662 .driver_state = &clientData.driverState
663 };
664 if (!shared->interface.workCanSubmit(device, &state, args)) {
665 return false;
666 }
667
668 work_context->thread_group = thread_group_retain(thread_group);
669 if (clientData.driverState.resource_accounting) {
670 auto *coalition = task_get_coalition(current_task(), COALITION_TYPE_RESOURCE);
671 assert(coalition != nullptr);
672 work_context->coal = coalition;
673 coalition_retain(coalition);
674 }
675
676 state.work_data = &work_context->perfcontrol_data;
677 state.work_data_size = sizeof(work_context->perfcontrol_data);
678
679 shared->interface.workSubmit(device, work_context->id, &state, args);
680
681 return true;
682 #else
683 return false;
684 #endif
685 }
686
687 void
workUpdateWithContext(IOService * device,OSObject * context,WorkUpdateArgs * args)688 IOPerfControlClient::workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args)
689 {
690 #if CONFIG_THREAD_GROUPS
691 IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
692
693 if (work_context == nullptr) {
694 return;
695 }
696
697 if (work_context->thread_group == nullptr) {
698 // This Work Context has not taken a refcount on a TG
699 return;
700 }
701
702 PerfControllerInterface::WorkState state{
703 .thread_group_id = thread_group_get_id(work_context->thread_group),
704 .thread_group_data = thread_group_get_machine_data(work_context->thread_group),
705 .work_data = &work_context->perfcontrol_data,
706 .work_data_size = sizeof(work_context->perfcontrol_data),
707 .driver_state = &clientData.driverState
708 };
709 shared->interface.workUpdate(device, work_context->id, &state, args);
710 #endif
711 }
712
713 void
workBeginWithContext(IOService * device,OSObject * context,WorkBeginArgs * args)714 IOPerfControlClient::workBeginWithContext(IOService *device, OSObject *context, WorkBeginArgs *args)
715 {
716 #if CONFIG_THREAD_GROUPS
717 IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
718
719 if (work_context == nullptr) {
720 return;
721 }
722
723 if (work_context->thread_group == nullptr) {
724 // This Work Context has not taken a refcount on a TG
725 return;
726 }
727
728 assertf(!work_context->started, "IOPerfControlWorkContext %llu was already started", work_context->id);
729
730 PerfControllerInterface::WorkState state{
731 .thread_group_id = thread_group_get_id(work_context->thread_group),
732 .thread_group_data = thread_group_get_machine_data(work_context->thread_group),
733 .work_data = &work_context->perfcontrol_data,
734 .work_data_size = sizeof(work_context->perfcontrol_data),
735 .started = true,
736 .driver_state = &clientData.driverState
737 };
738 shared->interface.workBegin(device, work_context->id, &state, args);
739
740 work_context->started = true;
741 #endif
742 }
743
744 void
workEndWithContext(IOService * device,OSObject * context,WorkEndArgs * args,bool done)745 IOPerfControlClient::workEndWithContext(IOService *device, OSObject *context, WorkEndArgs *args, bool done)
746 {
747 #if CONFIG_THREAD_GROUPS
748 IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
749
750 if (work_context == nullptr) {
751 return;
752 }
753
754 if (work_context->thread_group == nullptr) {
755 return;
756 }
757
758 PerfControllerInterface::WorkState state{
759 .thread_group_id = thread_group_get_id(work_context->thread_group),
760 .thread_group_data = thread_group_get_machine_data(work_context->thread_group),
761 .work_data = &work_context->perfcontrol_data,
762 .work_data_size = sizeof(work_context->perfcontrol_data),
763 .started = work_context->started,
764 .driver_state = &clientData.driverState
765 };
766
767 if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_4) {
768 PerfControllerInterface::ResourceAccounting resources;
769 shared->interface.workEndWithResources(device, work_context->id, &state, args, &resources, done);
770 if (clientData.driverState.resource_accounting) {
771 accountResources(work_context->coal, clientData.driverState.device_type, &resources);
772 }
773 } else {
774 shared->interface.workEnd(device, work_context->id, &state, args, done);
775 }
776
777 if (done) {
778 thread_group_release(work_context->thread_group);
779 if (work_context->coal != nullptr) {
780 coalition_release(work_context->coal);
781 }
782 work_context->reset();
783 } else {
784 work_context->started = false;
785 }
786
787 return;
788 #else
789 return;
790 #endif
791 }
792
793 IOReturn
querySubmitterRole(IOService * device,task_t clientTask,uint32_t * role_out)794 IOPerfControlClient::querySubmitterRole(IOService *device, task_t clientTask, uint32_t* role_out)
795 {
796 IOReturn result = kIOReturnNotFound;
797
798 uint32_t role;
799
800 switch (deviceType) {
801 case IOPCDeviceTypeGPU:
802 role = task_get_gpu_role(clientTask);
803
804 KDBG(IMPORTANCE_CODE(IMP_QUERY_GPU_ROLE, 0), role);
805
806 *role_out = role;
807
808 result = kIOReturnSuccess;
809 break;
810 default:
811 result = kIOReturnNotFound;
812 }
813
814 return result;
815 }
816
817 IOReturn
registerPerformanceController(PerfControllerInterface * pci)818 IOPerfControlClient::registerPerformanceController(PerfControllerInterface *pci)
819 {
820 IOReturn result = kIOReturnError;
821
822 IOLockLock(shared->interfaceLock);
823
824 if (shared->interface.version == PERFCONTROL_INTERFACE_VERSION_NONE) {
825 shared->interface.version = pci->version;
826
827 if (pci->version >= PERFCONTROL_INTERFACE_VERSION_1) {
828 assert(pci->registerDevice && pci->unregisterDevice && pci->workCanSubmit && pci->workSubmit && pci->workBegin && pci->workEnd);
829 shared->interface.registerDevice = pci->registerDevice;
830 shared->interface.unregisterDevice = pci->unregisterDevice;
831 shared->interface.workCanSubmit = pci->workCanSubmit;
832 shared->interface.workSubmit = pci->workSubmit;
833 shared->interface.workBegin = pci->workBegin;
834 shared->interface.workEnd = pci->workEnd;
835 }
836
837 if (pci->version >= PERFCONTROL_INTERFACE_VERSION_2) {
838 if (pci->workUpdate != nullptr) {
839 shared->interface.workUpdate = pci->workUpdate;
840 }
841 }
842
843 if (pci->version >= PERFCONTROL_INTERFACE_VERSION_3) {
844 assert(pci->registerDriverDevice && pci->unregisterDriverDevice);
845 shared->interface.registerDriverDevice = pci->registerDriverDevice;
846 shared->interface.unregisterDriverDevice = pci->unregisterDriverDevice;
847 }
848
849 if (pci->version >= PERFCONTROL_INTERFACE_VERSION_4) {
850 assert(pci->workEndWithResources);
851 shared->interface.workEndWithResources = pci->workEndWithResources;
852 }
853
854 result = kIOReturnSuccess;
855
856 OSObject *obj;
857 while ((obj = shared->deviceRegistrationList->getAnyObject())) {
858 IOPerfControlClient *client = OSDynamicCast(IOPerfControlClient, obj);
859 IOPerfControlClientData *clientData = client->getClientData();
860 if (clientData && clientData->device) {
861 if (pci->version >= PERFCONTROL_INTERFACE_VERSION_3) {
862 pci->registerDriverDevice(clientData->device->getProvider(), clientData->device, &(clientData->driverState));
863 } else if (pci->version >= PERFCONTROL_INTERFACE_VERSION_1) {
864 pci->registerDevice(clientData->device);
865 }
866 }
867 shared->deviceRegistrationList->removeObject(obj);
868 }
869 }
870
871 IOLockUnlock(shared->interfaceLock);
872
873 return result;
874 }
875