1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 */ 4 5 #pragma once 6 7 #ifdef KERNEL_PRIVATE 8 #ifdef __cplusplus 9 10 #include <IOKit/IOService.h> 11 #include <stdatomic.h> 12 #include <kern/bits.h> 13 #include <libkern/c++/OSPtr.h> 14 15 struct thread_group; 16 17 enum{ 18 kIOPerfControlClientWorkUntracked = 0, 19 }; 20 21 /*! 22 * @class IOPerfControlClient : public OSObject 23 * @abstract Class which implements an interface allowing device drivers to participate in performance control. 24 * @discussion TODO 25 */ 26 class IOPerfControlClient final : public OSObject 27 { 28 OSDeclareDefaultStructors(IOPerfControlClient); 29 30 protected: 31 virtual bool init(IOService *driver, uint64_t maxWorkCapacity); 32 virtual void free() APPLE_KEXT_OVERRIDE; 33 34 public: 35 /*! 36 * @function copyClient 37 * @abstract Return a retained reference to a client object, to be released by the driver. It may be 38 * shared with other drivers in the system. 39 * @param driver The device driver that will be using this interface. 40 * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver. 41 * @returns An instance of IOPerfControlClient. 42 */ 43 static IOPerfControlClient *copyClient(IOService *driver, uint64_t maxWorkCapacity); 44 45 __enum_decl(IOPCDeviceType, uint8_t, { 46 IOPCDeviceTypeUnknown = 0x0, 47 IOPCDeviceTypeGPU = 0x1, 48 IOPCDeviceTypeANE = 0x2, 49 IOPCDeviceTypeMSR = 0x3, 50 IOPCDeviceTypeStorage = 0x4, 51 IOPCDeviceTypeMax = 0x5, 52 }); 53 /*! 54 * @function copyClientForDeviceType 55 * @abstract Return a retained reference to a client object, to be released by the driver. It may be 56 * shared with other drivers in the system. 57 * @param driver The device driver that will be using this interface. 58 * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver. 59 * @param deviceType The type of device that this driver controls. Unknown is fine to use for devices not listed. 60 * @returns An instance of IOPerfControlClient. 61 */ 62 static IOPerfControlClient *copyClientForDeviceType(IOService *driver, uint64_t maxWorkCapacity, IOPCDeviceType deviceType); 63 64 /*! 65 * @function registerDevice 66 * @abstract Inform the system that work will be dispatched to a device in the future. 67 * @discussion The system will do some one-time setup work associated with the device, and may block the 68 * current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin, 69 * workBegin, or workEnd until they have been successfully registered. The unregistration process happens 70 * automatically when the device object is deallocated. 71 * @param device The device object. Some platforms require device to be a specific subclass of IOService. 72 * @returns kIOReturnSuccess or an IOReturn error code 73 */ 74 virtual IOReturn registerDevice(IOService *driver, IOService *device); 75 76 /*! 77 * @function unregisterDevice 78 * @abstract Inform the system that work will be no longer be dispatched to a device in the future. 79 * @discussion This call is optional as the unregistration process happens automatically when the device 80 * object is deallocated. This call may block the current thread and/or acquire locks. It should not be 81 * called until after all submitted work has been ended using workEnd. 82 * @param device The device object. Some platforms require device to be a specific subclass of IOService. 83 */ 84 virtual void unregisterDevice(IOService *driver, IOService *device); 85 86 /*! 87 * @struct WorkSubmitArgs 88 * @discussion Drivers may submit additional device-specific arguments related to the submission of a work item 89 * by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing 90 * a header file describing these arguments. 91 */ 92 struct WorkSubmitArgs { 93 uint32_t version; 94 uint32_t size; 95 uint64_t submit_time; 96 uint64_t reserved[4]; 97 void *driver_data; 98 }; 99 100 /*! 101 * @function workSubmit 102 * @abstract Tell the performance controller that work was submitted. 103 * @param device The device that will execute the work. Some platforms require device to be a 104 * specific subclass of IOService. 105 * @param args Optional device-specific arguments related to the submission of this work item. 106 * @returns A token representing this work item, which must be passed to workEnd when the work is finished 107 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks 108 * and a degradation of system performance. 109 */ 110 virtual uint64_t workSubmit(IOService *device, WorkSubmitArgs *args = nullptr); 111 112 /*! 113 * @struct WorkBeginArgs 114 * @discussion Drivers may submit additional device-specific arguments related to the start of a work item 115 * by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing 116 * a header file describing these arguments. 117 */ 118 struct WorkBeginArgs { 119 uint32_t version; 120 uint32_t size; 121 uint64_t begin_time; 122 uint64_t reserved[4]; 123 void *driver_data; 124 }; 125 126 /*! 127 * @function workSubmitAndBegin 128 * @abstract Tell the performance controller that work was submitted and immediately began executing. 129 * @param device The device that is executing the work. Some platforms require device to be a 130 * specific subclass of IOService. 131 * @param submitArgs Optional device-specific arguments related to the submission of this work item. 132 * @param beginArgs Optional device-specific arguments related to the start of this work item. 133 * @returns A token representing this work item, which must be passed to workEnd when the work is finished 134 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks 135 * and a degradation of system performance. 136 */ 137 virtual uint64_t workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs = nullptr, 138 WorkBeginArgs *beginArgs = nullptr); 139 140 /*! 141 * @function workBegin 142 * @abstract Tell the performance controller that previously submitted work began executing. 143 * @param device The device that is executing the work. Some platforms require device to be a 144 * specific subclass of IOService. 145 * @param args Optional device-specific arguments related to the start of this work item. 146 */ 147 virtual void workBegin(IOService *device, uint64_t token, WorkBeginArgs *args = nullptr); 148 149 /*! 150 * @struct WorkEndArgs 151 * @discussion Drivers may submit additional device-specific arguments related to the end of a work item 152 * by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing 153 * a header file describing these arguments. 154 */ 155 struct WorkEndArgs { 156 uint32_t version; 157 uint32_t size; 158 uint64_t end_time; 159 uint64_t reserved[4]; 160 void *driver_data; 161 }; 162 163 /*! 164 * @function workEnd 165 * @abstract Tell the performance controller that previously started work finished executing. 166 * @param device The device that executed the work. Some platforms require device to be a 167 * specific subclass of IOService. 168 * @param args Optional device-specific arguments related to the end of this work item. 169 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for 170 * calling workBegin when the work resumes and workEnd with done set to True when it has completed. A workEnd() call 171 * without a corresponding workBegin() call is a way to cancel a work item and return token to IOPerfControl. 172 */ 173 virtual void workEnd(IOService *device, uint64_t token, WorkEndArgs *args = nullptr, bool done = true); 174 175 /*! 176 * @function copyWorkContext 177 * @abstract Return a retained reference to an opaque OSObject, to be released by the driver. This object can 178 * be used by IOPerfControl to track a work item. This may perform dynamic memory allocation. 179 * @returns A pointer to an OSObject 180 */ 181 OSPtr<OSObject> copyWorkContext(); 182 183 /*! 184 * @function workSubmitAndBeginWithContext 185 * @abstract Tell the performance controller that work was submitted and immediately began executing 186 * @param device The device that is executing the work. Some platforms require device to be a 187 * specific subclass of IOService. 188 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track 189 * this work item. 190 * @param submitArgs Optional device-specific arguments related to the submission of this work item. 191 * @param beginArgs Optional device-specific arguments related to the start of this work item. 192 * @returns true if IOPerfControl is tracking this work item, else false. 193 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. 194 */ 195 bool workSubmitAndBeginWithContext(IOService *device, OSObject *context, WorkSubmitArgs *submitArgs = nullptr, 196 WorkBeginArgs *beginArgs = nullptr); 197 198 /*! 199 * @function workSubmitWithContext 200 * @abstract Tell the performance controller that work was submitted. 201 * @param device The device that will execute the work. Some platforms require device to be a 202 * specific subclass of IOService. 203 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track 204 * this work item. 205 * @param args Optional device-specific arguments related to the submission of this work item. 206 * @returns true if IOPerfControl is tracking this work item, else false. 207 */ 208 bool workSubmitWithContext(IOService *device, OSObject *context, WorkSubmitArgs *args = nullptr); 209 210 /*! 211 * @function workBeginWithContext 212 * @abstract Tell the performance controller that previously submitted work began executing. 213 * @param device The device that is executing the work. Some platforms require device to be a 214 * specific subclass of IOService. 215 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 216 * @param args Optional device-specific arguments related to the start of this work item. 217 * @note The workBeginWithContext() and workEndWithContext() calls are optional if the corresponding workSubmitWithContext() call returned false. 218 */ 219 void workBeginWithContext(IOService *device, OSObject *context, WorkBeginArgs *args = nullptr); 220 221 /*! 222 * @function workEndWithContext 223 * @abstract Tell the performance controller that previously started work finished executing. 224 * @param device The device that executed the work. Some platforms require device to be a 225 * specific subclass of IOService. 226 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 227 * @param args Optional device-specific arguments related to the end of this work item. 228 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for 229 * calling workBegin when the work resumes and workEnd with done set to True when it has completed. 230 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. A workEndWithContext() 231 * call without a corresponding workBeginWithContext() call is a way to cancel a work item. 232 */ 233 void workEndWithContext(IOService *device, OSObject *context, WorkEndArgs *args = nullptr, bool done = true); 234 235 /*! 236 * @struct WorkUpdateArgs 237 * @discussion Drivers may submit additional device-specific arguments related to a work item by passing a 238 * struct with WorkUpdateArgs as its first member. Note: Drivers are responsible for publishing 239 * a header file describing these arguments. 240 */ 241 struct WorkUpdateArgs { 242 uint32_t version; 243 uint32_t size; 244 uint64_t update_time; 245 uint64_t reserved[4]; 246 void *driver_data; 247 }; 248 249 /*! 250 * @function workUpdateWithContext 251 * @abstract Provide and receive additional information from the performance controller. If this call is 252 * made at all, it should be between workSubmit and workEnd. The purpose and implementation of this call are 253 * device specific, and may do nothing on some devices. 254 * @param device The device that submitted the work. Some platforms require device to be a 255 * specific subclass of IOService. 256 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 257 * @param args Optional device-specific arguments. 258 */ 259 void workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args = nullptr); 260 261 /*! 262 * @function querySubmitterRole 263 * @abstract Reports the current role configured on the submitting task by app lifecycle management policy 264 * for this type of device. May be queried before submit to inform which policies should apply to this work. 265 * @param device The device that will submit the work. Some platforms require device to be a 266 * specific subclass of IOService. 267 * @note Must use the copyClientForDeviceType init to convey the type of device to query the role of. 268 * GPU role enums are found in sys/resource_private.h and are configured via PRIO_DARWIN_GPU. 269 */ 270 IOReturn querySubmitterRole(IOService *device, task_t submitting_task, uint32_t* role_out); 271 272 #define PERFCONTROL_SUPPORTS_SUBMITTER_ROLE 1 273 274 /* 275 * Callers should always use the CURRENT version so that the kernel can detect both older 276 * and newer structure layouts. New callbacks should always be added at the end of the 277 * structure, and xnu should expect existing source recompiled against newer headers 278 * to pass NULL for unimplemented callbacks. 279 */ 280 281 #define PERFCONTROL_INTERFACE_VERSION_NONE (0) /* no interface */ 282 #define PERFCONTROL_INTERFACE_VERSION_1 (1) /* up-to workEnd */ 283 #define PERFCONTROL_INTERFACE_VERSION_2 (2) /* up-to workUpdate */ 284 #define PERFCONTROL_INTERFACE_VERSION_3 (3) /* up-to (un)registerDriverDevice */ 285 #define PERFCONTROL_INTERFACE_VERSION_4 (4) /* up-to workEndWithResources */ 286 #define PERFCONTROL_INTERFACE_VERSION_CURRENT PERFCONTROL_INTERFACE_VERSION_4 287 288 /*! 289 * @struct PerfControllerInterface 290 * @discussion Function pointers necessary to register a performance controller. Not for general driver use. 291 */ 292 struct PerfControllerInterface { 293 enum struct PerfDeviceID : uint32_t{ 294 kInvalid = 0, 295 kCPU = 0, 296 kANE = 0x4, 297 kGPU, 298 kMSR, 299 kStorage, 300 }; 301 302 struct DriverState { 303 uint32_t has_target_thread_group : 1; 304 uint32_t has_device_info : 1; 305 uint32_t reserved : 30; 306 307 uint64_t target_thread_group_id; 308 void *target_thread_group_data; 309 310 PerfDeviceID device_type; /* device-type determined by CLPC */ 311 uint32_t instance_id; 312 bool resource_accounting; 313 }; 314 315 struct WorkState { 316 uint64_t thread_group_id; 317 void *thread_group_data; 318 void *work_data; 319 uint32_t work_data_size; 320 uint32_t started : 1; 321 uint32_t reserved : 31; 322 const DriverState* driver_state; 323 }; 324 325 struct ResourceAccounting { 326 uint64_t mach_time_delta; 327 uint64_t energy_nj_delta; 328 }; 329 330 using RegisterDeviceFunction = IOReturn (*)(IOService *); 331 using RegisterDriverDeviceFunction = IOReturn (*)(IOService *, IOService *, DriverState *); 332 using WorkCanSubmitFunction = bool (*)(IOService *, WorkState *, WorkSubmitArgs *); 333 using WorkSubmitFunction = void (*)(IOService *, uint64_t, WorkState *, WorkSubmitArgs *); 334 using WorkBeginFunction = void (*)(IOService *, uint64_t, WorkState *, WorkBeginArgs *); 335 using WorkEndFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, bool); 336 using WorkEndWithResourcesFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, ResourceAccounting *, bool); 337 using WorkUpdateFunction = void (*)(IOService *, uint64_t, WorkState *, WorkUpdateArgs *); 338 339 uint64_t version; 340 RegisterDeviceFunction registerDevice; 341 RegisterDeviceFunction unregisterDevice; 342 WorkCanSubmitFunction workCanSubmit; 343 WorkSubmitFunction workSubmit; 344 WorkBeginFunction workBegin; 345 WorkEndFunction workEnd; 346 WorkUpdateFunction workUpdate; 347 RegisterDriverDeviceFunction registerDriverDevice; 348 RegisterDriverDeviceFunction unregisterDriverDevice; 349 WorkEndWithResourcesFunction workEndWithResources; 350 }; 351 352 struct IOPerfControlClientShared { 353 atomic_uint_fast8_t maxDriverIndex; 354 PerfControllerInterface interface; 355 IOLock *interfaceLock; 356 OSSet *deviceRegistrationList; 357 }; 358 359 struct IOPerfControlClientData { 360 struct thread_group *target_thread_group; 361 PerfControllerInterface::DriverState driverState; 362 IOService* device; 363 }; 364 /*! 365 * @function registerPerformanceController 366 * @abstract Register a performance controller to receive callbacks. Not for general driver use. 367 * @param interface Struct containing callback functions implemented by the performance controller. 368 * @returns kIOReturnSuccess or kIOReturnError if the interface was already registered. 369 */ 370 virtual IOReturn registerPerformanceController(PerfControllerInterface *interface); 371 372 /*! 373 * @function getClientData 374 * @abstract Not for general driver use. Only used by registerPerformanceController(). Allows performanceController to register existing IOPerfControlClient. 375 * @returns IOPerfControlData associated with a IOPerfControlClient 376 */ 377 IOPerfControlClientData * getClientData()378 getClientData() 379 { 380 return &clientData; 381 } 382 383 private: 384 385 void setDeviceType(IOPCDeviceType deviceType); 386 387 struct WorkTableEntry { 388 struct thread_group *thread_group; 389 coalition_t coal; 390 bool started; 391 uint8_t perfcontrol_data[32]; 392 }; 393 394 static constexpr size_t kMaxWorkTableNumEntries = 1024; 395 static constexpr size_t kWorkTableIndexBits = 24; 396 static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since 397 // kIOPerfControlClientWorkUntracked takes number 0 398 static constexpr size_t kWorkTableIndexMask = (const size_t)mask(kWorkTableIndexBits); 399 400 uint64_t allocateToken(thread_group *thread_group); 401 void deallocateToken(uint64_t token); 402 WorkTableEntry *getEntryForToken(uint64_t token); 403 void markEntryStarted(uint64_t token, bool started); 404 inline uint64_t tokenToGlobalUniqueToken(uint64_t token); 405 void accountResources(coalition_t coal, PerfControllerInterface::PerfDeviceID device_type, PerfControllerInterface::ResourceAccounting *resources); 406 407 IOPCDeviceType deviceType; /* device-type provided by client via copyClientForDeviceType */ 408 uint8_t driverIndex; 409 IOPerfControlClientShared *shared; 410 WorkTableEntry *workTable; 411 size_t workTableLength; 412 size_t workTableNextIndex; 413 IOSimpleLock *workTableLock; 414 415 IOPerfControlClientData clientData; 416 }; 417 418 #endif /* __cplusplus */ 419 #endif /* KERNEL_PRIVATE */ 420