1 /*
2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define IOKIT_ENABLE_SHARED_PTR
30
31 #define DISABLE_DATAQUEUE_WARNING
32
33 #include <IOKit/IODataQueue.h>
34
35 #undef DISABLE_DATAQUEUE_WARNING
36
37 #include <IOKit/IODataQueueShared.h>
38 #include <IOKit/IOLib.h>
39 #include <IOKit/IOMemoryDescriptor.h>
40 #include <libkern/OSAtomic.h>
41 #include <libkern/c++/OSSharedPtr.h>
42
43 struct IODataQueueInternal {
44 mach_msg_header_t msg;
45 UInt32 queueSize;
46 };
47
48 #ifdef enqueue
49 #undef enqueue
50 #endif
51
52 #ifdef dequeue
53 #undef dequeue
54 #endif
55
56 #define super OSObject
57
OSDefineMetaClassAndStructors(IODataQueue,OSObject)58 OSDefineMetaClassAndStructors(IODataQueue, OSObject)
59
60 OSSharedPtr<IODataQueue>
61 IODataQueue::withCapacity(UInt32 size)
62 {
63 OSSharedPtr<IODataQueue> dataQueue = OSMakeShared<IODataQueue>();
64
65 if (dataQueue) {
66 if (!dataQueue->initWithCapacity(size)) {
67 return nullptr;
68 }
69 }
70
71 return dataQueue;
72 }
73
74 OSSharedPtr<IODataQueue>
withEntries(UInt32 numEntries,UInt32 entrySize)75 IODataQueue::withEntries(UInt32 numEntries, UInt32 entrySize)
76 {
77 OSSharedPtr<IODataQueue> dataQueue = OSMakeShared<IODataQueue>();
78
79 if (dataQueue) {
80 if (!dataQueue->initWithEntries(numEntries, entrySize)) {
81 return nullptr;
82 }
83 }
84
85 return dataQueue;
86 }
87
88 Boolean
initWithCapacity(UInt32 size)89 IODataQueue::initWithCapacity(UInt32 size)
90 {
91 vm_size_t allocSize = 0;
92
93 if (!super::init()) {
94 return false;
95 }
96
97 if (size > UINT32_MAX - DATA_QUEUE_MEMORY_HEADER_SIZE) {
98 return false;
99 }
100
101 allocSize = round_page(size + DATA_QUEUE_MEMORY_HEADER_SIZE);
102
103 if (allocSize < size) {
104 return false;
105 }
106
107 assert(!notifyMsg);
108 notifyMsg = IOMallocType(IODataQueueInternal);
109 ((IODataQueueInternal *)notifyMsg)->queueSize = size;
110
111 dataQueue = (IODataQueueMemory *)IOMallocAligned(allocSize, PAGE_SIZE);
112 if (dataQueue == NULL) {
113 return false;
114 }
115 bzero(dataQueue, allocSize);
116
117 dataQueue->queueSize = size;
118 // dataQueue->head = 0;
119 // dataQueue->tail = 0;
120
121 return true;
122 }
123
124 Boolean
initWithEntries(UInt32 numEntries,UInt32 entrySize)125 IODataQueue::initWithEntries(UInt32 numEntries, UInt32 entrySize)
126 {
127 // Checking overflow for (numEntries + 1)*(entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE):
128 // check (entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE)
129 if ((entrySize > UINT32_MAX - DATA_QUEUE_ENTRY_HEADER_SIZE) ||
130 // check (numEntries + 1)
131 (numEntries > UINT32_MAX - 1) ||
132 // check (numEntries + 1)*(entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE)
133 (entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE > UINT32_MAX / (numEntries + 1))) {
134 return false;
135 }
136
137 return initWithCapacity((numEntries + 1) * (DATA_QUEUE_ENTRY_HEADER_SIZE + entrySize));
138 }
139
140 void
free()141 IODataQueue::free()
142 {
143 if (notifyMsg) {
144 if (dataQueue) {
145 IOFreeAligned(dataQueue, round_page(((IODataQueueInternal *)notifyMsg)->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE));
146 dataQueue = NULL;
147 }
148
149 IOFreeType(notifyMsg, IODataQueueInternal);
150 notifyMsg = NULL;
151 }
152
153 super::free();
154
155 return;
156 }
157
158 Boolean
enqueue(void * data,UInt32 dataSize)159 IODataQueue::enqueue(void * data, UInt32 dataSize)
160 {
161 UInt32 head;
162 UInt32 tail;
163 UInt32 newTail;
164 const UInt32 entrySize = dataSize + DATA_QUEUE_ENTRY_HEADER_SIZE;
165 UInt32 queueSize;
166 IODataQueueEntry * entry;
167
168 // Check for overflow of entrySize
169 if (dataSize > UINT32_MAX - DATA_QUEUE_ENTRY_HEADER_SIZE) {
170 return false;
171 }
172
173 // Force a single read of head and tail
174 // See rdar://problem/40780584 for an explanation of relaxed/acquire barriers
175 tail = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->tail, __ATOMIC_RELAXED);
176 head = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_ACQUIRE);
177
178 // Check for underflow of (dataQueue->queueSize - tail)
179 queueSize = ((IODataQueueInternal *) notifyMsg)->queueSize;
180 if ((queueSize < tail) || (queueSize < head)) {
181 return false;
182 }
183
184 if (tail >= head) {
185 // Is there enough room at the end for the entry?
186 if ((entrySize <= UINT32_MAX - tail) &&
187 ((tail + entrySize) <= queueSize)) {
188 entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
189
190 entry->size = dataSize;
191 __nochk_memcpy(&entry->data, data, dataSize);
192
193 // The tail can be out of bound when the size of the new entry
194 // exactly matches the available space at the end of the queue.
195 // The tail can range from 0 to dataQueue->queueSize inclusive.
196
197 newTail = tail + entrySize;
198 } else if (head > entrySize) { // Is there enough room at the beginning?
199 // Wrap around to the beginning, but do not allow the tail to catch
200 // up to the head.
201
202 dataQueue->queue->size = dataSize;
203
204 // We need to make sure that there is enough room to set the size before
205 // doing this. The user client checks for this and will look for the size
206 // at the beginning if there isn't room for it at the end.
207
208 if ((queueSize - tail) >= DATA_QUEUE_ENTRY_HEADER_SIZE) {
209 ((IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail))->size = dataSize;
210 }
211
212 __nochk_memcpy(&dataQueue->queue->data, data, dataSize);
213 newTail = entrySize;
214 } else {
215 return false; // queue is full
216 }
217 } else {
218 // Do not allow the tail to catch up to the head when the queue is full.
219 // That's why the comparison uses a '>' rather than '>='.
220
221 if ((head - tail) > entrySize) {
222 entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
223
224 entry->size = dataSize;
225 __nochk_memcpy(&entry->data, data, dataSize);
226 newTail = tail + entrySize;
227 } else {
228 return false; // queue is full
229 }
230 }
231
232 // Publish the data we just enqueued
233 __c11_atomic_store((_Atomic UInt32 *)&dataQueue->tail, newTail, __ATOMIC_RELEASE);
234
235 if (tail != head) {
236 //
237 // The memory barrier below paris with the one in ::dequeue
238 // so that either our store to the tail cannot be missed by
239 // the next dequeue attempt, or we will observe the dequeuer
240 // making the queue empty.
241 //
242 // Of course, if we already think the queue is empty,
243 // there's no point paying this extra cost.
244 //
245 __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
246 head = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_RELAXED);
247 }
248
249 if (tail == head) {
250 // Send notification (via mach message) that data is now available.
251 sendDataAvailableNotification();
252 }
253 return true;
254 }
255
256 void
setNotificationPort(mach_port_t port)257 IODataQueue::setNotificationPort(mach_port_t port)
258 {
259 mach_msg_header_t * msgh;
260
261 msgh = &((IODataQueueInternal *) notifyMsg)->msg;
262 bzero(msgh, sizeof(mach_msg_header_t));
263 msgh->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0);
264 msgh->msgh_size = sizeof(mach_msg_header_t);
265 msgh->msgh_remote_port = port;
266 }
267
268 void
sendDataAvailableNotification()269 IODataQueue::sendDataAvailableNotification()
270 {
271 kern_return_t kr;
272 mach_msg_header_t * msgh;
273
274 msgh = &((IODataQueueInternal *) notifyMsg)->msg;
275 if (msgh->msgh_remote_port) {
276 kr = mach_msg_send_from_kernel_with_options(msgh, msgh->msgh_size, MACH_SEND_TIMEOUT, MACH_MSG_TIMEOUT_NONE);
277 switch (kr) {
278 case MACH_SEND_TIMED_OUT: // Notification already sent
279 case MACH_MSG_SUCCESS:
280 case MACH_SEND_NO_BUFFER:
281 break;
282 default:
283 IOLog("%s: dataAvailableNotification failed - msg_send returned: %d\n", /*getName()*/ "IODataQueue", kr);
284 break;
285 }
286 }
287 }
288
289 OSSharedPtr<IOMemoryDescriptor>
getMemoryDescriptor()290 IODataQueue::getMemoryDescriptor()
291 {
292 OSSharedPtr<IOMemoryDescriptor> descriptor;
293 UInt32 queueSize;
294
295 queueSize = ((IODataQueueInternal *) notifyMsg)->queueSize;
296 if (dataQueue != NULL) {
297 descriptor = IOMemoryDescriptor::withAddress(dataQueue, queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE, kIODirectionOutIn);
298 }
299
300 return descriptor;
301 }
302