1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #ifndef _SKYWALK_NEXUS_PKTQ_H_
29 #define _SKYWALK_NEXUS_PKTQ_H_
30
31 #include <kern/locks.h>
32 #include <net/classq/classq.h>
33
34 #define NX_PKTQ_NO_LIMIT ((uint32_t)-1)
35
36 /*
37 * These function implement an packet tailq with an optional lock.
38 * The base functions act ONLY ON THE QUEUE, whereas the "safe"
39 * variants (nx_pktq_safe_*) also handle the lock.
40 */
41
42 /* A FIFO queue of packets with an optional lock. */
43 struct nx_pktq {
44 decl_lck_mtx_data(, nx_pktq_lock);
45 class_queue_t nx_pktq_q;
46 lck_grp_t *nx_pktq_grp;
47 struct __kern_channel_ring *nx_pktq_kring;
48 };
49
50 __attribute__((always_inline))
51 static inline void
nx_pktq_lock(struct nx_pktq * q)52 nx_pktq_lock(struct nx_pktq *q)
53 {
54 lck_mtx_lock(&q->nx_pktq_lock);
55 }
56
57 __attribute__((always_inline))
58 static inline void
nx_pktq_lock_spin(struct nx_pktq * q)59 nx_pktq_lock_spin(struct nx_pktq *q)
60 {
61 lck_mtx_lock_spin(&q->nx_pktq_lock);
62 }
63
64 __attribute__((always_inline))
65 static inline void
nx_pktq_convert_spin(struct nx_pktq * q)66 nx_pktq_convert_spin(struct nx_pktq *q)
67 {
68 lck_mtx_convert_spin(&q->nx_pktq_lock);
69 }
70
71 __attribute__((always_inline))
72 static inline void
nx_pktq_unlock(struct nx_pktq * q)73 nx_pktq_unlock(struct nx_pktq *q)
74 {
75 lck_mtx_unlock(&q->nx_pktq_lock);
76 }
77
78 __attribute__((always_inline))
79 static inline struct __kern_packet *
nx_pktq_peek(struct nx_pktq * q)80 nx_pktq_peek(struct nx_pktq *q)
81 {
82 return qhead(&q->nx_pktq_q);
83 }
84
85 __attribute__((always_inline))
86 static inline unsigned int
nx_pktq_len(struct nx_pktq * q)87 nx_pktq_len(struct nx_pktq *q)
88 {
89 return qlen(&q->nx_pktq_q);
90 }
91
92 __attribute__((always_inline))
93 static inline size_t
nx_pktq_size(struct nx_pktq * q)94 nx_pktq_size(struct nx_pktq *q)
95 {
96 u_int64_t qsize = qsize(&q->nx_pktq_q);
97 VERIFY(qsize <= UINT_MAX);
98 return (size_t)qsize;
99 }
100
101 __attribute__((always_inline))
102 static inline unsigned int
nx_pktq_limit(struct nx_pktq * q)103 nx_pktq_limit(struct nx_pktq *q)
104 {
105 return qlimit(&q->nx_pktq_q);
106 }
107
108 __attribute__((always_inline))
109 static inline void
__nx_pktq_enq(struct nx_pktq * q,struct __kern_packet * p)110 __nx_pktq_enq(struct nx_pktq *q, struct __kern_packet *p)
111 {
112 classq_pkt_t pkt;
113
114 CLASSQ_PKT_INIT_PACKET(&pkt, p);
115 _addq(&q->nx_pktq_q, &pkt);
116 }
117
118 __attribute__((always_inline))
119 static inline void
nx_pktq_safe_enq(struct nx_pktq * q,struct __kern_packet * p)120 nx_pktq_safe_enq(struct nx_pktq *q, struct __kern_packet *p)
121 {
122 nx_pktq_lock(q);
123 __nx_pktq_enq(q, p);
124 nx_pktq_unlock(q);
125 }
126
127 __attribute__((always_inline))
128 static inline void
nx_pktq_enq(struct nx_pktq * q,struct __kern_packet * p)129 nx_pktq_enq(struct nx_pktq *q, struct __kern_packet *p)
130 {
131 __nx_pktq_enq(q, p);
132 }
133
134 __attribute__((always_inline))
135 static inline void
__nx_pktq_enq_multi(struct nx_pktq * q,struct __kern_packet * p_head,struct __kern_packet * p_tail,uint32_t cnt,uint32_t size)136 __nx_pktq_enq_multi(struct nx_pktq *q, struct __kern_packet *p_head,
137 struct __kern_packet *p_tail,
138 uint32_t cnt, uint32_t size)
139 {
140 classq_pkt_t head, tail;
141
142 CLASSQ_PKT_INIT_PACKET(&head, p_head);
143 CLASSQ_PKT_INIT_PACKET(&tail, p_tail);
144 _addq_multi(&q->nx_pktq_q, &head, &tail, cnt, size);
145 }
146
147 __attribute__((always_inline))
148 static inline void
nx_pktq_safe_enq_multi(struct nx_pktq * q,struct __kern_packet * p_head,struct __kern_packet * p_tail,uint32_t cnt,uint32_t size)149 nx_pktq_safe_enq_multi(struct nx_pktq *q, struct __kern_packet *p_head,
150 struct __kern_packet *p_tail, uint32_t cnt, uint32_t size)
151 {
152 nx_pktq_lock(q);
153 __nx_pktq_enq_multi(q, p_head, p_tail, cnt, size);
154 nx_pktq_unlock(q);
155 }
156
157 __attribute__((always_inline))
158 static inline void
nx_pktq_enq_multi(struct nx_pktq * q,struct __kern_packet * p_head,struct __kern_packet * p_tail,uint32_t cnt,uint32_t size)159 nx_pktq_enq_multi(struct nx_pktq *q, struct __kern_packet *p_head,
160 struct __kern_packet *p_tail, uint32_t cnt, uint32_t size)
161 {
162 __nx_pktq_enq_multi(q, p_head, p_tail, cnt, size);
163 }
164
165 __attribute__((always_inline))
166 static inline struct __kern_packet *
__pktq_deq(struct nx_pktq * q)167 __pktq_deq(struct nx_pktq *q)
168 {
169 classq_pkt_t pkt = CLASSQ_PKT_INITIALIZER(pkt);
170
171 _getq(&q->nx_pktq_q, &pkt);
172 ASSERT((pkt.cp_kpkt == NULL) || (pkt.cp_ptype == QP_PACKET));
173 return pkt.cp_kpkt;
174 }
175
176 __attribute__((always_inline))
177 static inline struct __kern_packet *
nx_pktq_safe_deq(struct nx_pktq * q)178 nx_pktq_safe_deq(struct nx_pktq *q)
179 {
180 struct __kern_packet *ret;
181
182 nx_pktq_lock(q);
183 ret = __pktq_deq(q);
184 nx_pktq_unlock(q);
185
186 return ret;
187 }
188
189 __attribute__((always_inline))
190 static inline struct __kern_packet *
nx_pktq_deq(struct nx_pktq * q)191 nx_pktq_deq(struct nx_pktq *q)
192 {
193 return __pktq_deq(q);
194 }
195
196 __attribute__((always_inline))
197 static inline struct __kern_packet *
__pktq_deq_all(struct nx_pktq * q,struct __kern_packet ** plast,uint32_t * qlenp,uint64_t * qsizep)198 __pktq_deq_all(struct nx_pktq *q, struct __kern_packet **plast, uint32_t *qlenp,
199 uint64_t *qsizep)
200 {
201 classq_pkt_t first = CLASSQ_PKT_INITIALIZER(first);
202 classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
203
204 _getq_all(&q->nx_pktq_q, &first, &last, qlenp, qsizep);
205 *plast = last.cp_kpkt;
206 ASSERT((first.cp_kpkt == NULL) || (first.cp_ptype == QP_PACKET));
207 return first.cp_kpkt;
208 }
209
210 __attribute__((always_inline))
211 static inline struct __kern_packet *
nx_pktq_safe_deq_all(struct nx_pktq * q,struct __kern_packet ** last,uint32_t * qlenp,uint64_t * qsizep)212 nx_pktq_safe_deq_all(struct nx_pktq *q, struct __kern_packet **last,
213 uint32_t *qlenp, uint64_t *qsizep)
214 {
215 struct __kern_packet *ret;
216
217 nx_pktq_lock(q);
218 ret = __pktq_deq_all(q, last, qlenp, qsizep);
219 nx_pktq_unlock(q);
220
221 return ret;
222 }
223
224 __attribute__((always_inline))
225 static inline struct __kern_packet *
nx_pktq_deq_all(struct nx_pktq * q,struct __kern_packet ** last,uint32_t * qlenp,uint64_t * qsizep)226 nx_pktq_deq_all(struct nx_pktq *q, struct __kern_packet **last, uint32_t *qlenp,
227 uint64_t *qsizep)
228 {
229 return __pktq_deq_all(q, last, qlenp, qsizep);
230 }
231
232 __BEGIN_DECLS
233 extern void nx_pktq_init(struct nx_pktq *q, uint32_t lim);
234 extern void nx_pktq_concat(struct nx_pktq *q1, struct nx_pktq *q2);
235 extern boolean_t nx_pktq_empty(struct nx_pktq *q);
236 extern void nx_pktq_destroy(struct nx_pktq *q);
237 extern void nx_pktq_purge(struct nx_pktq *q);
238
239 extern void nx_pktq_safe_init(struct __kern_channel_ring *kr, struct nx_pktq *q,
240 uint32_t lim, lck_grp_t *lck_grp, lck_attr_t *lck_attr);
241 extern void nx_pktq_safe_destroy(struct nx_pktq *q);
242 extern void nx_pktq_safe_purge(struct nx_pktq *q);
243 __END_DECLS
244 #endif /* _SKYWALK_NEXUS_PKTQ_H_ */
245