1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_PACKET_COMMON_H_
30 #define _SKYWALK_PACKET_COMMON_H_
31
32 #if defined(PRIVATE) || defined(BSD_KERNEL_PRIVATE)
33 /*
34 * Routines common to kernel and userland. This file is intended to
35 * be included by code implementing the packet APIs, in particular,
36 * the Skywalk kernel and libsyscall code.
37 */
38
39 #include <skywalk/os_packet_private.h>
40 #include <net/if_vlan_var.h>
41 #include <sys/errno.h>
42 #include <sys/kdebug.h>
43
44 #ifndef KERNEL
45 /*
46 * User.
47 */
48 #if !defined(LIBSYSCALL_INTERFACE)
49 #error "LIBSYSCALL_INTERFACE not defined"
50 #endif /* !LIBSYSCALL_INTERFACE */
51 #define QUM_ADDR(_ph) SK_PTR_ADDR_UQUM(_ph)
52 #define PKT_ADDR(_ph) SK_PTR_ADDR_UPKT(_ph)
53 #define BLT_ADDR(_bp) ((struct __user_buflet *)(uintptr_t)_bp)
54 #else /* KERNEL */
55 /*
56 * Kernel.
57 */
58 #include <skywalk/packet/packet_var.h>
59 #include <skywalk/packet/pbufpool_var.h>
60 #define QUM_ADDR(_ph) SK_PTR_ADDR_KQUM(_ph)
61 #define PKT_ADDR(_ph) SK_PTR_ADDR_KPKT(_ph)
62 #define BLT_ADDR(_bp) ((struct __kern_buflet *)(uintptr_t)_bp)
63 #define PKT_HAS_ATTACHED_MBUF(_ph) \
64 ((PKT_ADDR(_ph)->pkt_pflags & PKT_F_MBUF_DATA) != 0)
65 #endif /* KERNEL */
66
67 /*
68 * Common.
69 */
70 #if (DEBUG || DEVELOPMENT)
71 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) do { \
72 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type) || \
73 SK_PTR_SUBTYPE(_ph) != (uint64_t)(_subtype))) { \
74 pkt_subtype_assert_fail(_ph, _type, _subtype); \
75 /* NOTREACHED */ \
76 __builtin_unreachable(); \
77 } \
78 } while (0)
79
80 #define PKT_TYPE_ASSERT(_ph, _type) do { \
81 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type))) { \
82 pkt_type_assert_fail(_ph, _type); \
83 /* NOTREACHED */ \
84 __builtin_unreachable(); \
85 } \
86 } while (0)
87 #else /* !DEBUG && !DEVELOPMENT */
88 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) ((void)0)
89 #define PKT_TYPE_ASSERT(_ph, _type) ((void)0)
90 #endif /* !DEBUG && !DEVELOPMENT */
91
92 #define QUM_GET_NEXT_BUFLET(_qum, _pbuf, _buf) do { \
93 ASSERT((_pbuf) == NULL || (_pbuf) == (_qum)->qum_buf); \
94 (_buf) = (((_pbuf) == NULL) ? (_qum)->qum_buf : NULL); \
95 } while (0)
96
97 #define PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf) do { \
98 if (__improbable((_bcnt) == 0)) { \
99 (_buf) = NULL; \
100 break; \
101 } \
102 if (__probable((_pkt)->pkt_qum_buf.buf_addr != 0)) { \
103 (_buf) = &(_pkt)->pkt_qum_buf; \
104 } else { \
105 (_buf) = __DECONST(void *, (_pkt)->pkt_qum_buf.buf_nbft_addr);\
106 } \
107 } while (0)
108
109 #define _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
110 if ((_pbuf) == NULL) { \
111 PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf); \
112 } else { \
113 (_buf) = __DECONST(void *, (_pbuf)->buf_nbft_addr); \
114 } \
115 } while (0)
116
117 #ifndef KERNEL
118 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
119 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
120 } while (0)
121 #else /* KERNEL */
122 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
123 ASSERT(((_bcnt) >= 1) || ((_pbuf) == NULL)); \
124 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
125 } while (0)
126 #endif /* KERNEL */
127
128 #ifdef KERNEL
129 #define PKT_COMPOSE_NX_PORT_ID(_nx_port, _gencnt) \
130 ((uint32_t)((_gencnt & 0xffff) << 16) | (_nx_port & 0xffff))
131
132 #define PKT_DECOMPOSE_NX_PORT_ID(_nx_port_id, _nx_port, _gencnt) do { \
133 _nx_port = _nx_port_id & 0xffff; \
134 _gencnt = (_nx_port_id >> 16) & 0xffff; \
135 } while (0)
136 #endif /* KERNEL */
137
138 __attribute__((always_inline))
139 static inline int
__packet_set_headroom(const uint64_t ph,const uint8_t headroom)140 __packet_set_headroom(const uint64_t ph, const uint8_t headroom)
141 {
142 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
143 if (__probable(headroom < PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
144 PKT_ADDR(ph)->pkt_headroom = headroom;
145 return 0;
146 }
147 return ERANGE;
148 }
149
150 __attribute__((always_inline))
151 static inline uint8_t
__packet_get_headroom(const uint64_t ph)152 __packet_get_headroom(const uint64_t ph)
153 {
154 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
155 return PKT_ADDR(ph)->pkt_headroom;
156 }
157
158 __attribute__((always_inline))
159 static inline int
__packet_set_link_header_length(const uint64_t ph,const uint8_t len)160 __packet_set_link_header_length(const uint64_t ph, const uint8_t len)
161 {
162 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
163 if (__probable(len <= PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
164 PKT_ADDR(ph)->pkt_l2_len = len;
165 return 0;
166 }
167 return ERANGE;
168 }
169
170 __attribute__((always_inline))
171 static inline uint8_t
__packet_get_link_header_length(const uint64_t ph)172 __packet_get_link_header_length(const uint64_t ph)
173 {
174 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
175 return PKT_ADDR(ph)->pkt_l2_len;
176 }
177
178 __attribute__((always_inline))
179 static inline int
__packet_set_link_broadcast(const uint64_t ph)180 __packet_set_link_broadcast(const uint64_t ph)
181 {
182 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
183 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_BCAST;
184 return 0;
185 }
186
187 __attribute__((always_inline))
188 static inline boolean_t
__packet_get_link_broadcast(const uint64_t ph)189 __packet_get_link_broadcast(const uint64_t ph)
190 {
191 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
192 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_BCAST) != 0;
193 }
194
195 __attribute__((always_inline))
196 static inline int
__packet_set_link_multicast(const uint64_t ph)197 __packet_set_link_multicast(const uint64_t ph)
198 {
199 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
200 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_MCAST;
201 return 0;
202 }
203
204 __attribute__((always_inline))
205 static inline boolean_t
__packet_get_link_multicast(const uint64_t ph)206 __packet_get_link_multicast(const uint64_t ph)
207 {
208 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
209 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_MCAST) != 0;
210 }
211
212 __attribute__((always_inline))
213 static inline int
__packet_set_link_ethfcs(const uint64_t ph)214 __packet_set_link_ethfcs(const uint64_t ph)
215 {
216 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
217 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_ETHFCS;
218 return 0;
219 }
220
221 __attribute__((always_inline))
222 static inline boolean_t
__packet_get_link_ethfcs(const uint64_t ph)223 __packet_get_link_ethfcs(const uint64_t ph)
224 {
225 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
226 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_ETHFCS) != 0;
227 }
228
229 __attribute__((always_inline))
230 static inline int
__packet_set_transport_traffic_background(const uint64_t ph)231 __packet_set_transport_traffic_background(const uint64_t ph)
232 {
233 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
234 PKT_ADDR(ph)->pkt_pflags |= PKT_F_BACKGROUND;
235 return 0;
236 }
237
238 __attribute__((always_inline))
239 static inline boolean_t
__packet_get_transport_traffic_background(const uint64_t ph)240 __packet_get_transport_traffic_background(const uint64_t ph)
241 {
242 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
243 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_BACKGROUND) != 0;
244 }
245
246 __attribute__((always_inline))
247 static inline int
__packet_set_transport_traffic_realtime(const uint64_t ph)248 __packet_set_transport_traffic_realtime(const uint64_t ph)
249 {
250 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
251 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REALTIME;
252 return 0;
253 }
254
255 __attribute__((always_inline))
256 static inline boolean_t
__packet_get_transport_traffic_realtime(const uint64_t ph)257 __packet_get_transport_traffic_realtime(const uint64_t ph)
258 {
259 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
260 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REALTIME) != 0;
261 }
262
263 __attribute__((always_inline))
264 static inline int
__packet_set_transport_retransmit(const uint64_t ph)265 __packet_set_transport_retransmit(const uint64_t ph)
266 {
267 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
268 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REXMT;
269 return 0;
270 }
271
272 __attribute__((always_inline))
273 static inline boolean_t
__packet_get_transport_retransmit(const uint64_t ph)274 __packet_get_transport_retransmit(const uint64_t ph)
275 {
276 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
277 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REXMT) != 0;
278 }
279
280 __attribute__((always_inline))
281 static inline int
__packet_set_transport_last_packet(const uint64_t ph)282 __packet_set_transport_last_packet(const uint64_t ph)
283 {
284 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
285 PKT_ADDR(ph)->pkt_pflags |= PKT_F_LAST_PKT;
286 return 0;
287 }
288
289 __attribute__((always_inline))
290 static inline int
__packet_set_group_start(const uint64_t ph)291 __packet_set_group_start(const uint64_t ph)
292 {
293 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
294 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_START;
295 return 0;
296 }
297
298 __attribute__((always_inline))
299 static inline boolean_t
__packet_get_group_start(const uint64_t ph)300 __packet_get_group_start(const uint64_t ph)
301 {
302 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
303 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_START) != 0;
304 }
305
306 __attribute__((always_inline))
307 static inline int
__packet_set_group_end(const uint64_t ph)308 __packet_set_group_end(const uint64_t ph)
309 {
310 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
311 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_END;
312 return 0;
313 }
314
315 __attribute__((always_inline))
316 static inline boolean_t
__packet_get_group_end(const uint64_t ph)317 __packet_get_group_end(const uint64_t ph)
318 {
319 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
320 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_END) != 0;
321 }
322
323 __attribute__((always_inline))
324 static inline errno_t
__packet_get_expire_time(const uint64_t ph,uint64_t * ts)325 __packet_get_expire_time(const uint64_t ph, uint64_t *ts)
326 {
327 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
328 #ifdef KERNEL
329 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
330 #else /* !KERNEL */
331 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
332 #endif /* !KERNEL */
333 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) == 0) {
334 return ENOENT;
335 }
336 if (ts == NULL) {
337 return EINVAL;
338 }
339 *ts = po->__po_expire_ts;
340 return 0;
341 }
342
343 __attribute__((always_inline))
344 static inline errno_t
__packet_set_expire_time(const uint64_t ph,const uint64_t ts)345 __packet_set_expire_time(const uint64_t ph, const uint64_t ts)
346 {
347 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
348 #ifdef KERNEL
349 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
350 #else /* !KERNEL */
351 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
352 #endif /* !KERNEL */
353 if (ts != 0) {
354 po->__po_expire_ts = ts;
355 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXPIRE_TS;
356 } else {
357 po->__po_expire_ts = 0;
358 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXPIRE_TS;
359 }
360 return 0;
361 }
362
363 __attribute__((always_inline))
364 static inline errno_t
__packet_get_expiry_action(const uint64_t ph,packet_expiry_action_t * pea)365 __packet_get_expiry_action(const uint64_t ph, packet_expiry_action_t *pea)
366 {
367 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
368 #ifdef KERNEL
369 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
370 #else /* !KERNEL */
371 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
372 #endif /* !KERNEL */
373 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXP_ACTION) == 0) {
374 return ENOENT;
375 }
376 if (pea == NULL) {
377 return EINVAL;
378 }
379 *pea = po->__po_expiry_action;
380 return 0;
381 }
382
383 __attribute__((always_inline))
384 static inline errno_t
__packet_set_expiry_action(const uint64_t ph,packet_expiry_action_t pea)385 __packet_set_expiry_action(const uint64_t ph, packet_expiry_action_t pea)
386 {
387 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
388 #ifdef KERNEL
389 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
390 #else /* !KERNEL */
391 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
392 #endif /* !KERNEL */
393 if (pea != PACKET_EXPIRY_ACTION_NONE) {
394 po->__po_expiry_action = (uint8_t)pea;
395 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXP_ACTION;
396 } else {
397 po->__po_expiry_action = 0;
398 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXP_ACTION;
399 }
400 return 0;
401 }
402
403 __attribute__((always_inline))
404 static inline errno_t
__packet_opt_get_token(const struct __packet_opt * po,void * token,uint16_t * len,uint8_t * type)405 __packet_opt_get_token(const struct __packet_opt *po, void *token,
406 uint16_t *len, uint8_t *type)
407 {
408 uint16_t tlen = po->__po_token_len;
409 uint8_t ttype;
410
411 if (token == NULL || len == NULL || type == NULL || tlen > *len) {
412 return EINVAL;
413 }
414 ttype = (uint8_t)po->__po_token_type;
415
416 ASSERT(tlen <= PKT_OPT_MAX_TOKEN_SIZE);
417 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
418 bcopy(po->__po_token, token, tlen);
419 *len = tlen;
420 *type = ttype;
421 return 0;
422 }
423
424 __attribute__((always_inline))
425 static inline errno_t
__packet_get_token(const uint64_t ph,void * token,uint16_t * len)426 __packet_get_token(const uint64_t ph, void *token, uint16_t *len)
427 {
428 #ifdef KERNEL
429 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
430 #else /* !KERNEL */
431 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
432 #endif /* !KERNEL */
433 uint8_t type;
434 errno_t err;
435
436 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
437 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
438 return ENOENT;
439 }
440 err = __packet_opt_get_token(po, token, len, &type);
441 if ((err == 0) && (type != PKT_OPT_TOKEN_TYPE_OPAQUE)) {
442 err = ENOENT;
443 }
444 return err;
445 }
446
447 __attribute__((always_inline))
448 static inline errno_t
__packet_opt_set_token(struct __packet_opt * po,const void * token,const uint16_t len,const uint8_t type,volatile uint64_t * pflags)449 __packet_opt_set_token(struct __packet_opt *po, const void *token,
450 const uint16_t len, const uint8_t type, volatile uint64_t *pflags)
451 {
452 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
453 if (len != 0) {
454 if (token == NULL || len > PKT_OPT_MAX_TOKEN_SIZE ||
455 type == 0) {
456 return EINVAL;
457 }
458 if (__probable(IS_P2ALIGNED(token, 8))) {
459 uint64_t *token64 = __DECONST(void *, token);
460 po->__po_token_data[0] = *token64;
461 po->__po_token_data[1] = *(token64 + 1);
462 } else {
463 bcopy(token, po->__po_token, len);
464 }
465 po->__po_token_len = len;
466 po->__po_token_type = type;
467 *pflags |= PKT_F_OPT_TOKEN;
468 } else {
469 _CASSERT(sizeof(po->__po_token_data[0]) == 8);
470 _CASSERT(sizeof(po->__po_token_data[1]) == 8);
471 _CASSERT(sizeof(po->__po_token) == 16);
472 po->__po_token_data[0] = 0;
473 po->__po_token_data[1] = 0;
474 po->__po_token_len = 0;
475 po->__po_token_type = 0;
476 *pflags &= ~PKT_F_OPT_TOKEN;
477 }
478 return 0;
479 }
480
481 #ifndef KERNEL
482 __attribute__((always_inline))
483 static inline void
__packet_set_tx_timestamp(const uint64_t ph,const uint64_t ts)484 __packet_set_tx_timestamp(const uint64_t ph, const uint64_t ts)
485 {
486 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
487 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
488
489 po->__po_pkt_tx_time = ts;
490 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_TX_TIMESTAMP;
491 }
492 #endif /* !KERNEL */
493
494 __attribute__((always_inline))
495 static inline errno_t
__packet_set_token(const uint64_t ph,const void * token,const uint16_t len)496 __packet_set_token(const uint64_t ph, const void *token, const uint16_t len)
497 {
498 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
499 #ifdef KERNEL
500 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, token, len,
501 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
502 #else /* !KERNEL */
503 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, token, len,
504 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
505 #endif /* !KERNEL */
506 }
507
508 __attribute__((always_inline))
509 static inline errno_t
__packet_get_packetid(const uint64_t ph,packet_id_t * pktid)510 __packet_get_packetid(const uint64_t ph, packet_id_t *pktid)
511 {
512 #ifdef KERNEL
513 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
514 #else /* !KERNEL */
515 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
516 #endif /* !KERNEL */
517 uint16_t len = sizeof(packet_id_t);
518 uint8_t type;
519 errno_t err;
520
521 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
522 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
523 return ENOENT;
524 }
525 err = __packet_opt_get_token(po, pktid, &len, &type);
526 if ((err == 0) && ((type != PKT_OPT_TOKEN_TYPE_PACKET_ID) ||
527 (len != sizeof(packet_id_t)))) {
528 err = ENOENT;
529 }
530 return err;
531 }
532
533 __attribute__((always_inline))
534 static inline errno_t
__packet_set_packetid(const uint64_t ph,const packet_id_t * pktid)535 __packet_set_packetid(const uint64_t ph, const packet_id_t *pktid)
536 {
537 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
538 #ifdef KERNEL
539 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, pktid,
540 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
541 &PKT_ADDR(ph)->pkt_pflags);
542 #else /* !KERNEL */
543 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, pktid,
544 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
545 &PKT_ADDR(ph)->pkt_pflags);
546 #endif /* !KERNEL */
547 }
548
549 __attribute__((always_inline))
550 static inline errno_t
__packet_get_vlan_tag(const uint64_t ph,uint16_t * vlan_tag,boolean_t * tag_in_pkt)551 __packet_get_vlan_tag(const uint64_t ph, uint16_t *vlan_tag,
552 boolean_t *tag_in_pkt)
553 {
554 #ifdef KERNEL
555 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
556 #else /* !KERNEL */
557 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
558 #endif /* !KERNEL */
559 uint64_t pflags;
560
561 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
562 pflags = PKT_ADDR(ph)->pkt_pflags;
563 if ((pflags & PKT_F_OPT_VLTAG) == 0) {
564 return ENOENT;
565 }
566 if (vlan_tag != NULL) {
567 *vlan_tag = po->__po_vlan_tag;
568 }
569 if (tag_in_pkt != NULL) {
570 *tag_in_pkt = ((pflags & PKT_F_OPT_VLTAG_IN_PKT) != 0);
571 }
572 return 0;
573 }
574
575 __attribute__((always_inline))
576 static inline errno_t
__packet_set_vlan_tag(const uint64_t ph,const uint16_t vlan_tag,const boolean_t tag_in_pkt)577 __packet_set_vlan_tag(const uint64_t ph, const uint16_t vlan_tag,
578 const boolean_t tag_in_pkt)
579 {
580 #ifdef KERNEL
581 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
582 #else /* !KERNEL */
583 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
584 #endif /* !KERNEL */
585
586 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
587 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG;
588 po->__po_vlan_tag = vlan_tag;
589
590 if (tag_in_pkt) {
591 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG_IN_PKT;
592 }
593 return 0;
594 }
595
596 __attribute__((always_inline))
597 static inline uint16_t
__packet_get_vlan_id(const uint16_t vlan_tag)598 __packet_get_vlan_id(const uint16_t vlan_tag)
599 {
600 return EVL_VLANOFTAG(vlan_tag);
601 }
602
603 __attribute__((always_inline))
604 static inline uint8_t
__packet_get_vlan_priority(const uint16_t vlan_tag)605 __packet_get_vlan_priority(const uint16_t vlan_tag)
606 {
607 return EVL_PRIOFTAG(vlan_tag);
608 }
609
610 __attribute__((always_inline))
611 static inline errno_t
__packet_get_app_metadata(const uint64_t ph,packet_app_metadata_type_t * app_type,uint8_t * app_metadata)612 __packet_get_app_metadata(const uint64_t ph,
613 packet_app_metadata_type_t *app_type, uint8_t *app_metadata)
614 {
615 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
616 if (app_type == NULL || app_metadata == NULL) {
617 return EINVAL;
618 }
619 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_APP_METADATA) == 0) {
620 return ENOENT;
621 }
622 #ifdef KERNEL
623 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
624 #else /* !KERNEL */
625 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
626 #endif /* !KERNEL */
627 if (po->__po_app_type == PACKET_APP_METADATA_TYPE_UNSPECIFIED) {
628 return ENOENT;
629 }
630 *app_type = po->__po_app_type;
631 *app_metadata = po->__po_app_metadata;
632 return 0;
633 }
634
635 __attribute__((always_inline))
636 static inline errno_t
__packet_set_app_metadata(const uint64_t ph,const packet_app_metadata_type_t app_type,const uint8_t app_metadata)637 __packet_set_app_metadata(const uint64_t ph,
638 const packet_app_metadata_type_t app_type, const uint8_t app_metadata)
639 {
640 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
641 #ifdef KERNEL
642 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
643 #else /* !KERNEL */
644 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
645 #endif /* !KERNEL */
646 if (app_type < PACKET_APP_METADATA_TYPE_MIN ||
647 app_type > PACKET_APP_METADATA_TYPE_MAX) {
648 po->__po_app_type = PACKET_APP_METADATA_TYPE_UNSPECIFIED;
649 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_APP_METADATA;
650 return EINVAL;
651 }
652 po->__po_app_type = app_type;
653 po->__po_app_metadata = app_metadata;
654 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_APP_METADATA;
655 return 0;
656 }
657
658 #ifdef KERNEL
659 __attribute__((always_inline))
660 static inline void
__packet_set_wake_flag(const uint64_t ph)661 __packet_set_wake_flag(const uint64_t ph)
662 {
663 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
664 PKT_ADDR(ph)->pkt_pflags |= PKT_F_WAKE_PKT;
665 }
666 #endif
667
668 __attribute__((always_inline))
669 static inline boolean_t
__packet_get_wake_flag(const uint64_t ph)670 __packet_get_wake_flag(const uint64_t ph)
671 {
672 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_WAKE_PKT) != 0;
673 }
674
675 __attribute__((always_inline))
676 static inline void
__packet_set_keep_alive(const uint64_t ph,const boolean_t is_keep_alive)677 __packet_set_keep_alive(const uint64_t ph, const boolean_t is_keep_alive)
678 {
679 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
680 if (is_keep_alive) {
681 PKT_ADDR(ph)->pkt_pflags |= PKT_F_KEEPALIVE;
682 } else {
683 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_KEEPALIVE;
684 }
685 }
686
687 __attribute__((always_inline))
688 static inline boolean_t
__packet_get_keep_alive(const uint64_t ph)689 __packet_get_keep_alive(const uint64_t ph)
690 {
691 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_KEEPALIVE) != 0;
692 }
693
694 __attribute__((always_inline))
695 static inline boolean_t
__packet_get_truncated(const uint64_t ph)696 __packet_get_truncated(const uint64_t ph)
697 {
698 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
699 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_TRUNCATED) != 0;
700 }
701
702 #ifdef KERNEL
703 __attribute__((always_inline))
704 static inline boolean_t
__packet_get_transport_new_flow(const uint64_t ph)705 __packet_get_transport_new_flow(const uint64_t ph)
706 {
707 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
708 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_NEW_FLOW) != 0;
709 }
710
711 __attribute__((always_inline))
712 static inline boolean_t
__packet_get_transport_last_packet(const uint64_t ph)713 __packet_get_transport_last_packet(const uint64_t ph)
714 {
715 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
716 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_LAST_PKT) != 0;
717 }
718
719 __attribute__((always_inline))
720 static inline boolean_t
__packet_get_l4s_flag(const uint64_t ph)721 __packet_get_l4s_flag(const uint64_t ph)
722 {
723 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
724 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_L4S) != 0;
725 }
726 #endif /* KERNEL */
727
728 __attribute__((always_inline))
729 static inline void
__packet_set_l4s_flag(const uint64_t ph)730 __packet_set_l4s_flag(const uint64_t ph)
731 {
732 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
733 PKT_ADDR(ph)->pkt_pflags |= PKT_F_L4S;
734 }
735
736 __attribute__((always_inline))
737 static inline int
__packet_set_service_class(const uint64_t ph,const uint32_t sc)738 __packet_set_service_class(const uint64_t ph, const uint32_t sc)
739 {
740 int err = 0;
741
742 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
743
744 switch (sc) {
745 case PKT_SC_BE:
746 case PKT_SC_BK_SYS:
747 case PKT_SC_BK:
748 case PKT_SC_RD:
749 case PKT_SC_OAM:
750 case PKT_SC_AV:
751 case PKT_SC_RV:
752 case PKT_SC_VI:
753 case PKT_SC_SIG:
754 case PKT_SC_VO:
755 case PKT_SC_CTL:
756 QUM_ADDR(ph)->qum_svc_class = sc;
757 break;
758
759 default:
760 err = EINVAL;
761 break;
762 }
763
764 return err;
765 }
766
767 __attribute__((always_inline))
768 static inline uint32_t
__packet_get_service_class(const uint64_t ph)769 __packet_get_service_class(const uint64_t ph)
770 {
771 uint32_t sc;
772
773 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
774
775 switch (QUM_ADDR(ph)->qum_svc_class) {
776 case PKT_SC_BE: /* most likely best effort */
777 case PKT_SC_BK_SYS:
778 case PKT_SC_BK:
779 case PKT_SC_RD:
780 case PKT_SC_OAM:
781 case PKT_SC_AV:
782 case PKT_SC_RV:
783 case PKT_SC_VI:
784 case PKT_SC_SIG:
785 case PKT_SC_VO:
786 case PKT_SC_CTL:
787 sc = QUM_ADDR(ph)->qum_svc_class;
788 break;
789
790 default:
791 sc = PKT_SC_BE;
792 break;
793 }
794
795 return sc;
796 }
797
798 __attribute__((always_inline))
799 static inline errno_t
__packet_set_comp_gencnt(const uint64_t ph,const uint32_t gencnt)800 __packet_set_comp_gencnt(const uint64_t ph, const uint32_t gencnt)
801 {
802 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
803 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
804
805 PKT_ADDR(ph)->pkt_comp_gencnt = gencnt;
806
807 return 0;
808 }
809
810 __attribute__((always_inline))
811 static inline errno_t
__packet_get_comp_gencnt(const uint64_t ph,uint32_t * pgencnt)812 __packet_get_comp_gencnt(const uint64_t ph, uint32_t *pgencnt)
813 {
814 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
815 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
816
817 if (pgencnt == NULL) {
818 return EINVAL;
819 }
820
821 if (PKT_ADDR(ph)->pkt_comp_gencnt == 0) {
822 return ENOENT;
823 }
824
825 *pgencnt = PKT_ADDR(ph)->pkt_comp_gencnt;
826 return 0;
827 }
828
829 __attribute__((always_inline))
830 static inline int
__packet_set_traffic_class(const uint64_t ph,const uint32_t tc)831 __packet_set_traffic_class(const uint64_t ph, const uint32_t tc)
832 {
833 uint32_t val = PKT_TC2SCVAL(tc); /* just the val portion */
834 uint32_t sc;
835
836 switch (val) {
837 case PKT_SCVAL_BK_SYS:
838 sc = PKT_SC_BK_SYS;
839 break;
840 case PKT_SCVAL_BK:
841 sc = PKT_SC_BK;
842 break;
843 case PKT_SCVAL_BE:
844 sc = PKT_SC_BE;
845 break;
846 case PKT_SCVAL_RD:
847 sc = PKT_SC_RD;
848 break;
849 case PKT_SCVAL_OAM:
850 sc = PKT_SC_OAM;
851 break;
852 case PKT_SCVAL_AV:
853 sc = PKT_SC_AV;
854 break;
855 case PKT_SCVAL_RV:
856 sc = PKT_SC_RV;
857 break;
858 case PKT_SCVAL_VI:
859 sc = PKT_SC_VI;
860 break;
861 case PKT_SCVAL_SIG:
862 sc = PKT_SC_SIG;
863 break;
864 case PKT_SCVAL_VO:
865 sc = PKT_SC_VO;
866 break;
867 case PKT_SCVAL_CTL:
868 sc = PKT_SC_CTL;
869 break;
870 default:
871 sc = PKT_SC_BE;
872 break;
873 }
874
875 return __packet_set_service_class(ph, sc);
876 }
877
878 __attribute__((always_inline))
879 static inline uint32_t
__packet_get_traffic_class(const uint64_t ph)880 __packet_get_traffic_class(const uint64_t ph)
881 {
882 return PKT_SC2TC(__packet_get_service_class(ph));
883 }
884
885 __attribute__((always_inline))
886 static inline int
__packet_set_inet_checksum(const uint64_t ph,const packet_csum_flags_t flags,const uint16_t start,const uint16_t stuff_val,boolean_t tx)887 __packet_set_inet_checksum(const uint64_t ph, const packet_csum_flags_t flags,
888 const uint16_t start, const uint16_t stuff_val, boolean_t tx)
889 {
890 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
891
892 PKT_ADDR(ph)->pkt_csum_flags = flags & PACKET_CSUM_FLAGS;
893
894 if (tx) {
895 PKT_ADDR(ph)->pkt_csum_tx_start_off = start;
896 PKT_ADDR(ph)->pkt_csum_tx_stuff_off = stuff_val;
897 } else {
898 PKT_ADDR(ph)->pkt_csum_rx_start_off = start;
899 PKT_ADDR(ph)->pkt_csum_rx_value = stuff_val;
900 }
901 return 0;
902 }
903
904 __attribute__((always_inline))
905 static inline void
__packet_add_inet_csum_flags(const uint64_t ph,const packet_csum_flags_t flags)906 __packet_add_inet_csum_flags(const uint64_t ph, const packet_csum_flags_t flags)
907 {
908 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
909
910 PKT_ADDR(ph)->pkt_csum_flags |= flags & PACKET_CSUM_FLAGS;
911 }
912
913 __attribute__((always_inline))
914 static inline packet_csum_flags_t
__packet_get_inet_checksum(const uint64_t ph,uint16_t * start,uint16_t * stuff_val,boolean_t tx)915 __packet_get_inet_checksum(const uint64_t ph, uint16_t *start,
916 uint16_t *stuff_val, boolean_t tx)
917 {
918 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
919
920 if (tx) {
921 if (__probable(start != NULL)) {
922 *start = PKT_ADDR(ph)->pkt_csum_tx_start_off;
923 }
924 if (__probable(stuff_val != NULL)) {
925 *stuff_val = PKT_ADDR(ph)->pkt_csum_tx_stuff_off;
926 }
927 } else {
928 if (__probable(start != NULL)) {
929 *start = PKT_ADDR(ph)->pkt_csum_rx_start_off;
930 }
931 if (__probable(stuff_val != NULL)) {
932 *stuff_val = PKT_ADDR(ph)->pkt_csum_rx_value;
933 }
934 }
935 return PKT_ADDR(ph)->pkt_csum_flags & PACKET_CSUM_FLAGS;
936 }
937
938 __attribute__((always_inline))
939 static inline void
__packet_set_flow_uuid(const uint64_t ph,const uuid_t flow_uuid)940 __packet_set_flow_uuid(const uint64_t ph, const uuid_t flow_uuid)
941 {
942 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
943
944 /*
945 * Anticipate a nicely (8-bytes) aligned UUID from caller;
946 * the one in qum_flow_id is always 8-byte aligned.
947 */
948 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
949 uint64_t *id_64 = (uint64_t *)(uintptr_t)flow_uuid;
950 q->__q_flow_id_val64[0] = id_64[0];
951 q->__q_flow_id_val64[1] = id_64[1];
952 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
953 uint32_t *id_32 = (uint32_t *)(uintptr_t)flow_uuid;
954 q->__q_flow_id_val32[0] = id_32[0];
955 q->__q_flow_id_val32[1] = id_32[1];
956 q->__q_flow_id_val32[2] = id_32[2];
957 q->__q_flow_id_val32[3] = id_32[3];
958 } else {
959 bcopy(flow_uuid, q->__q_flow_id, sizeof(uuid_t));
960 }
961 }
962
963 __attribute__((always_inline))
964 static inline void
__packet_get_flow_uuid(const uint64_t ph,uuid_t flow_uuid)965 __packet_get_flow_uuid(const uint64_t ph, uuid_t flow_uuid)
966 {
967 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
968
969 /*
970 * Anticipate a nicely (8-bytes) aligned UUID from caller;
971 * the one in qum_flow_id is always 8-byte aligned.
972 */
973 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
974 uint64_t *id_64 = (uint64_t *)(uintptr_t)flow_uuid;
975 id_64[0] = q->__q_flow_id_val64[0];
976 id_64[1] = q->__q_flow_id_val64[1];
977 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
978 uint32_t *id_32 = (uint32_t *)(uintptr_t)flow_uuid;
979 id_32[0] = q->__q_flow_id_val32[0];
980 id_32[1] = q->__q_flow_id_val32[1];
981 id_32[2] = q->__q_flow_id_val32[2];
982 id_32[3] = q->__q_flow_id_val32[3];
983 } else {
984 bcopy(q->__q_flow_id, flow_uuid, sizeof(uuid_t));
985 }
986 }
987
988 __attribute__((always_inline))
989 static inline void
__packet_clear_flow_uuid(const uint64_t ph)990 __packet_clear_flow_uuid(const uint64_t ph)
991 {
992 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
993 q->__q_flow_id_val64[0] = 0;
994 q->__q_flow_id_val64[1] = 0;
995 }
996
997 __attribute__((always_inline))
998 static inline uint8_t
__packet_get_aggregation_type(const uint64_t ph)999 __packet_get_aggregation_type(const uint64_t ph)
1000 {
1001 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_aggr_type == sizeof(uint8_t)));
1002 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1003
1004 return PKT_ADDR(ph)->pkt_aggr_type;
1005 }
1006
1007 __attribute__((always_inline))
1008 static inline uint32_t
__packet_get_data_length(const uint64_t ph)1009 __packet_get_data_length(const uint64_t ph)
1010 {
1011 return QUM_ADDR(ph)->qum_len;
1012 }
1013
1014 __attribute__((always_inline))
1015 static inline uint16_t
__packet_get_buflet_count(const uint64_t ph)1016 __packet_get_buflet_count(const uint64_t ph)
1017 {
1018 uint16_t bcnt = 0;
1019
1020 switch (SK_PTR_TYPE(ph)) {
1021 case NEXUS_META_TYPE_PACKET:
1022 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1023 #ifdef KERNEL
1024 VERIFY(bcnt != 0 ||
1025 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1026 #else /* !KERNEL */
1027 /*
1028 * Handle the case where the metadata region gets
1029 * redirected to anonymous zero-filled pages at
1030 * defunct time. There's always 1 buflet in the
1031 * packet metadata, so pretend that's the count.
1032 */
1033 if (__improbable(bcnt == 0)) {
1034 bcnt = 1;
1035 }
1036 #endif /* !KERNEL */
1037 break;
1038 case NEXUS_META_TYPE_QUANTUM:
1039 bcnt = 1;
1040 break;
1041 default:
1042 #ifdef KERNEL
1043 VERIFY(0);
1044 /* NOTREACHED */
1045 __builtin_unreachable();
1046 #endif /* KERNEL */
1047 break;
1048 }
1049 return bcnt;
1050 }
1051
1052 __attribute__((always_inline))
1053 static inline int
__packet_add_buflet(const uint64_t ph,const void * bprev0,const void * bnew0)1054 __packet_add_buflet(const uint64_t ph, const void *bprev0, const void *bnew0)
1055 {
1056 uint16_t bcnt;
1057
1058 #ifdef KERNEL
1059 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1060 kern_buflet_t bnew = __DECONST(kern_buflet_t, bnew0);
1061
1062 VERIFY(PKT_ADDR(ph) && bnew && (bnew != bprev));
1063 VERIFY(PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1064 #else /* !KERNEL */
1065 buflet_t bprev = __DECONST(buflet_t, bprev0);
1066 buflet_t bnew = __DECONST(buflet_t, bnew0);
1067
1068 if (__improbable(!PKT_ADDR(ph) || !bnew || (bnew == bprev))) {
1069 return EINVAL;
1070 }
1071 #endif /* !KERNEL */
1072
1073 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1074 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1075
1076 #ifdef KERNEL
1077 VERIFY((bprev != NULL || bcnt == 0) &&
1078 (bcnt < PKT_ADDR(ph)->pkt_bufs_max));
1079 #else /* !KERNEL */
1080 if (__improbable(bcnt >= PKT_ADDR(ph)->pkt_bufs_max) ||
1081 (bprev == NULL && bcnt != 0)) {
1082 return EINVAL;
1083 }
1084 #endif /* !KERNEL */
1085
1086 #ifdef KERNEL
1087 #if DEVELOPMENT || DEBUG
1088 /* check if bprev is the last buflet in the chain */
1089 struct __kern_buflet *pbft, *kbft;
1090 int n = bcnt;
1091
1092 PKT_GET_FIRST_BUFLET(PKT_ADDR(ph), bcnt, pbft);
1093 kbft = pbft;
1094
1095 while ((kbft != NULL) && n--) {
1096 pbft = kbft;
1097 kbft = __DECONST(struct __kern_buflet *, kbft->buf_nbft_addr);
1098 }
1099 ASSERT(n == 0);
1100 ASSERT(bprev == pbft);
1101 #endif /* DEVELOPMENT || DEBUG */
1102 #endif /* KERNEL */
1103
1104 if (bprev == NULL) {
1105 bprev = &PKT_ADDR(ph)->pkt_qum_buf;
1106 }
1107 #ifdef KERNEL
1108 KBUF_LINK(bprev, bnew);
1109 #else /* !KERNEL */
1110 UBUF_LINK(bprev, bnew);
1111 #endif /* !KERNEL */
1112
1113 *(uint16_t *)(uintptr_t)&PKT_ADDR(ph)->pkt_bufs_cnt = ++bcnt;
1114 return 0;
1115 }
1116
1117 __attribute__((always_inline))
1118 static inline void *
__packet_get_next_buflet(const uint64_t ph,const void * bprev0)1119 __packet_get_next_buflet(const uint64_t ph, const void *bprev0)
1120 {
1121 #ifdef KERNEL
1122 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1123 #else /* !KERNEL */
1124 buflet_t bprev = __DECONST(buflet_t, bprev0);
1125 #endif /* !KERNEL */
1126 void *bcur = NULL;
1127
1128 switch (SK_PTR_TYPE(ph)) {
1129 case NEXUS_META_TYPE_PACKET: {
1130 uint32_t bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1131 #ifdef KERNEL
1132 ASSERT(bcnt != 0 ||
1133 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1134 #else /* !KERNEL */
1135 /*
1136 * Handle the case where the metadata region gets
1137 * redirected to anonymous zero-filled pages at
1138 * defunct time. There's always 1 buflet in the
1139 * packet metadata, so pretend that's the count.
1140 */
1141 if (__improbable(bcnt == 0)) {
1142 bcnt = 1;
1143 bprev = NULL;
1144 }
1145 #endif /* !KERNEL */
1146 PKT_GET_NEXT_BUFLET(PKT_ADDR(ph), bcnt, BLT_ADDR(bprev), bcur);
1147 break;
1148 }
1149 case NEXUS_META_TYPE_QUANTUM:
1150 QUM_GET_NEXT_BUFLET(QUM_ADDR(ph), BLT_ADDR(bprev), bcur);
1151 break;
1152 default:
1153 #ifdef KERNEL
1154 VERIFY(0);
1155 /* NOTREACHED */
1156 __builtin_unreachable();
1157 #endif /* KERNEL */
1158 break;
1159 }
1160 return bcur;
1161 }
1162
1163 __attribute__((always_inline))
1164 static inline uint8_t
__packet_get_segment_count(const uint64_t ph)1165 __packet_get_segment_count(const uint64_t ph)
1166 {
1167 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1168 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1169
1170 return PKT_ADDR(ph)->pkt_seg_cnt;
1171 }
1172
1173 __attribute__((always_inline))
1174 static inline void
__packet_set_segment_count(const uint64_t ph,uint8_t segcount)1175 __packet_set_segment_count(const uint64_t ph, uint8_t segcount)
1176 {
1177 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1178 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1179
1180 PKT_ADDR(ph)->pkt_seg_cnt = segcount;
1181 }
1182
1183 __attribute__((always_inline))
1184 static inline uint16_t
__packet_get_protocol_segment_size(const uint64_t ph)1185 __packet_get_protocol_segment_size(const uint64_t ph)
1186 {
1187 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1188 return PKT_ADDR(ph)->pkt_proto_seg_sz;
1189 }
1190
1191 __attribute__((always_inline))
1192 static inline errno_t
__packet_set_protocol_segment_size(const uint64_t ph,uint16_t proto_seg_sz)1193 __packet_set_protocol_segment_size(const uint64_t ph, uint16_t proto_seg_sz)
1194 {
1195 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1196 PKT_ADDR(ph)->pkt_proto_seg_sz = proto_seg_sz;
1197 return 0;
1198 }
1199
1200 __attribute__((always_inline))
1201 static inline void
__packet_get_tso_flags(const uint64_t ph,packet_tso_flags_t * flags)1202 __packet_get_tso_flags(const uint64_t ph, packet_tso_flags_t *flags)
1203 {
1204 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_proto_seg_sz == sizeof(uint16_t)));
1205
1206 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1207 *flags = PKT_ADDR(ph)->pkt_csum_flags & (PACKET_CSUM_TSO_FLAGS);
1208 }
1209
1210 __attribute__((always_inline))
1211 static inline void
__packet_set_tso_flags(const uint64_t ph,packet_tso_flags_t flags)1212 __packet_set_tso_flags(const uint64_t ph, packet_tso_flags_t flags)
1213 {
1214 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1215
1216 PKT_ADDR(ph)->pkt_csum_flags |= flags & (PACKET_CSUM_TSO_FLAGS);
1217 }
1218
1219 __attribute__((always_inline))
1220 static inline uint32_t
__buflet_get_data_limit(const void * buf)1221 __buflet_get_data_limit(const void *buf)
1222 {
1223 return BLT_ADDR(buf)->buf_dlim;
1224 }
1225
1226 #ifdef KERNEL
1227 __attribute__((always_inline))
1228 static inline errno_t
__buflet_set_data_limit(const void * buf,const uint32_t dlim)1229 __buflet_set_data_limit(const void *buf, const uint32_t dlim)
1230 {
1231 /* buffer region is always marked as shareable */
1232 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1233
1234 /* full bounds checking will be performed during finalize */
1235 if (__probable((uint32_t)dlim <= BLT_ADDR(buf)->buf_objlim)) {
1236 _CASSERT(sizeof(BLT_ADDR(buf)->buf_dlim) == sizeof(uint32_t));
1237 /* deconst */
1238 *(uint32_t *)(uintptr_t)&BLT_ADDR(buf)->buf_dlim = dlim;
1239 return 0;
1240 }
1241 return ERANGE;
1242 }
1243 #endif /* KERNEL */
1244
1245 __attribute__((always_inline))
1246 static inline uint32_t
__buflet_get_data_offset(const void * buf)1247 __buflet_get_data_offset(const void *buf)
1248 {
1249 return BLT_ADDR(buf)->buf_doff;
1250 }
1251
1252 /*
1253 * ******************************************************************
1254 * Checks in __packet_finalize for packet finalized from userland
1255 * ******************************************************************
1256 * +-------+---------------------------+---------------------------+
1257 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1258 * |-------+---------------------------+---------------------------+
1259 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1260 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1261 * |l3_off | l3 = l2 | l3 == 0 |
1262 * |l4_off | l4 = l3 = l2 | l4 = l3 = 0 |
1263 * +-------+---------------------------+---------------------------+
1264 *
1265 * ******************************************************************
1266 * Checks in __packet_finalize for packet finalized from kernel
1267 * ******************************************************************
1268 * +-------+---------------------------+---------------------------+
1269 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1270 * |-------+---------------------------+---------------------------+
1271 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1272 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1273 * |l3_off | l3 >= l2 && l3 <bdlim | l3 == 0 |
1274 * |l4_off | l4 = l3 | l4 = l3 = 0 |
1275 * +-------+---------------------------+---------------------------+
1276 *
1277 */
1278 __attribute__((always_inline))
1279 static inline int
__packet_finalize(const uint64_t ph)1280 __packet_finalize(const uint64_t ph)
1281 {
1282 void *bcur = NULL, *bprev = NULL;
1283 uint32_t len, bcnt, bdoff0, bdlim0;
1284 int err = 0;
1285
1286 #ifdef KERNEL
1287 ASSERT(QUM_ADDR(ph)->qum_qflags & QUM_F_INTERNALIZED);
1288 #endif /* KERNEL */
1289 QUM_ADDR(ph)->qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1290
1291 bcnt = __packet_get_buflet_count(ph);
1292 len = QUM_ADDR(ph)->qum_len = 0;
1293
1294 while (bcnt--) {
1295 bcur = __packet_get_next_buflet(ph, bprev);
1296
1297 #ifdef KERNEL
1298 ASSERT(bcur != NULL);
1299 ASSERT(BLT_ADDR(bcur)->buf_addr != 0);
1300 #else /* !KERNEL */
1301 if (__improbable(bcur == NULL)) {
1302 err = ERANGE;
1303 break;
1304 }
1305 #endif /* KERNEL */
1306
1307 /* save data offset from the first buflet */
1308 if (bprev == NULL) {
1309 bdoff0 = __buflet_get_data_offset(bcur);
1310 bdlim0 = __buflet_get_data_limit(bcur);
1311 }
1312
1313 #ifndef KERNEL
1314 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)))) {
1315 err = ERANGE;
1316 break;
1317 }
1318 #else /* !KERNEL */
1319 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)) &&
1320 !PKT_HAS_ATTACHED_MBUF(ph))) {
1321 err = ERANGE;
1322 break;
1323 }
1324 #endif /* KERNEL */
1325 len += BLT_ADDR(bcur)->buf_dlen;
1326 bprev = bcur;
1327 }
1328
1329 if (__improbable(err != 0)) {
1330 goto done;
1331 }
1332
1333 switch (SK_PTR_TYPE(ph)) {
1334 case NEXUS_META_TYPE_PACKET:
1335 if (__improbable(bdoff0 > UINT8_MAX)) {
1336 err = ERANGE;
1337 goto done;
1338 }
1339 /* internalize headroom value from offset */
1340 PKT_ADDR(ph)->pkt_headroom = (uint8_t)bdoff0;
1341 /* validate header offsets in packet */
1342 switch (SK_PTR_SUBTYPE(ph)) {
1343 case NEXUS_META_SUBTYPE_RAW:
1344 #ifndef KERNEL
1345 /* Overwrite L2 len for raw packets from user space */
1346 PKT_ADDR(ph)->pkt_l2_len = 0;
1347 #else /* !KERNEL */
1348 /* ensure that L3 >= L2 && L3 < bdlim */
1349 if (__improbable((PKT_ADDR(ph)->pkt_headroom +
1350 PKT_ADDR(ph)->pkt_l2_len) >= bdlim0)) {
1351 err = ERANGE;
1352 goto done;
1353 }
1354 #endif /* KERNEL */
1355 break;
1356 case NEXUS_META_SUBTYPE_PAYLOAD:
1357 /*
1358 * For payload packet there is no concept of headroom
1359 * and L3 offset should always be 0
1360 */
1361 if (__improbable((PKT_ADDR(ph)->pkt_headroom != 0) ||
1362 (PKT_ADDR(ph)->pkt_l2_len != 0))) {
1363 err = ERANGE;
1364 goto done;
1365 }
1366 break;
1367 default:
1368 #ifdef KERNEL
1369 VERIFY(0);
1370 /* NOTREACHED */
1371 __builtin_unreachable();
1372 #endif /* KERNEL */
1373 break;
1374 }
1375
1376 if (__improbable(PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_DATA)) {
1377 #ifdef KERNEL
1378 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
1379 #else /* !KERNEL */
1380 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
1381 #endif /* !KERNEL */
1382 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1383 po->__po_expire_ts == 0) {
1384 err = EINVAL;
1385 goto done;
1386 }
1387 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) &&
1388 po->__po_token_len == 0) {
1389 err = EINVAL;
1390 goto done;
1391 }
1392 ASSERT(err == 0);
1393 }
1394
1395 /*
1396 * NOTE: we don't need the validation for total packet length
1397 * as checking if each buflet is in range and that
1398 * (pkt_headroom == bdoff0), should cover this check.
1399 */
1400 break;
1401
1402 default:
1403 /* nothing to do currently for quantum */
1404 break;
1405 }
1406
1407 done:
1408 if (__probable(err == 0)) {
1409 QUM_ADDR(ph)->qum_len = len;
1410 QUM_ADDR(ph)->qum_qflags |= QUM_F_FINALIZED;
1411 } else {
1412 QUM_ADDR(ph)->qum_len = 0;
1413 QUM_ADDR(ph)->qum_qflags |= QUM_F_DROPPED;
1414 }
1415
1416 return err;
1417 }
1418
1419 __attribute__((always_inline))
1420 static inline boolean_t
__packet_is_finalized(const uint64_t ph)1421 __packet_is_finalized(const uint64_t ph)
1422 {
1423 return QUM_ADDR(ph)->qum_qflags & QUM_F_FINALIZED;
1424 }
1425
1426 #ifdef KERNEL
1427 /*
1428 * function to initialize a packet with mbuf chain.
1429 * Apart from the attached mbuf, the packet can also be used to convey
1430 * additional metadata like the headroom and L2 header length.
1431 * For a packet with attached mbuf, the pkt_length conveys the length of
1432 * the attached mbuf. If the data copied is partial then PKT_F_TRUNCATED is
1433 * also set.
1434 */
1435 __attribute__((always_inline))
1436 static inline int
__packet_initialize_with_mbufchain(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1437 __packet_initialize_with_mbufchain(struct __kern_packet *pkt, struct mbuf *mbuf,
1438 uint8_t headroom, uint8_t l2len)
1439 {
1440 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1441 VERIFY(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1442 VERIFY((pkt->pkt_pflags & PKT_F_MBUF_MASK) == 0);
1443 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
1444 VERIFY(pkt->pkt_mbuf == NULL);
1445
1446 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1447 pkt->pkt_mbuf = mbuf;
1448 pkt->pkt_pflags |= (PKT_F_MBUF_DATA | PKT_F_TRUNCATED);
1449 pkt->pkt_headroom = headroom;
1450 pkt->pkt_l2_len = l2len;
1451 pkt->pkt_length = m_pktlen(mbuf);
1452 pkt->pkt_qum_buf.buf_dlen = 0;
1453 pkt->pkt_qum_buf.buf_doff = 0;
1454 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1455 return 0;
1456 }
1457
1458 __attribute__((always_inline))
1459 static inline int
__packet_initialize_with_mbuf(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1460 __packet_initialize_with_mbuf(struct __kern_packet *pkt, struct mbuf *mbuf,
1461 uint8_t headroom, uint8_t l2len)
1462 {
1463 __packet_initialize_with_mbufchain(pkt, mbuf, headroom, l2len);
1464 VERIFY(mbuf->m_nextpkt == NULL);
1465 return 0;
1466 }
1467
1468 /*
1469 * function to finalize a packet with attached mbuf.
1470 */
1471 __attribute__((always_inline))
1472 static inline int
__packet_finalize_with_mbuf(struct __kern_packet * pkt)1473 __packet_finalize_with_mbuf(struct __kern_packet *pkt)
1474 {
1475 uint32_t bdlen, bdoff, bdlim;
1476 struct __kern_buflet *buf;
1477 int err = 0;
1478
1479 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1480 VERIFY((pkt->pkt_pflags & (PKT_F_MBUF_DATA | PKT_F_PKT_DATA)) ==
1481 PKT_F_MBUF_DATA);
1482 VERIFY(pkt->pkt_mbuf != NULL);
1483 ASSERT(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1484 VERIFY(pkt->pkt_bufs_cnt == 1);
1485 PKT_GET_FIRST_BUFLET(pkt, pkt->pkt_bufs_cnt, buf);
1486 ASSERT(buf->buf_addr != 0);
1487
1488 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1489 pkt->pkt_pflags &= ~PKT_F_TRUNCATED;
1490 bdlen = buf->buf_dlen;
1491 bdlim = buf->buf_dlim;
1492 bdoff = buf->buf_doff;
1493 if (__improbable(!BUF_IN_RANGE(buf))) {
1494 err = ERANGE;
1495 goto done;
1496 }
1497
1498 /* validate header offsets in packet */
1499 switch (METADATA_SUBTYPE(pkt)) {
1500 case NEXUS_META_SUBTYPE_RAW:
1501 if (__improbable((pkt->pkt_headroom != bdoff) ||
1502 (pkt->pkt_headroom >= bdlim))) {
1503 err = ERANGE;
1504 goto done;
1505 }
1506 if (__improbable((pkt->pkt_headroom +
1507 pkt->pkt_l2_len) >= bdlim)) {
1508 err = ERANGE;
1509 goto done;
1510 }
1511 break;
1512
1513 case NEXUS_META_SUBTYPE_PAYLOAD:
1514 /*
1515 * For payload packet there is no concept of headroom.
1516 */
1517 if (__improbable((pkt->pkt_headroom != 0) || (bdoff != 0) ||
1518 (pkt->pkt_l2_len != 0))) {
1519 err = ERANGE;
1520 goto done;
1521 }
1522 break;
1523
1524 default:
1525 VERIFY(0);
1526 /* NOTREACHED */
1527 __builtin_unreachable();
1528 break;
1529 }
1530
1531
1532 if (__improbable(pkt->pkt_pflags & PKT_F_OPT_DATA)) {
1533 struct __packet_opt *po = pkt->pkt_com_opt;
1534
1535 if ((pkt->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1536 po->__po_expire_ts == 0) {
1537 err = EINVAL;
1538 goto done;
1539 }
1540 if ((pkt->pkt_pflags & PKT_F_OPT_TOKEN) &&
1541 po->__po_token_len == 0) {
1542 err = EINVAL;
1543 goto done;
1544 }
1545 }
1546 ASSERT(err == 0);
1547
1548 done:
1549 if (__probable(err == 0)) {
1550 pkt->pkt_length = (uint32_t)m_pktlen(pkt->pkt_mbuf);
1551 if (bdlen < pkt->pkt_length) {
1552 pkt->pkt_pflags |= PKT_F_TRUNCATED;
1553 }
1554 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1555 } else {
1556 pkt->pkt_length = 0;
1557 pkt->pkt_qum.qum_qflags |= QUM_F_DROPPED;
1558 }
1559
1560 return err;
1561 }
1562
1563 __attribute__((always_inline))
1564 static inline uint32_t
__packet_get_object_index(const uint64_t ph)1565 __packet_get_object_index(const uint64_t ph)
1566 {
1567 return METADATA_IDX(QUM_ADDR(ph));
1568 }
1569
1570 __attribute__((always_inline))
1571 static inline errno_t
__packet_get_timestamp(const uint64_t ph,uint64_t * ts,boolean_t * valid)1572 __packet_get_timestamp(const uint64_t ph, uint64_t *ts, boolean_t *valid)
1573 {
1574 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1575
1576 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TS_VALID) != 0) {
1577 if (valid != NULL) {
1578 *valid = TRUE;
1579 }
1580 *ts = PKT_ADDR(ph)->pkt_timestamp;
1581 } else {
1582 if (valid != NULL) {
1583 *valid = FALSE;
1584 }
1585 *ts = 0;
1586 }
1587
1588 return 0;
1589 }
1590
1591 __attribute__((always_inline))
1592 static inline errno_t
__packet_set_timestamp(const uint64_t ph,uint64_t ts,boolean_t valid)1593 __packet_set_timestamp(const uint64_t ph, uint64_t ts, boolean_t valid)
1594 {
1595 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1596
1597 if (valid) {
1598 PKT_ADDR(ph)->pkt_timestamp = ts;
1599 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TS_VALID;
1600 } else {
1601 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_TS_VALID;
1602 PKT_ADDR(ph)->pkt_timestamp = 0;
1603 }
1604
1605 return 0;
1606 }
1607
1608 __attribute__((always_inline))
1609 static inline errno_t
__packet_get_tx_completion_data(const uint64_t ph,uintptr_t * cb_arg,uintptr_t * cb_data)1610 __packet_get_tx_completion_data(const uint64_t ph, uintptr_t *cb_arg,
1611 uintptr_t *cb_data)
1612 {
1613 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1614 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1615 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1616 *cb_arg = PKT_ADDR(ph)->pkt_tx_compl_cb_arg;
1617 *cb_data = PKT_ADDR(ph)->pkt_tx_compl_cb_data;
1618 } else {
1619 *cb_arg = 0;
1620 *cb_data = 0;
1621 }
1622 return 0;
1623 }
1624
1625 __attribute__((always_inline))
1626 static inline errno_t
__packet_set_tx_completion_data(const uint64_t ph,uintptr_t cb_arg,uintptr_t cb_data)1627 __packet_set_tx_completion_data(const uint64_t ph, uintptr_t cb_arg,
1628 uintptr_t cb_data)
1629 {
1630 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1631 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1632 PKT_ADDR(ph)->pkt_tx_compl_cb_arg = cb_arg;
1633 PKT_ADDR(ph)->pkt_tx_compl_cb_data = cb_data;
1634 return 0;
1635 }
1636
1637 __attribute__((always_inline))
1638 static inline errno_t
__packet_get_timestamp_requested(const uint64_t ph,boolean_t * requested)1639 __packet_get_timestamp_requested(const uint64_t ph, boolean_t *requested)
1640 {
1641 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1642 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0) {
1643 *requested = TRUE;
1644 } else {
1645 *requested = FALSE;
1646 }
1647 return 0;
1648 }
1649
1650 __attribute__((always_inline))
1651 static inline errno_t
__packet_get_tx_completion_status(const uint64_t ph,kern_return_t * status)1652 __packet_get_tx_completion_status(const uint64_t ph, kern_return_t *status)
1653 {
1654 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1655 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1656 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1657 *status = (kern_return_t)PKT_ADDR(ph)->pkt_tx_compl_status;
1658 } else {
1659 *status = 0;
1660 }
1661 return 0;
1662 }
1663
1664 __attribute__((always_inline))
1665 static inline errno_t
__packet_set_tx_completion_status(const uint64_t ph,kern_return_t status)1666 __packet_set_tx_completion_status(const uint64_t ph, kern_return_t status)
1667 {
1668 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1669 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1670 PKT_ADDR(ph)->pkt_tx_compl_status = (uint32_t)status;
1671 return 0;
1672 }
1673
1674 __attribute__((always_inline))
1675 static inline errno_t
__packet_set_tx_nx_port(const uint64_t ph,nexus_port_t nx_port,uint16_t vpna_gencnt)1676 __packet_set_tx_nx_port(const uint64_t ph, nexus_port_t nx_port,
1677 uint16_t vpna_gencnt)
1678 {
1679 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1680 PKT_ADDR(ph)->pkt_nx_port = nx_port;
1681 PKT_ADDR(ph)->pkt_vpna_gencnt = vpna_gencnt;
1682 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TX_PORT_DATA;
1683 return 0;
1684 }
1685
1686 __attribute__((always_inline))
1687 static inline errno_t
__packet_get_tx_nx_port(const uint64_t ph,nexus_port_t * nx_port,uint16_t * vpna_gencnt)1688 __packet_get_tx_nx_port(const uint64_t ph, nexus_port_t *nx_port,
1689 uint16_t *vpna_gencnt)
1690 {
1691 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1692 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_PORT_DATA) == 0) {
1693 return ENOTSUP;
1694 }
1695
1696 *nx_port = PKT_ADDR(ph)->pkt_nx_port;
1697 *vpna_gencnt = PKT_ADDR(ph)->pkt_vpna_gencnt;
1698 return 0;
1699 }
1700
1701 __attribute__((always_inline))
1702 static inline errno_t
__packet_get_tx_nx_port_id(const uint64_t ph,uint32_t * nx_port_id)1703 __packet_get_tx_nx_port_id(const uint64_t ph, uint32_t *nx_port_id)
1704 {
1705 errno_t err;
1706 nexus_port_t nx_port;
1707 uint16_t vpna_gencnt;
1708
1709 _CASSERT(sizeof(nx_port) == sizeof(uint16_t));
1710
1711 err = __packet_get_tx_nx_port(ph, &nx_port, &vpna_gencnt);
1712 if (err == 0) {
1713 *nx_port_id = PKT_COMPOSE_NX_PORT_ID(nx_port, vpna_gencnt);
1714 }
1715 return err;
1716 }
1717
1718
1719 __attribute__((always_inline))
1720 static inline errno_t
__packet_get_flowid(const uint64_t ph,packet_flowid_t * pflowid)1721 __packet_get_flowid(const uint64_t ph, packet_flowid_t *pflowid)
1722 {
1723 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1724 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_FLOW_ID) == 0) {
1725 return ENOENT;
1726 }
1727 *pflowid = PKT_ADDR(ph)->pkt_flow_token;
1728 return 0;
1729 }
1730 #endif /* KERNEL */
1731
1732 extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t);
1733
1734 __attribute__((always_inline))
1735 static inline uint16_t
__packet_fold_sum(uint32_t sum)1736 __packet_fold_sum(uint32_t sum)
1737 {
1738 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1739 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1740 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1741 return sum & 0xffff;
1742 }
1743
1744 __attribute__((always_inline))
1745 static inline uint16_t
__packet_fold_sum_final(uint32_t sum)1746 __packet_fold_sum_final(uint32_t sum)
1747 {
1748 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1749 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1750 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1751 return ~sum & 0xffff;
1752 }
1753
1754 __attribute__((always_inline))
1755 static inline uint32_t
__packet_cksum(const void * data,uint32_t len,uint32_t sum0)1756 __packet_cksum(const void *data, uint32_t len, uint32_t sum0)
1757 {
1758 return os_cpu_in_cksum(data, len, sum0);
1759 }
1760
1761 extern uint32_t os_cpu_copy_in_cksum(const void *, void *, uint32_t, uint32_t);
1762
1763 __attribute__((always_inline))
1764 static inline uint32_t
__packet_copy_and_sum(const void * src,void * dst,uint32_t len,uint32_t sum0)1765 __packet_copy_and_sum(const void *src, void *dst, uint32_t len, uint32_t sum0)
1766 {
1767 return os_cpu_copy_in_cksum(src, dst, len, sum0);
1768 }
1769
1770 __attribute__((always_inline))
1771 static inline uint16_t
__packet_fix_sum(uint16_t csum,uint16_t old,uint16_t new)1772 __packet_fix_sum(uint16_t csum, uint16_t old, uint16_t new)
1773 {
1774 uint32_t c = csum + old - new;
1775 c = (c >> 16) + (c & 0xffff); /* Only add carry once */
1776
1777 return c & 0xffff;
1778 }
1779
1780 /* MUST be used for uint32_t fields */
1781 __attribute__((always_inline))
1782 static inline void
__packet_fix_hdr_sum(uint8_t * field,uint16_t * csum,uint32_t new)1783 __packet_fix_hdr_sum(uint8_t *field, uint16_t *csum, uint32_t new)
1784 {
1785 uint32_t old;
1786 memcpy(&old, field, sizeof(old));
1787 memcpy(field, &new, sizeof(uint32_t));
1788 *csum = __packet_fix_sum(__packet_fix_sum(*csum, (uint16_t)(old >> 16),
1789 (uint16_t)(new >> 16)), (uint16_t)(old & 0xffff),
1790 (uint16_t)(new & 0xffff));
1791 }
1792
1793 __attribute__((always_inline))
1794 static inline void *
__buflet_get_data_address(const void * buf)1795 __buflet_get_data_address(const void *buf)
1796 {
1797 return (void *)(BLT_ADDR(buf)->buf_addr);
1798 }
1799
1800 #ifdef KERNEL
1801 __attribute__((always_inline))
1802 static inline errno_t
__buflet_set_data_address(const void * buf,const void * addr)1803 __buflet_set_data_address(const void *buf, const void *addr)
1804 {
1805 /* buffer region is always marked as shareable */
1806 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1807
1808 /* full bounds checking will be performed during finalize */
1809 if (__probable((uintptr_t)addr >=
1810 (uintptr_t)BLT_ADDR(buf)->buf_objaddr)) {
1811 _CASSERT(sizeof(BLT_ADDR(buf)->buf_addr) ==
1812 sizeof(mach_vm_address_t));
1813 /* deconst */
1814 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1815 (mach_vm_address_t)addr;
1816 return 0;
1817 }
1818 return ERANGE;
1819 }
1820
1821 /*
1822 * Equivalent to __buflet_set_data_address but based on offset, packets/buflets
1823 * set with this should not be directly passed to userspace, since shared buffer
1824 * is not yet supported by user facing pool.
1825 */
1826 __attribute__((always_inline))
1827 static inline int
__buflet_set_buffer_offset(const void * buf,const uint32_t off)1828 __buflet_set_buffer_offset(const void *buf, const uint32_t off)
1829 {
1830 ASSERT(BLT_ADDR(buf)->buf_objlim != 0);
1831
1832 if (__probable(off <= BLT_ADDR(buf)->buf_objlim)) {
1833 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1834 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr + off;
1835 return 0;
1836 }
1837 return ERANGE;
1838 }
1839 #endif /* KERNEL */
1840
1841 __attribute__((always_inline))
1842 static inline int
__buflet_set_data_offset(const void * buf,const uint32_t doff)1843 __buflet_set_data_offset(const void *buf, const uint32_t doff)
1844 {
1845 #ifdef KERNEL
1846 /*
1847 * Kernel-specific assertion. For user space, the metadata
1848 * region gets redirected to anonymous zero-filled pages at
1849 * defunct time, so ignore it there.
1850 */
1851 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1852
1853 if (__probable((uint32_t)doff <= BLT_ADDR(buf)->buf_objlim)) {
1854 BLT_ADDR(buf)->buf_doff = doff;
1855 return 0;
1856 }
1857 return ERANGE;
1858 #else /* !KERNEL */
1859 BLT_ADDR(buf)->buf_doff = doff;
1860 return 0;
1861 #endif /* KERNEL */
1862 }
1863
1864 __attribute__((always_inline))
1865 static inline int
__buflet_set_data_length(const void * buf,const uint32_t dlen)1866 __buflet_set_data_length(const void *buf, const uint32_t dlen)
1867 {
1868 #ifdef KERNEL
1869 /*
1870 * Kernel-specific assertion. For user space, the metadata
1871 * region gets redirected to anonymous zero-filled pages at
1872 * defunct time, so ignore it there.
1873 */
1874 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1875
1876 if (__probable((uint32_t)dlen <= BLT_ADDR(buf)->buf_objlim)) {
1877 BLT_ADDR(buf)->buf_dlen = dlen;
1878 return 0;
1879 }
1880 return ERANGE;
1881 #else /* !KERNEL */
1882 BLT_ADDR(buf)->buf_dlen = dlen;
1883 return 0;
1884 #endif /* KERNEL */
1885 }
1886
1887 __attribute__((always_inline))
1888 static inline uint32_t
__buflet_get_data_length(const void * buf)1889 __buflet_get_data_length(const void *buf)
1890 {
1891 return BLT_ADDR(buf)->buf_dlen;
1892 }
1893
1894 #ifdef KERNEL
1895 __attribute__((always_inline))
1896 static inline struct sksegment *
__buflet_get_object_segment(const void * buf,kern_obj_idx_seg_t * idx)1897 __buflet_get_object_segment(const void *buf, kern_obj_idx_seg_t *idx)
1898 {
1899 _CASSERT(sizeof(obj_idx_t) == sizeof(kern_obj_idx_seg_t));
1900
1901 if (idx != NULL) {
1902 *idx = BLT_ADDR(buf)->buf_ctl->bc_idx;
1903 }
1904
1905 return BLT_ADDR(buf)->buf_ctl->bc_slab->sl_seg;
1906 }
1907 #endif /* KERNEL */
1908
1909 __attribute__((always_inline))
1910 static inline void *
__buflet_get_object_address(const void * buf)1911 __buflet_get_object_address(const void *buf)
1912 {
1913 #ifdef KERNEL
1914 return (void *)(BLT_ADDR(buf)->buf_objaddr);
1915 #else /* !KERNEL */
1916 /*
1917 * For user space, shared buffer is not available and hence the data
1918 * address is immutable and is always the same as the underlying
1919 * buffer object address itself.
1920 */
1921 return __buflet_get_data_address(buf);
1922 #endif /* !KERNEL */
1923 }
1924
1925 __attribute__((always_inline))
1926 static inline uint32_t
__buflet_get_object_limit(const void * buf)1927 __buflet_get_object_limit(const void *buf)
1928 {
1929 #ifdef KERNEL
1930 return BLT_ADDR(buf)->buf_objlim;
1931 #else /* !KERNEL */
1932 /*
1933 * For user space, shared buffer is not available and hence the data
1934 * limit is immutable and is always the same as the underlying buffer
1935 * object limit itself.
1936 */
1937 return (uint32_t)__buflet_get_data_limit(buf);
1938 #endif /* !KERNEL */
1939 }
1940
1941 __attribute__((always_inline))
1942 static inline packet_trace_id_t
__packet_get_trace_id(const uint64_t ph)1943 __packet_get_trace_id(const uint64_t ph)
1944 {
1945 switch (SK_PTR_TYPE(ph)) {
1946 case NEXUS_META_TYPE_PACKET:
1947 return PKT_ADDR(ph)->pkt_trace_id;
1948 break;
1949 default:
1950 return 0;
1951 }
1952 }
1953
1954 __attribute__((always_inline))
1955 static inline void
__packet_set_trace_id(const uint64_t ph,packet_trace_id_t id)1956 __packet_set_trace_id(const uint64_t ph, packet_trace_id_t id)
1957 {
1958 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1959 PKT_ADDR(ph)->pkt_trace_id = id;
1960 }
1961
1962 __attribute__((always_inline))
1963 static inline void
__packet_trace_event(const uint64_t ph,uint32_t event)1964 __packet_trace_event(const uint64_t ph, uint32_t event)
1965 {
1966 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1967 #ifdef KERNEL
1968 #pragma unused(event, ph)
1969 KDBG(event, PKT_ADDR(ph)->pkt_trace_id);
1970 #else /* !KERNEL */
1971 kdebug_trace(event, PKT_ADDR(ph)->pkt_trace_id, 0, 0, 0);
1972 #endif /* !KERNEL */
1973 }
1974
1975 #ifdef KERNEL
1976 __attribute__((always_inline))
1977 static inline packet_trace_tag_t
__packet_get_trace_tag(const uint64_t ph)1978 __packet_get_trace_tag(const uint64_t ph)
1979 {
1980 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1981 return PKT_ADDR(ph)->pkt_trace_tag;
1982 }
1983
1984 __attribute__((always_inline))
1985 static inline void
__packet_set_trace_tag(const uint64_t ph,packet_trace_tag_t tag)1986 __packet_set_trace_tag(const uint64_t ph, packet_trace_tag_t tag)
1987 {
1988 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1989 PKT_ADDR(ph)->pkt_trace_tag = tag;
1990 }
1991
1992 static inline void
__packet_perform_tx_completion_callbacks(const kern_packet_t ph,ifnet_t ifp)1993 __packet_perform_tx_completion_callbacks(const kern_packet_t ph, ifnet_t ifp)
1994 {
1995 /*
1996 * NOTE: this function can be called with ifp as NULL.
1997 */
1998 uint64_t ts;
1999 kern_return_t tx_status;
2000 uintptr_t cb_arg, cb_data;
2001 struct __kern_packet *kpkt = SK_PTR_ADDR_KPKT(ph);
2002
2003 ASSERT((kpkt->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0);
2004 (void) __packet_get_tx_completion_status(ph, &tx_status);
2005 __packet_get_tx_completion_data(ph, &cb_arg, &cb_data);
2006 __packet_get_timestamp(ph, &ts, NULL);
2007 while (kpkt->pkt_tx_compl_callbacks != 0) {
2008 mbuf_tx_compl_func cb;
2009 uint32_t i;
2010
2011 i = ffs(kpkt->pkt_tx_compl_callbacks) - 1;
2012 kpkt->pkt_tx_compl_callbacks &= ~(1 << i);
2013 cb = m_get_tx_compl_callback(i);
2014 if (__probable(cb != NULL)) {
2015 cb(kpkt->pkt_tx_compl_context, ifp, ts, cb_arg, cb_data,
2016 tx_status);
2017 }
2018 }
2019 kpkt->pkt_pflags &= ~PKT_F_TX_COMPL_TS_REQ;
2020 }
2021
2022 static inline void *
__packet_get_priv(const kern_packet_t ph)2023 __packet_get_priv(const kern_packet_t ph)
2024 {
2025 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2026 return PKT_ADDR(ph)->pkt_priv;
2027 }
2028
2029 static inline void
__packet_set_priv(const uint64_t ph,void * priv)2030 __packet_set_priv(const uint64_t ph, void *priv)
2031 {
2032 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2033 PKT_ADDR(ph)->pkt_priv = priv;
2034 }
2035 #endif /* KERNEL */
2036
2037 #endif /* PRIVATE || BSD_KERNEL_PRIVATE */
2038 #endif /* !_SKYWALK_PACKET_COMMON_H_ */
2039