1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_PACKET_COMMON_H_
30 #define _SKYWALK_PACKET_COMMON_H_
31
32 #if defined(PRIVATE) || defined(BSD_KERNEL_PRIVATE)
33 /*
34 * Routines common to kernel and userland. This file is intended to
35 * be included by code implementing the packet APIs, in particular,
36 * the Skywalk kernel and libsyscall code.
37 */
38
39 #include <skywalk/os_packet_private.h>
40 #include <net/if_vlan_var.h>
41 #include <sys/errno.h>
42 #include <sys/kdebug.h>
43
44 #ifndef KERNEL
45 /*
46 * User.
47 */
48 #if !defined(LIBSYSCALL_INTERFACE)
49 #error "LIBSYSCALL_INTERFACE not defined"
50 #endif /* !LIBSYSCALL_INTERFACE */
51 #define QUM_ADDR(_ph) SK_PTR_ADDR_UQUM(_ph)
52 #define PKT_ADDR(_ph) SK_PTR_ADDR_UPKT(_ph)
53 #define BLT_ADDR(_bp) ((struct __user_buflet *)(uintptr_t)_bp)
54 #else /* KERNEL */
55 /*
56 * Kernel.
57 */
58 #include <skywalk/packet/packet_var.h>
59 #include <skywalk/packet/pbufpool_var.h>
60 #define QUM_ADDR(_ph) SK_PTR_ADDR_KQUM(_ph)
61 #define PKT_ADDR(_ph) SK_PTR_ADDR_KPKT(_ph)
62 #define BLT_ADDR(_bp) ((struct __kern_buflet *)(uintptr_t)_bp)
63 #define PKT_HAS_ATTACHED_MBUF(_ph) \
64 ((PKT_ADDR(_ph)->pkt_pflags & PKT_F_MBUF_DATA) != 0)
65 #endif /* KERNEL */
66
67 /*
68 * Common.
69 */
70 #if (DEBUG || DEVELOPMENT)
71 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) do { \
72 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type) || \
73 SK_PTR_SUBTYPE(_ph) != (uint64_t)(_subtype))) { \
74 pkt_subtype_assert_fail(_ph, _type, _subtype); \
75 /* NOTREACHED */ \
76 __builtin_unreachable(); \
77 } \
78 } while (0)
79
80 #define PKT_TYPE_ASSERT(_ph, _type) do { \
81 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type))) { \
82 pkt_type_assert_fail(_ph, _type); \
83 /* NOTREACHED */ \
84 __builtin_unreachable(); \
85 } \
86 } while (0)
87 #else /* !DEBUG && !DEVELOPMENT */
88 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) ((void)0)
89 #define PKT_TYPE_ASSERT(_ph, _type) ((void)0)
90 #endif /* !DEBUG && !DEVELOPMENT */
91
92 #define QUM_GET_NEXT_BUFLET(_qum, _pbuf, _buf) do { \
93 ASSERT((_pbuf) == NULL || (_pbuf) == (_qum)->qum_buf); \
94 (_buf) = (((_pbuf) == NULL) ? (_qum)->qum_buf : NULL); \
95 } while (0)
96
97 #define PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf) do { \
98 if (__improbable((_bcnt) == 0)) { \
99 (_buf) = NULL; \
100 break; \
101 } \
102 if (__probable((_pkt)->pkt_qum_buf.buf_addr != 0)) { \
103 (_buf) = &(_pkt)->pkt_qum_buf; \
104 } else { \
105 (_buf) = __DECONST(void *, (_pkt)->pkt_qum_buf.buf_nbft_addr);\
106 } \
107 } while (0)
108
109 #define _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
110 if ((_pbuf) == NULL) { \
111 PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf); \
112 } else { \
113 (_buf) = __DECONST(void *, (_pbuf)->buf_nbft_addr); \
114 } \
115 } while (0)
116
117 #ifndef KERNEL
118 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
119 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
120 } while (0)
121 #else /* KERNEL */
122 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
123 ASSERT(((_bcnt) >= 1) || ((_pbuf) == NULL)); \
124 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
125 } while (0)
126 #endif /* KERNEL */
127
128 #ifdef KERNEL
129 #define PKT_COMPOSE_NX_PORT_ID(_nx_port, _gencnt) \
130 ((uint32_t)((_gencnt & 0xffff) << 16) | (_nx_port & 0xffff))
131
132 #define PKT_DECOMPOSE_NX_PORT_ID(_nx_port_id, _nx_port, _gencnt) do { \
133 _nx_port = _nx_port_id & 0xffff; \
134 _gencnt = (_nx_port_id >> 16) & 0xffff; \
135 } while (0)
136 #endif /* KERNEL */
137
138 __attribute__((always_inline))
139 static inline int
__packet_set_headroom(const uint64_t ph,const uint8_t headroom)140 __packet_set_headroom(const uint64_t ph, const uint8_t headroom)
141 {
142 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
143 if (__probable(headroom < PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
144 PKT_ADDR(ph)->pkt_headroom = headroom;
145 return 0;
146 }
147 return ERANGE;
148 }
149
150 __attribute__((always_inline))
151 static inline uint8_t
__packet_get_headroom(const uint64_t ph)152 __packet_get_headroom(const uint64_t ph)
153 {
154 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
155 return PKT_ADDR(ph)->pkt_headroom;
156 }
157
158 __attribute__((always_inline))
159 static inline int
__packet_set_link_header_length(const uint64_t ph,const uint8_t len)160 __packet_set_link_header_length(const uint64_t ph, const uint8_t len)
161 {
162 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
163 if (__probable(len <= PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
164 PKT_ADDR(ph)->pkt_l2_len = len;
165 return 0;
166 }
167 return ERANGE;
168 }
169
170 __attribute__((always_inline))
171 static inline uint8_t
__packet_get_link_header_length(const uint64_t ph)172 __packet_get_link_header_length(const uint64_t ph)
173 {
174 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
175 return PKT_ADDR(ph)->pkt_l2_len;
176 }
177
178 __attribute__((always_inline))
179 static inline int
__packet_set_link_broadcast(const uint64_t ph)180 __packet_set_link_broadcast(const uint64_t ph)
181 {
182 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
183 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_BCAST;
184 return 0;
185 }
186
187 __attribute__((always_inline))
188 static inline boolean_t
__packet_get_link_broadcast(const uint64_t ph)189 __packet_get_link_broadcast(const uint64_t ph)
190 {
191 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
192 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_BCAST) != 0;
193 }
194
195 __attribute__((always_inline))
196 static inline int
__packet_set_link_multicast(const uint64_t ph)197 __packet_set_link_multicast(const uint64_t ph)
198 {
199 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
200 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_MCAST;
201 return 0;
202 }
203
204 __attribute__((always_inline))
205 static inline boolean_t
__packet_get_link_multicast(const uint64_t ph)206 __packet_get_link_multicast(const uint64_t ph)
207 {
208 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
209 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_MCAST) != 0;
210 }
211
212 __attribute__((always_inline))
213 static inline int
__packet_set_link_ethfcs(const uint64_t ph)214 __packet_set_link_ethfcs(const uint64_t ph)
215 {
216 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
217 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_ETHFCS;
218 return 0;
219 }
220
221 __attribute__((always_inline))
222 static inline boolean_t
__packet_get_link_ethfcs(const uint64_t ph)223 __packet_get_link_ethfcs(const uint64_t ph)
224 {
225 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
226 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_ETHFCS) != 0;
227 }
228
229 __attribute__((always_inline))
230 static inline int
__packet_set_transport_traffic_background(const uint64_t ph)231 __packet_set_transport_traffic_background(const uint64_t ph)
232 {
233 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
234 PKT_ADDR(ph)->pkt_pflags |= PKT_F_BACKGROUND;
235 return 0;
236 }
237
238 __attribute__((always_inline))
239 static inline boolean_t
__packet_get_transport_traffic_background(const uint64_t ph)240 __packet_get_transport_traffic_background(const uint64_t ph)
241 {
242 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
243 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_BACKGROUND) != 0;
244 }
245
246 __attribute__((always_inline))
247 static inline int
__packet_set_transport_traffic_realtime(const uint64_t ph)248 __packet_set_transport_traffic_realtime(const uint64_t ph)
249 {
250 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
251 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REALTIME;
252 return 0;
253 }
254
255 __attribute__((always_inline))
256 static inline boolean_t
__packet_get_transport_traffic_realtime(const uint64_t ph)257 __packet_get_transport_traffic_realtime(const uint64_t ph)
258 {
259 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
260 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REALTIME) != 0;
261 }
262
263 __attribute__((always_inline))
264 static inline int
__packet_set_transport_retransmit(const uint64_t ph)265 __packet_set_transport_retransmit(const uint64_t ph)
266 {
267 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
268 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REXMT;
269 return 0;
270 }
271
272 __attribute__((always_inline))
273 static inline boolean_t
__packet_get_transport_retransmit(const uint64_t ph)274 __packet_get_transport_retransmit(const uint64_t ph)
275 {
276 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
277 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REXMT) != 0;
278 }
279
280 __attribute__((always_inline))
281 static inline int
__packet_set_transport_last_packet(const uint64_t ph)282 __packet_set_transport_last_packet(const uint64_t ph)
283 {
284 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
285 PKT_ADDR(ph)->pkt_pflags |= PKT_F_LAST_PKT;
286 return 0;
287 }
288
289 __attribute__((always_inline))
290 static inline int
__packet_set_group_start(const uint64_t ph)291 __packet_set_group_start(const uint64_t ph)
292 {
293 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
294 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_START;
295 return 0;
296 }
297
298 __attribute__((always_inline))
299 static inline boolean_t
__packet_get_group_start(const uint64_t ph)300 __packet_get_group_start(const uint64_t ph)
301 {
302 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
303 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_START) != 0;
304 }
305
306 __attribute__((always_inline))
307 static inline int
__packet_set_group_end(const uint64_t ph)308 __packet_set_group_end(const uint64_t ph)
309 {
310 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
311 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_END;
312 return 0;
313 }
314
315 __attribute__((always_inline))
316 static inline boolean_t
__packet_get_group_end(const uint64_t ph)317 __packet_get_group_end(const uint64_t ph)
318 {
319 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
320 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_END) != 0;
321 }
322
323 __attribute__((always_inline))
324 static inline errno_t
__packet_get_expire_time(const uint64_t ph,uint64_t * ts)325 __packet_get_expire_time(const uint64_t ph, uint64_t *ts)
326 {
327 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
328 #ifdef KERNEL
329 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
330 #else /* !KERNEL */
331 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
332 #endif /* !KERNEL */
333 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) == 0) {
334 return ENOENT;
335 }
336 if (ts == NULL) {
337 return EINVAL;
338 }
339 *ts = po->__po_expire_ts;
340 return 0;
341 }
342
343 __attribute__((always_inline))
344 static inline errno_t
__packet_set_expire_time(const uint64_t ph,const uint64_t ts)345 __packet_set_expire_time(const uint64_t ph, const uint64_t ts)
346 {
347 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
348 #ifdef KERNEL
349 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
350 #else /* !KERNEL */
351 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
352 #endif /* !KERNEL */
353 if (ts != 0) {
354 po->__po_expire_ts = ts;
355 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXPIRE_TS;
356 } else {
357 po->__po_expire_ts = 0;
358 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXPIRE_TS;
359 }
360 return 0;
361 }
362
363 __attribute__((always_inline))
364 static inline errno_t
__packet_get_expiry_action(const uint64_t ph,packet_expiry_action_t * pea)365 __packet_get_expiry_action(const uint64_t ph, packet_expiry_action_t *pea)
366 {
367 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
368 #ifdef KERNEL
369 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
370 #else /* !KERNEL */
371 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
372 #endif /* !KERNEL */
373 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXP_ACTION) == 0) {
374 return ENOENT;
375 }
376 if (pea == NULL) {
377 return EINVAL;
378 }
379 *pea = po->__po_expiry_action;
380 return 0;
381 }
382
383 __attribute__((always_inline))
384 static inline errno_t
__packet_set_expiry_action(const uint64_t ph,packet_expiry_action_t pea)385 __packet_set_expiry_action(const uint64_t ph, packet_expiry_action_t pea)
386 {
387 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
388 #ifdef KERNEL
389 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
390 #else /* !KERNEL */
391 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
392 #endif /* !KERNEL */
393 if (pea != PACKET_EXPIRY_ACTION_NONE) {
394 po->__po_expiry_action = (uint8_t)pea;
395 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXP_ACTION;
396 } else {
397 po->__po_expiry_action = 0;
398 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXP_ACTION;
399 }
400 return 0;
401 }
402
403 __attribute__((always_inline))
404 static inline errno_t
__packet_opt_get_token(const struct __packet_opt * po,void * token,uint16_t * len,uint8_t * type)405 __packet_opt_get_token(const struct __packet_opt *po, void *token,
406 uint16_t *len, uint8_t *type)
407 {
408 uint16_t tlen = po->__po_token_len;
409 uint8_t ttype;
410
411 if (token == NULL || len == NULL || type == NULL || tlen > *len) {
412 return EINVAL;
413 }
414 ttype = (uint8_t)po->__po_token_type;
415
416 ASSERT(tlen <= PKT_OPT_MAX_TOKEN_SIZE);
417 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
418 bcopy(po->__po_token, token, tlen);
419 *len = tlen;
420 *type = ttype;
421 return 0;
422 }
423
424 __attribute__((always_inline))
425 static inline errno_t
__packet_get_token(const uint64_t ph,void * token,uint16_t * len)426 __packet_get_token(const uint64_t ph, void *token, uint16_t *len)
427 {
428 #ifdef KERNEL
429 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
430 #else /* !KERNEL */
431 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
432 #endif /* !KERNEL */
433 uint8_t type;
434 errno_t err;
435
436 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
437 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
438 return ENOENT;
439 }
440 err = __packet_opt_get_token(po, token, len, &type);
441 if ((err == 0) && (type != PKT_OPT_TOKEN_TYPE_OPAQUE)) {
442 err = ENOENT;
443 }
444 return err;
445 }
446
447 __attribute__((always_inline))
448 static inline errno_t
__packet_opt_set_token(struct __packet_opt * po,const void * token,const uint16_t len,const uint8_t type,volatile uint64_t * pflags)449 __packet_opt_set_token(struct __packet_opt *po, const void *token,
450 const uint16_t len, const uint8_t type, volatile uint64_t *pflags)
451 {
452 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
453 if (len != 0) {
454 if (token == NULL || len > PKT_OPT_MAX_TOKEN_SIZE ||
455 type == 0) {
456 return EINVAL;
457 }
458 if (__probable(IS_P2ALIGNED(token, 8))) {
459 uint64_t *token64 = __DECONST(void *, token);
460 po->__po_token_data[0] = *token64;
461 po->__po_token_data[1] = *(token64 + 1);
462 } else {
463 bcopy(token, po->__po_token, len);
464 }
465 po->__po_token_len = len;
466 po->__po_token_type = type;
467 *pflags |= PKT_F_OPT_TOKEN;
468 } else {
469 _CASSERT(sizeof(po->__po_token_data[0]) == 8);
470 _CASSERT(sizeof(po->__po_token_data[1]) == 8);
471 _CASSERT(sizeof(po->__po_token) == 16);
472 po->__po_token_data[0] = 0;
473 po->__po_token_data[1] = 0;
474 po->__po_token_len = 0;
475 po->__po_token_type = 0;
476 *pflags &= ~PKT_F_OPT_TOKEN;
477 }
478 return 0;
479 }
480
481 __attribute__((always_inline))
482 static inline errno_t
__packet_set_token(const uint64_t ph,const void * token,const uint16_t len)483 __packet_set_token(const uint64_t ph, const void *token, const uint16_t len)
484 {
485 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
486 #ifdef KERNEL
487 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, token, len,
488 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
489 #else /* !KERNEL */
490 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, token, len,
491 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
492 #endif /* !KERNEL */
493 }
494
495 __attribute__((always_inline))
496 static inline errno_t
__packet_get_packetid(const uint64_t ph,packet_id_t * pktid)497 __packet_get_packetid(const uint64_t ph, packet_id_t *pktid)
498 {
499 #ifdef KERNEL
500 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
501 #else /* !KERNEL */
502 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
503 #endif /* !KERNEL */
504 uint16_t len = sizeof(packet_id_t);
505 uint8_t type;
506 errno_t err;
507
508 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
509 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
510 return ENOENT;
511 }
512 err = __packet_opt_get_token(po, pktid, &len, &type);
513 if ((err == 0) && ((type != PKT_OPT_TOKEN_TYPE_PACKET_ID) ||
514 (len != sizeof(packet_id_t)))) {
515 err = ENOENT;
516 }
517 return err;
518 }
519
520 __attribute__((always_inline))
521 static inline errno_t
__packet_set_packetid(const uint64_t ph,const packet_id_t * pktid)522 __packet_set_packetid(const uint64_t ph, const packet_id_t *pktid)
523 {
524 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
525 #ifdef KERNEL
526 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, pktid,
527 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
528 &PKT_ADDR(ph)->pkt_pflags);
529 #else /* !KERNEL */
530 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, pktid,
531 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
532 &PKT_ADDR(ph)->pkt_pflags);
533 #endif /* !KERNEL */
534 }
535
536 __attribute__((always_inline))
537 static inline errno_t
__packet_get_vlan_tag(const uint64_t ph,uint16_t * vlan_tag,boolean_t * tag_in_pkt)538 __packet_get_vlan_tag(const uint64_t ph, uint16_t *vlan_tag,
539 boolean_t *tag_in_pkt)
540 {
541 #ifdef KERNEL
542 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
543 #else /* !KERNEL */
544 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
545 #endif /* !KERNEL */
546 uint64_t pflags;
547
548 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
549 pflags = PKT_ADDR(ph)->pkt_pflags;
550 if ((pflags & PKT_F_OPT_VLTAG) == 0) {
551 return ENOENT;
552 }
553 if (vlan_tag != NULL) {
554 *vlan_tag = po->__po_vlan_tag;
555 }
556 if (tag_in_pkt != NULL) {
557 *tag_in_pkt = ((pflags & PKT_F_OPT_VLTAG_IN_PKT) != 0);
558 }
559 return 0;
560 }
561
562 __attribute__((always_inline))
563 static inline errno_t
__packet_set_vlan_tag(const uint64_t ph,const uint16_t vlan_tag,const boolean_t tag_in_pkt)564 __packet_set_vlan_tag(const uint64_t ph, const uint16_t vlan_tag,
565 const boolean_t tag_in_pkt)
566 {
567 #ifdef KERNEL
568 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
569 #else /* !KERNEL */
570 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
571 #endif /* !KERNEL */
572
573 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
574 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG;
575 po->__po_vlan_tag = vlan_tag;
576
577 if (tag_in_pkt) {
578 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG_IN_PKT;
579 }
580 return 0;
581 }
582
583 __attribute__((always_inline))
584 static inline uint16_t
__packet_get_vlan_id(const uint16_t vlan_tag)585 __packet_get_vlan_id(const uint16_t vlan_tag)
586 {
587 return EVL_VLANOFTAG(vlan_tag);
588 }
589
590 __attribute__((always_inline))
591 static inline uint8_t
__packet_get_vlan_priority(const uint16_t vlan_tag)592 __packet_get_vlan_priority(const uint16_t vlan_tag)
593 {
594 return EVL_PRIOFTAG(vlan_tag);
595 }
596
597 __attribute__((always_inline))
598 static inline errno_t
__packet_get_app_metadata(const uint64_t ph,packet_app_metadata_type_t * app_type,uint8_t * app_metadata)599 __packet_get_app_metadata(const uint64_t ph,
600 packet_app_metadata_type_t *app_type, uint8_t *app_metadata)
601 {
602 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
603 if (app_type == NULL || app_metadata == NULL) {
604 return EINVAL;
605 }
606 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_APP_METADATA) == 0) {
607 return ENOENT;
608 }
609 #ifdef KERNEL
610 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
611 #else /* !KERNEL */
612 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
613 #endif /* !KERNEL */
614 if (po->__po_app_type == PACKET_APP_METADATA_TYPE_UNSPECIFIED) {
615 return ENOENT;
616 }
617 *app_type = po->__po_app_type;
618 *app_metadata = po->__po_app_metadata;
619 return 0;
620 }
621
622 __attribute__((always_inline))
623 static inline errno_t
__packet_set_app_metadata(const uint64_t ph,const packet_app_metadata_type_t app_type,const uint8_t app_metadata)624 __packet_set_app_metadata(const uint64_t ph,
625 const packet_app_metadata_type_t app_type, const uint8_t app_metadata)
626 {
627 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
628 #ifdef KERNEL
629 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
630 #else /* !KERNEL */
631 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
632 #endif /* !KERNEL */
633 if (app_type < PACKET_APP_METADATA_TYPE_MIN ||
634 app_type > PACKET_APP_METADATA_TYPE_MAX) {
635 po->__po_app_type = PACKET_APP_METADATA_TYPE_UNSPECIFIED;
636 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_APP_METADATA;
637 return EINVAL;
638 }
639 po->__po_app_type = app_type;
640 po->__po_app_metadata = app_metadata;
641 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_APP_METADATA;
642 return 0;
643 }
644
645 #ifdef KERNEL
646 __attribute__((always_inline))
647 static inline void
__packet_set_wake_flag(const uint64_t ph)648 __packet_set_wake_flag(const uint64_t ph)
649 {
650 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
651 PKT_ADDR(ph)->pkt_pflags |= PKT_F_WAKE_PKT;
652 }
653 #endif
654
655 __attribute__((always_inline))
656 static inline boolean_t
__packet_get_wake_flag(const uint64_t ph)657 __packet_get_wake_flag(const uint64_t ph)
658 {
659 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_WAKE_PKT) != 0;
660 }
661
662 __attribute__((always_inline))
663 static inline void
__packet_set_keep_alive(const uint64_t ph,const boolean_t is_keep_alive)664 __packet_set_keep_alive(const uint64_t ph, const boolean_t is_keep_alive)
665 {
666 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
667 if (is_keep_alive) {
668 PKT_ADDR(ph)->pkt_pflags |= PKT_F_KEEPALIVE;
669 } else {
670 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_KEEPALIVE;
671 }
672 }
673
674 __attribute__((always_inline))
675 static inline boolean_t
__packet_get_keep_alive(const uint64_t ph)676 __packet_get_keep_alive(const uint64_t ph)
677 {
678 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_KEEPALIVE) != 0;
679 }
680
681 __attribute__((always_inline))
682 static inline boolean_t
__packet_get_truncated(const uint64_t ph)683 __packet_get_truncated(const uint64_t ph)
684 {
685 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
686 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_TRUNCATED) != 0;
687 }
688
689 #ifdef KERNEL
690 __attribute__((always_inline))
691 static inline boolean_t
__packet_get_transport_new_flow(const uint64_t ph)692 __packet_get_transport_new_flow(const uint64_t ph)
693 {
694 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
695 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_NEW_FLOW) != 0;
696 }
697
698 __attribute__((always_inline))
699 static inline boolean_t
__packet_get_transport_last_packet(const uint64_t ph)700 __packet_get_transport_last_packet(const uint64_t ph)
701 {
702 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
703 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_LAST_PKT) != 0;
704 }
705 #endif /* KERNEL */
706
707 __attribute__((always_inline))
708 static inline int
__packet_set_service_class(const uint64_t ph,const uint32_t sc)709 __packet_set_service_class(const uint64_t ph, const uint32_t sc)
710 {
711 int err = 0;
712
713 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
714
715 switch (sc) {
716 case PKT_SC_BE:
717 case PKT_SC_BK_SYS:
718 case PKT_SC_BK:
719 case PKT_SC_RD:
720 case PKT_SC_OAM:
721 case PKT_SC_AV:
722 case PKT_SC_RV:
723 case PKT_SC_VI:
724 case PKT_SC_SIG:
725 case PKT_SC_VO:
726 case PKT_SC_CTL:
727 QUM_ADDR(ph)->qum_svc_class = sc;
728 break;
729
730 default:
731 err = EINVAL;
732 break;
733 }
734
735 return err;
736 }
737
738 __attribute__((always_inline))
739 static inline uint32_t
__packet_get_service_class(const uint64_t ph)740 __packet_get_service_class(const uint64_t ph)
741 {
742 uint32_t sc;
743
744 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
745
746 switch (QUM_ADDR(ph)->qum_svc_class) {
747 case PKT_SC_BE: /* most likely best effort */
748 case PKT_SC_BK_SYS:
749 case PKT_SC_BK:
750 case PKT_SC_RD:
751 case PKT_SC_OAM:
752 case PKT_SC_AV:
753 case PKT_SC_RV:
754 case PKT_SC_VI:
755 case PKT_SC_SIG:
756 case PKT_SC_VO:
757 case PKT_SC_CTL:
758 sc = QUM_ADDR(ph)->qum_svc_class;
759 break;
760
761 default:
762 sc = PKT_SC_BE;
763 break;
764 }
765
766 return sc;
767 }
768
769 __attribute__((always_inline))
770 static inline errno_t
__packet_set_comp_gencnt(const uint64_t ph,const uint32_t gencnt)771 __packet_set_comp_gencnt(const uint64_t ph, const uint32_t gencnt)
772 {
773 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
774 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
775
776 PKT_ADDR(ph)->pkt_comp_gencnt = gencnt;
777
778 return 0;
779 }
780
781 __attribute__((always_inline))
782 static inline errno_t
__packet_get_comp_gencnt(const uint64_t ph,uint32_t * pgencnt)783 __packet_get_comp_gencnt(const uint64_t ph, uint32_t *pgencnt)
784 {
785 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
786 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
787
788 if (pgencnt == NULL) {
789 return EINVAL;
790 }
791
792 if (PKT_ADDR(ph)->pkt_comp_gencnt == 0) {
793 return ENOENT;
794 }
795
796 *pgencnt = PKT_ADDR(ph)->pkt_comp_gencnt;
797 return 0;
798 }
799
800 __attribute__((always_inline))
801 static inline int
__packet_set_traffic_class(const uint64_t ph,const uint32_t tc)802 __packet_set_traffic_class(const uint64_t ph, const uint32_t tc)
803 {
804 uint32_t val = PKT_TC2SCVAL(tc); /* just the val portion */
805 uint32_t sc;
806
807 switch (val) {
808 case PKT_SCVAL_BK_SYS:
809 sc = PKT_SC_BK_SYS;
810 break;
811 case PKT_SCVAL_BK:
812 sc = PKT_SC_BK;
813 break;
814 case PKT_SCVAL_BE:
815 sc = PKT_SC_BE;
816 break;
817 case PKT_SCVAL_RD:
818 sc = PKT_SC_RD;
819 break;
820 case PKT_SCVAL_OAM:
821 sc = PKT_SC_OAM;
822 break;
823 case PKT_SCVAL_AV:
824 sc = PKT_SC_AV;
825 break;
826 case PKT_SCVAL_RV:
827 sc = PKT_SC_RV;
828 break;
829 case PKT_SCVAL_VI:
830 sc = PKT_SC_VI;
831 break;
832 case PKT_SCVAL_SIG:
833 sc = PKT_SC_SIG;
834 break;
835 case PKT_SCVAL_VO:
836 sc = PKT_SC_VO;
837 break;
838 case PKT_SCVAL_CTL:
839 sc = PKT_SC_CTL;
840 break;
841 default:
842 sc = PKT_SC_BE;
843 break;
844 }
845
846 return __packet_set_service_class(ph, sc);
847 }
848
849 __attribute__((always_inline))
850 static inline uint32_t
__packet_get_traffic_class(const uint64_t ph)851 __packet_get_traffic_class(const uint64_t ph)
852 {
853 return PKT_SC2TC(__packet_get_service_class(ph));
854 }
855
856 __attribute__((always_inline))
857 static inline int
__packet_set_inet_checksum(const uint64_t ph,const packet_csum_flags_t flags,const uint16_t start,const uint16_t stuff_val,boolean_t tx)858 __packet_set_inet_checksum(const uint64_t ph, const packet_csum_flags_t flags,
859 const uint16_t start, const uint16_t stuff_val, boolean_t tx)
860 {
861 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
862
863 PKT_ADDR(ph)->pkt_csum_flags = flags & (~PACKET_CSUM_TSO_FLAGS);
864
865 if (tx) {
866 PKT_ADDR(ph)->pkt_csum_tx_start_off = start;
867 PKT_ADDR(ph)->pkt_csum_tx_stuff_off = stuff_val;
868 } else {
869 PKT_ADDR(ph)->pkt_csum_rx_start_off = start;
870 PKT_ADDR(ph)->pkt_csum_rx_value = stuff_val;
871 }
872 return 0;
873 }
874
875 __attribute__((always_inline))
876 static inline packet_csum_flags_t
__packet_get_inet_checksum(const uint64_t ph,uint16_t * start,uint16_t * stuff_val,boolean_t tx)877 __packet_get_inet_checksum(const uint64_t ph, uint16_t *start,
878 uint16_t *stuff_val, boolean_t tx)
879 {
880 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
881
882 if (tx) {
883 if (__probable(start != NULL)) {
884 *start = PKT_ADDR(ph)->pkt_csum_tx_start_off;
885 }
886 if (__probable(stuff_val != NULL)) {
887 *stuff_val = PKT_ADDR(ph)->pkt_csum_tx_stuff_off;
888 }
889 } else {
890 if (__probable(start != NULL)) {
891 *start = PKT_ADDR(ph)->pkt_csum_rx_start_off;
892 }
893 if (__probable(stuff_val != NULL)) {
894 *stuff_val = PKT_ADDR(ph)->pkt_csum_rx_value;
895 }
896 }
897 return PKT_ADDR(ph)->pkt_csum_flags & (~PACKET_CSUM_TSO_FLAGS);
898 }
899
900 __attribute__((always_inline))
901 static inline void
__packet_set_flow_uuid(const uint64_t ph,const uuid_t flow_uuid)902 __packet_set_flow_uuid(const uint64_t ph, const uuid_t flow_uuid)
903 {
904 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
905
906 /*
907 * Anticipate a nicely (8-bytes) aligned UUID from caller;
908 * the one in qum_flow_id is always 8-byte aligned.
909 */
910 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
911 uint64_t *id_64 = (uint64_t *)(uintptr_t)flow_uuid;
912 q->__q_flow_id_val64[0] = id_64[0];
913 q->__q_flow_id_val64[1] = id_64[1];
914 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
915 uint32_t *id_32 = (uint32_t *)(uintptr_t)flow_uuid;
916 q->__q_flow_id_val32[0] = id_32[0];
917 q->__q_flow_id_val32[1] = id_32[1];
918 q->__q_flow_id_val32[2] = id_32[2];
919 q->__q_flow_id_val32[3] = id_32[3];
920 } else {
921 bcopy(flow_uuid, q->__q_flow_id, sizeof(uuid_t));
922 }
923 }
924
925 __attribute__((always_inline))
926 static inline void
__packet_get_flow_uuid(const uint64_t ph,uuid_t flow_uuid)927 __packet_get_flow_uuid(const uint64_t ph, uuid_t flow_uuid)
928 {
929 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
930
931 /*
932 * Anticipate a nicely (8-bytes) aligned UUID from caller;
933 * the one in qum_flow_id is always 8-byte aligned.
934 */
935 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
936 uint64_t *id_64 = (uint64_t *)(uintptr_t)flow_uuid;
937 id_64[0] = q->__q_flow_id_val64[0];
938 id_64[1] = q->__q_flow_id_val64[1];
939 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
940 uint32_t *id_32 = (uint32_t *)(uintptr_t)flow_uuid;
941 id_32[0] = q->__q_flow_id_val32[0];
942 id_32[1] = q->__q_flow_id_val32[1];
943 id_32[2] = q->__q_flow_id_val32[2];
944 id_32[3] = q->__q_flow_id_val32[3];
945 } else {
946 bcopy(q->__q_flow_id, flow_uuid, sizeof(uuid_t));
947 }
948 }
949
950 __attribute__((always_inline))
951 static inline void
__packet_clear_flow_uuid(const uint64_t ph)952 __packet_clear_flow_uuid(const uint64_t ph)
953 {
954 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
955 q->__q_flow_id_val64[0] = 0;
956 q->__q_flow_id_val64[1] = 0;
957 }
958
959 __attribute__((always_inline))
960 static inline uint8_t
__packet_get_aggregation_type(const uint64_t ph)961 __packet_get_aggregation_type(const uint64_t ph)
962 {
963 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_aggr_type == sizeof(uint8_t)));
964 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
965
966 return PKT_ADDR(ph)->pkt_aggr_type;
967 }
968
969 __attribute__((always_inline))
970 static inline uint32_t
__packet_get_data_length(const uint64_t ph)971 __packet_get_data_length(const uint64_t ph)
972 {
973 return QUM_ADDR(ph)->qum_len;
974 }
975
976 __attribute__((always_inline))
977 static inline uint16_t
__packet_get_buflet_count(const uint64_t ph)978 __packet_get_buflet_count(const uint64_t ph)
979 {
980 uint16_t bcnt = 0;
981
982 switch (SK_PTR_TYPE(ph)) {
983 case NEXUS_META_TYPE_PACKET:
984 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
985 #ifdef KERNEL
986 VERIFY(bcnt != 0 ||
987 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
988 #else /* !KERNEL */
989 /*
990 * Handle the case where the metadata region gets
991 * redirected to anonymous zero-filled pages at
992 * defunct time. There's always 1 buflet in the
993 * packet metadata, so pretend that's the count.
994 */
995 if (__improbable(bcnt == 0)) {
996 bcnt = 1;
997 }
998 #endif /* !KERNEL */
999 break;
1000 case NEXUS_META_TYPE_QUANTUM:
1001 bcnt = 1;
1002 break;
1003 default:
1004 #ifdef KERNEL
1005 VERIFY(0);
1006 /* NOTREACHED */
1007 __builtin_unreachable();
1008 #endif /* KERNEL */
1009 break;
1010 }
1011 return bcnt;
1012 }
1013
1014 __attribute__((always_inline))
1015 static inline int
__packet_add_buflet(const uint64_t ph,const void * bprev0,const void * bnew0)1016 __packet_add_buflet(const uint64_t ph, const void *bprev0, const void *bnew0)
1017 {
1018 uint16_t bcnt;
1019
1020 #ifdef KERNEL
1021 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1022 kern_buflet_t bnew = __DECONST(kern_buflet_t, bnew0);
1023
1024 VERIFY(PKT_ADDR(ph) && bnew && (bnew != bprev));
1025 VERIFY(PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1026 VERIFY(bnew->buf_ctl != NULL);
1027 #else /* !KERNEL */
1028 buflet_t bprev = __DECONST(buflet_t, bprev0);
1029 buflet_t bnew = __DECONST(buflet_t, bnew0);
1030
1031 if (__improbable(!PKT_ADDR(ph) || !bnew || (bnew == bprev))) {
1032 return EINVAL;
1033 }
1034 #endif /* !KERNEL */
1035
1036 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1037 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1038
1039 #ifdef KERNEL
1040 VERIFY((bprev != NULL || bcnt == 0) &&
1041 (bcnt < PKT_ADDR(ph)->pkt_bufs_max));
1042 #else /* !KERNEL */
1043 if (__improbable(bcnt >= PKT_ADDR(ph)->pkt_bufs_max) ||
1044 (bprev == NULL && bcnt != 0)) {
1045 return EINVAL;
1046 }
1047 #endif /* !KERNEL */
1048
1049 #ifdef KERNEL
1050 #if DEVELOPMENT || DEBUG
1051 /* check if bprev is the last buflet in the chain */
1052 struct __kern_buflet *pbft, *kbft;
1053 int n = bcnt;
1054
1055 PKT_GET_FIRST_BUFLET(PKT_ADDR(ph), bcnt, pbft);
1056 kbft = pbft;
1057
1058 while ((kbft != NULL) && n--) {
1059 pbft = kbft;
1060 kbft = __DECONST(struct __kern_buflet *, kbft->buf_nbft_addr);
1061 }
1062 ASSERT(n == 0);
1063 ASSERT(bprev == pbft);
1064 #endif /* DEVELOPMENT || DEBUG */
1065 #endif /* KERNEL */
1066
1067 if (bprev == NULL) {
1068 bprev = &PKT_ADDR(ph)->pkt_qum_buf;
1069 }
1070 #ifdef KERNEL
1071 KBUF_LINK(bprev, bnew);
1072 #else /* !KERNEL */
1073 UBUF_LINK(bprev, bnew);
1074 #endif /* !KERNEL */
1075
1076 *(uint16_t *)(uintptr_t)&PKT_ADDR(ph)->pkt_bufs_cnt = ++bcnt;
1077 return 0;
1078 }
1079
1080 __attribute__((always_inline))
1081 static inline void *
__packet_get_next_buflet(const uint64_t ph,const void * bprev0)1082 __packet_get_next_buflet(const uint64_t ph, const void *bprev0)
1083 {
1084 #ifdef KERNEL
1085 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1086 #else /* !KERNEL */
1087 buflet_t bprev = __DECONST(buflet_t, bprev0);
1088 #endif /* !KERNEL */
1089 void *bcur = NULL;
1090
1091 switch (SK_PTR_TYPE(ph)) {
1092 case NEXUS_META_TYPE_PACKET: {
1093 uint32_t bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1094 #ifdef KERNEL
1095 ASSERT(bcnt != 0 ||
1096 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1097 #else /* !KERNEL */
1098 /*
1099 * Handle the case where the metadata region gets
1100 * redirected to anonymous zero-filled pages at
1101 * defunct time. There's always 1 buflet in the
1102 * packet metadata, so pretend that's the count.
1103 */
1104 if (__improbable(bcnt == 0)) {
1105 bcnt = 1;
1106 bprev = NULL;
1107 }
1108 #endif /* !KERNEL */
1109 PKT_GET_NEXT_BUFLET(PKT_ADDR(ph), bcnt, BLT_ADDR(bprev), bcur);
1110 break;
1111 }
1112 case NEXUS_META_TYPE_QUANTUM:
1113 QUM_GET_NEXT_BUFLET(QUM_ADDR(ph), BLT_ADDR(bprev), bcur);
1114 break;
1115 default:
1116 #ifdef KERNEL
1117 VERIFY(0);
1118 /* NOTREACHED */
1119 __builtin_unreachable();
1120 #endif /* KERNEL */
1121 break;
1122 }
1123 return bcur;
1124 }
1125
1126 __attribute__((always_inline))
1127 static inline uint8_t
__packet_get_segment_count(const uint64_t ph)1128 __packet_get_segment_count(const uint64_t ph)
1129 {
1130 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1131 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1132
1133 return PKT_ADDR(ph)->pkt_seg_cnt;
1134 }
1135
1136 __attribute__((always_inline))
1137 static inline void
__packet_set_segment_count(const uint64_t ph,uint8_t segcount)1138 __packet_set_segment_count(const uint64_t ph, uint8_t segcount)
1139 {
1140 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1141 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1142
1143 PKT_ADDR(ph)->pkt_seg_cnt = segcount;
1144 }
1145
1146 __attribute__((always_inline))
1147 static inline errno_t
__packet_get_protocol_segment_size(const uint64_t ph,uint16_t * proto_seg_sz)1148 __packet_get_protocol_segment_size(const uint64_t ph, uint16_t *proto_seg_sz)
1149 {
1150 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_proto_seg_sz == sizeof(uint16_t)));
1151 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1152 *proto_seg_sz = PKT_ADDR(ph)->pkt_proto_seg_sz;
1153 return 0;
1154 }
1155
1156 __attribute__((always_inline))
1157 static inline errno_t
__packet_set_protocol_segment_size(const uint64_t ph,uint16_t proto_seg_sz)1158 __packet_set_protocol_segment_size(const uint64_t ph, uint16_t proto_seg_sz)
1159 {
1160 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1161 PKT_ADDR(ph)->pkt_proto_seg_sz = proto_seg_sz;
1162 return 0;
1163 }
1164
1165 __attribute__((always_inline))
1166 static inline void
__packet_get_tso_flags(const uint64_t ph,packet_tso_flags_t * flags)1167 __packet_get_tso_flags(const uint64_t ph, packet_tso_flags_t *flags)
1168 {
1169 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_proto_seg_sz == sizeof(uint16_t)));
1170
1171 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1172 *flags = PKT_ADDR(ph)->pkt_csum_flags & (PACKET_CSUM_TSO_FLAGS);
1173 }
1174
1175 __attribute__((always_inline))
1176 static inline void
__packet_set_tso_flags(const uint64_t ph,packet_tso_flags_t flags)1177 __packet_set_tso_flags(const uint64_t ph, packet_tso_flags_t flags)
1178 {
1179 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1180
1181 PKT_ADDR(ph)->pkt_csum_flags |= flags & (PACKET_CSUM_TSO_FLAGS);
1182 }
1183
1184 __attribute__((always_inline))
1185 static inline uint16_t
__buflet_get_data_limit(const void * buf)1186 __buflet_get_data_limit(const void *buf)
1187 {
1188 return BLT_ADDR(buf)->buf_dlim;
1189 }
1190
1191 #ifdef KERNEL
1192 __attribute__((always_inline))
1193 static inline errno_t
__buflet_set_data_limit(const void * buf,const uint16_t dlim)1194 __buflet_set_data_limit(const void *buf, const uint16_t dlim)
1195 {
1196 /* buffer region is always marked as shareable */
1197 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1198
1199 /* full bounds checking will be performed during finalize */
1200 if (__probable((uint32_t)dlim + BLT_ADDR(buf)->buf_boff <=
1201 BLT_ADDR(buf)->buf_objlim)) {
1202 _CASSERT(sizeof(BLT_ADDR(buf)->buf_dlim) == sizeof(uint16_t));
1203 /* deconst */
1204 *(uint16_t *)(uintptr_t)&BLT_ADDR(buf)->buf_dlim = dlim;
1205 return 0;
1206 }
1207 return ERANGE;
1208 }
1209 #endif /* KERNEL */
1210
1211 __attribute__((always_inline))
1212 static inline uint16_t
__buflet_get_data_offset(const void * buf)1213 __buflet_get_data_offset(const void *buf)
1214 {
1215 return BLT_ADDR(buf)->buf_doff;
1216 }
1217
1218 /*
1219 * ******************************************************************
1220 * Checks in __packet_finalize for packet finalized from userland
1221 * ******************************************************************
1222 * +-------+---------------------------+---------------------------+
1223 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1224 * |-------+---------------------------+---------------------------+
1225 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1226 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1227 * |l3_off | l3 = l2 | l3 == 0 |
1228 * |l4_off | l4 = l3 = l2 | l4 = l3 = 0 |
1229 * +-------+---------------------------+---------------------------+
1230 *
1231 * ******************************************************************
1232 * Checks in __packet_finalize for packet finalized from kernel
1233 * ******************************************************************
1234 * +-------+---------------------------+---------------------------+
1235 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1236 * |-------+---------------------------+---------------------------+
1237 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1238 * |buflet | (boff + objaddr) == addr | (boff + objaddr) <= addr |
1239 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1240 * |l3_off | l3 >= l2 && l3 <bdlim | l3 == 0 |
1241 * |l4_off | l4 = l3 | l4 = l3 = 0 |
1242 * +-------+---------------------------+---------------------------+
1243 *
1244 */
1245 __attribute__((always_inline))
1246 static inline int
__packet_finalize(const uint64_t ph)1247 __packet_finalize(const uint64_t ph)
1248 {
1249 void *bcur = NULL, *bprev = NULL;
1250 uint32_t len, bcnt, bdoff0, bdlim0;
1251 int err = 0;
1252
1253 #ifdef KERNEL
1254 ASSERT(QUM_ADDR(ph)->qum_qflags & QUM_F_INTERNALIZED);
1255 #endif /* KERNEL */
1256 QUM_ADDR(ph)->qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1257
1258 bcnt = __packet_get_buflet_count(ph);
1259 len = QUM_ADDR(ph)->qum_len = 0;
1260
1261 while (bcnt--) {
1262 bcur = __packet_get_next_buflet(ph, bprev);
1263
1264 #ifdef KERNEL
1265 ASSERT(bcur != NULL);
1266 ASSERT(BLT_ADDR(bcur)->buf_addr != 0);
1267 #else /* !KERNEL */
1268 if (__improbable(bcur == NULL || BLT_ADDR(bcur)->buf_grolen != 0)) {
1269 err = ERANGE;
1270 break;
1271 }
1272 #endif /* KERNEL */
1273
1274 /* save data offset from the first buflet */
1275 if (bprev == NULL) {
1276 bdoff0 = __buflet_get_data_offset(bcur);
1277 bdlim0 = __buflet_get_data_limit(bcur);
1278 }
1279
1280 #ifndef KERNEL
1281 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)))) {
1282 err = ERANGE;
1283 break;
1284 }
1285 #else /* !KERNEL */
1286 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)) &&
1287 !PKT_HAS_ATTACHED_MBUF(ph))) {
1288 err = ERANGE;
1289 break;
1290 }
1291 #endif /* KERNEL */
1292 len += BLT_ADDR(bcur)->buf_dlen;
1293 bprev = bcur;
1294 }
1295
1296 if (__improbable(err != 0)) {
1297 goto done;
1298 }
1299
1300 switch (SK_PTR_TYPE(ph)) {
1301 case NEXUS_META_TYPE_PACKET:
1302 if (__improbable(bdoff0 > UINT8_MAX)) {
1303 err = ERANGE;
1304 goto done;
1305 }
1306 /* internalize headroom value from offset */
1307 PKT_ADDR(ph)->pkt_headroom = (uint8_t)bdoff0;
1308 /* validate header offsets in packet */
1309 switch (SK_PTR_SUBTYPE(ph)) {
1310 case NEXUS_META_SUBTYPE_RAW:
1311 #ifndef KERNEL
1312 /* Overwrite L2 len for raw packets from user space */
1313 PKT_ADDR(ph)->pkt_l2_len = 0;
1314 #else /* !KERNEL */
1315 /* ensure that L3 >= L2 && L3 < bdlim */
1316 if (__improbable((PKT_ADDR(ph)->pkt_headroom +
1317 PKT_ADDR(ph)->pkt_l2_len) >= bdlim0)) {
1318 err = ERANGE;
1319 goto done;
1320 }
1321 #endif /* KERNEL */
1322 break;
1323 case NEXUS_META_SUBTYPE_PAYLOAD:
1324 /*
1325 * For payload packet there is no concept of headroom
1326 * and L3 offset should always be 0
1327 */
1328 if (__improbable((PKT_ADDR(ph)->pkt_headroom != 0) ||
1329 (PKT_ADDR(ph)->pkt_l2_len != 0))) {
1330 err = ERANGE;
1331 goto done;
1332 }
1333 break;
1334 default:
1335 #ifdef KERNEL
1336 VERIFY(0);
1337 /* NOTREACHED */
1338 __builtin_unreachable();
1339 #endif /* KERNEL */
1340 break;
1341 }
1342
1343 if (__improbable(PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_DATA)) {
1344 #ifdef KERNEL
1345 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
1346 #else /* !KERNEL */
1347 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
1348 #endif /* !KERNEL */
1349 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1350 po->__po_expire_ts == 0) {
1351 err = EINVAL;
1352 goto done;
1353 }
1354 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) &&
1355 po->__po_token_len == 0) {
1356 err = EINVAL;
1357 goto done;
1358 }
1359 ASSERT(err == 0);
1360 }
1361
1362 /*
1363 * NOTE: we don't need the validation for total packet length
1364 * as checking if each buflet is in range and that
1365 * (pkt_headroom == bdoff0), should cover this check.
1366 */
1367 break;
1368
1369 default:
1370 /* nothing to do currently for quantum */
1371 break;
1372 }
1373
1374 done:
1375 if (__probable(err == 0)) {
1376 QUM_ADDR(ph)->qum_len = len;
1377 QUM_ADDR(ph)->qum_qflags |= QUM_F_FINALIZED;
1378 } else {
1379 QUM_ADDR(ph)->qum_len = 0;
1380 QUM_ADDR(ph)->qum_qflags |= QUM_F_DROPPED;
1381 }
1382
1383 return err;
1384 }
1385
1386 __attribute__((always_inline))
1387 static inline boolean_t
__packet_is_finalized(const uint64_t ph)1388 __packet_is_finalized(const uint64_t ph)
1389 {
1390 return QUM_ADDR(ph)->qum_qflags & QUM_F_FINALIZED;
1391 }
1392
1393 #ifdef KERNEL
1394 /*
1395 * function to initialize a packet with mbuf chain.
1396 * Apart from the attached mbuf, the packet can also be used to convey
1397 * additional metadata like the headroom and L2 header length.
1398 * For a packet with attached mbuf, the pkt_length conveys the length of
1399 * the attached mbuf. If the data copied is partial then PKT_F_TRUNCATED is
1400 * also set.
1401 */
1402 __attribute__((always_inline))
1403 static inline int
__packet_initialize_with_mbufchain(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1404 __packet_initialize_with_mbufchain(struct __kern_packet *pkt, struct mbuf *mbuf,
1405 uint8_t headroom, uint8_t l2len)
1406 {
1407 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1408 VERIFY(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1409 VERIFY((pkt->pkt_pflags & PKT_F_MBUF_MASK) == 0);
1410 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
1411 VERIFY(pkt->pkt_mbuf == NULL);
1412
1413 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1414 pkt->pkt_mbuf = mbuf;
1415 pkt->pkt_pflags |= (PKT_F_MBUF_DATA | PKT_F_TRUNCATED);
1416 pkt->pkt_headroom = headroom;
1417 pkt->pkt_l2_len = l2len;
1418 pkt->pkt_length = m_pktlen(mbuf);
1419 pkt->pkt_qum_buf.buf_dlen = 0;
1420 pkt->pkt_qum_buf.buf_doff = 0;
1421 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1422 return 0;
1423 }
1424
1425 __attribute__((always_inline))
1426 static inline int
__packet_initialize_with_mbuf(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1427 __packet_initialize_with_mbuf(struct __kern_packet *pkt, struct mbuf *mbuf,
1428 uint8_t headroom, uint8_t l2len)
1429 {
1430 __packet_initialize_with_mbufchain(pkt, mbuf, headroom, l2len);
1431 VERIFY(mbuf->m_nextpkt == NULL);
1432 return 0;
1433 }
1434
1435 /*
1436 * function to finalize a packet with attached mbuf.
1437 */
1438 __attribute__((always_inline))
1439 static inline int
__packet_finalize_with_mbuf(struct __kern_packet * pkt)1440 __packet_finalize_with_mbuf(struct __kern_packet *pkt)
1441 {
1442 uint16_t bdoff, bdlim, bdlen;
1443 struct __kern_buflet *buf;
1444 int err = 0;
1445
1446 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1447 VERIFY((pkt->pkt_pflags & (PKT_F_MBUF_DATA | PKT_F_PKT_DATA)) ==
1448 PKT_F_MBUF_DATA);
1449 VERIFY(pkt->pkt_mbuf != NULL);
1450 ASSERT(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1451 VERIFY(pkt->pkt_bufs_cnt == 1);
1452 PKT_GET_FIRST_BUFLET(pkt, pkt->pkt_bufs_cnt, buf);
1453 ASSERT(buf->buf_addr != 0);
1454
1455 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1456 pkt->pkt_pflags &= ~PKT_F_TRUNCATED;
1457 bdlen = buf->buf_dlen;
1458 bdlim = buf->buf_dlim;
1459 bdoff = buf->buf_doff;
1460 if (__improbable(!BUF_IN_RANGE(buf))) {
1461 err = ERANGE;
1462 goto done;
1463 }
1464
1465 /* validate header offsets in packet */
1466 switch (METADATA_SUBTYPE(pkt)) {
1467 case NEXUS_META_SUBTYPE_RAW:
1468 if (__improbable((pkt->pkt_headroom != bdoff) ||
1469 (pkt->pkt_headroom >= bdlim))) {
1470 err = ERANGE;
1471 goto done;
1472 }
1473 if (__improbable((pkt->pkt_headroom +
1474 pkt->pkt_l2_len) >= bdlim)) {
1475 err = ERANGE;
1476 goto done;
1477 }
1478 break;
1479
1480 case NEXUS_META_SUBTYPE_PAYLOAD:
1481 /*
1482 * For payload packet there is no concept of headroom.
1483 */
1484 if (__improbable((pkt->pkt_headroom != 0) || (bdoff != 0) ||
1485 (pkt->pkt_l2_len != 0))) {
1486 err = ERANGE;
1487 goto done;
1488 }
1489 break;
1490
1491 default:
1492 VERIFY(0);
1493 /* NOTREACHED */
1494 __builtin_unreachable();
1495 break;
1496 }
1497
1498
1499 if (__improbable(pkt->pkt_pflags & PKT_F_OPT_DATA)) {
1500 struct __packet_opt *po = pkt->pkt_com_opt;
1501
1502 if ((pkt->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1503 po->__po_expire_ts == 0) {
1504 err = EINVAL;
1505 goto done;
1506 }
1507 if ((pkt->pkt_pflags & PKT_F_OPT_TOKEN) &&
1508 po->__po_token_len == 0) {
1509 err = EINVAL;
1510 goto done;
1511 }
1512 }
1513 ASSERT(err == 0);
1514
1515 done:
1516 if (__probable(err == 0)) {
1517 pkt->pkt_length = (uint32_t)m_pktlen(pkt->pkt_mbuf);
1518 if (bdlen < pkt->pkt_length) {
1519 pkt->pkt_pflags |= PKT_F_TRUNCATED;
1520 }
1521 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1522 } else {
1523 pkt->pkt_length = 0;
1524 pkt->pkt_qum.qum_qflags |= QUM_F_DROPPED;
1525 }
1526
1527 return err;
1528 }
1529
1530 __attribute__((always_inline))
1531 static inline uint32_t
__packet_get_object_index(const uint64_t ph)1532 __packet_get_object_index(const uint64_t ph)
1533 {
1534 return METADATA_IDX(QUM_ADDR(ph));
1535 }
1536
1537 __attribute__((always_inline))
1538 static inline errno_t
__packet_get_timestamp(const uint64_t ph,uint64_t * ts,boolean_t * valid)1539 __packet_get_timestamp(const uint64_t ph, uint64_t *ts, boolean_t *valid)
1540 {
1541 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1542
1543 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TS_VALID) != 0) {
1544 if (valid != NULL) {
1545 *valid = TRUE;
1546 }
1547 *ts = PKT_ADDR(ph)->pkt_timestamp;
1548 } else {
1549 if (valid != NULL) {
1550 *valid = FALSE;
1551 }
1552 *ts = 0;
1553 }
1554
1555 return 0;
1556 }
1557
1558 __attribute__((always_inline))
1559 static inline errno_t
__packet_set_timestamp(const uint64_t ph,uint64_t ts,boolean_t valid)1560 __packet_set_timestamp(const uint64_t ph, uint64_t ts, boolean_t valid)
1561 {
1562 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1563
1564 if (valid) {
1565 PKT_ADDR(ph)->pkt_timestamp = ts;
1566 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TS_VALID;
1567 } else {
1568 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_TS_VALID;
1569 PKT_ADDR(ph)->pkt_timestamp = 0;
1570 }
1571
1572 return 0;
1573 }
1574
1575 __attribute__((always_inline))
1576 static inline errno_t
__packet_get_tx_completion_data(const uint64_t ph,uintptr_t * cb_arg,uintptr_t * cb_data)1577 __packet_get_tx_completion_data(const uint64_t ph, uintptr_t *cb_arg,
1578 uintptr_t *cb_data)
1579 {
1580 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1581 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1582 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1583 *cb_arg = PKT_ADDR(ph)->pkt_tx_compl_cb_arg;
1584 *cb_data = PKT_ADDR(ph)->pkt_tx_compl_cb_data;
1585 } else {
1586 *cb_arg = 0;
1587 *cb_data = 0;
1588 }
1589 return 0;
1590 }
1591
1592 __attribute__((always_inline))
1593 static inline errno_t
__packet_set_tx_completion_data(const uint64_t ph,uintptr_t cb_arg,uintptr_t cb_data)1594 __packet_set_tx_completion_data(const uint64_t ph, uintptr_t cb_arg,
1595 uintptr_t cb_data)
1596 {
1597 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1598 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1599 PKT_ADDR(ph)->pkt_tx_compl_cb_arg = cb_arg;
1600 PKT_ADDR(ph)->pkt_tx_compl_cb_data = cb_data;
1601 return 0;
1602 }
1603
1604 __attribute__((always_inline))
1605 static inline errno_t
__packet_get_timestamp_requested(const uint64_t ph,boolean_t * requested)1606 __packet_get_timestamp_requested(const uint64_t ph, boolean_t *requested)
1607 {
1608 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1609 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0) {
1610 *requested = TRUE;
1611 } else {
1612 *requested = FALSE;
1613 }
1614 return 0;
1615 }
1616
1617 __attribute__((always_inline))
1618 static inline errno_t
__packet_get_tx_completion_status(const uint64_t ph,kern_return_t * status)1619 __packet_get_tx_completion_status(const uint64_t ph, kern_return_t *status)
1620 {
1621 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1622 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1623 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1624 *status = (kern_return_t)PKT_ADDR(ph)->pkt_tx_compl_status;
1625 } else {
1626 *status = 0;
1627 }
1628 return 0;
1629 }
1630
1631 __attribute__((always_inline))
1632 static inline errno_t
__packet_set_tx_completion_status(const uint64_t ph,kern_return_t status)1633 __packet_set_tx_completion_status(const uint64_t ph, kern_return_t status)
1634 {
1635 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1636 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1637 PKT_ADDR(ph)->pkt_tx_compl_status = (uint32_t)status;
1638 return 0;
1639 }
1640
1641 __attribute__((always_inline))
1642 static inline errno_t
__packet_set_tx_nx_port(const uint64_t ph,nexus_port_t nx_port,uint16_t vpna_gencnt)1643 __packet_set_tx_nx_port(const uint64_t ph, nexus_port_t nx_port,
1644 uint16_t vpna_gencnt)
1645 {
1646 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1647 PKT_ADDR(ph)->pkt_nx_port = nx_port;
1648 PKT_ADDR(ph)->pkt_vpna_gencnt = vpna_gencnt;
1649 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TX_PORT_DATA;
1650 return 0;
1651 }
1652
1653 __attribute__((always_inline))
1654 static inline errno_t
__packet_get_tx_nx_port(const uint64_t ph,nexus_port_t * nx_port,uint16_t * vpna_gencnt)1655 __packet_get_tx_nx_port(const uint64_t ph, nexus_port_t *nx_port,
1656 uint16_t *vpna_gencnt)
1657 {
1658 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1659 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_PORT_DATA) == 0) {
1660 return ENOTSUP;
1661 }
1662
1663 *nx_port = PKT_ADDR(ph)->pkt_nx_port;
1664 *vpna_gencnt = PKT_ADDR(ph)->pkt_vpna_gencnt;
1665 return 0;
1666 }
1667
1668 __attribute__((always_inline))
1669 static inline errno_t
__packet_get_tx_nx_port_id(const uint64_t ph,uint32_t * nx_port_id)1670 __packet_get_tx_nx_port_id(const uint64_t ph, uint32_t *nx_port_id)
1671 {
1672 errno_t err;
1673 nexus_port_t nx_port;
1674 uint16_t vpna_gencnt;
1675
1676 _CASSERT(sizeof(nx_port) == sizeof(uint16_t));
1677
1678 err = __packet_get_tx_nx_port(ph, &nx_port, &vpna_gencnt);
1679 if (err == 0) {
1680 *nx_port_id = PKT_COMPOSE_NX_PORT_ID(nx_port, vpna_gencnt);
1681 }
1682 return err;
1683 }
1684
1685
1686 __attribute__((always_inline))
1687 static inline errno_t
__packet_get_flowid(const uint64_t ph,packet_flowid_t * pflowid)1688 __packet_get_flowid(const uint64_t ph, packet_flowid_t *pflowid)
1689 {
1690 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1691 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_FLOW_ID) == 0) {
1692 return ENOENT;
1693 }
1694 *pflowid = PKT_ADDR(ph)->pkt_flow_token;
1695 return 0;
1696 }
1697 #endif /* KERNEL */
1698
1699 extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t);
1700
1701 __attribute__((always_inline))
1702 static inline uint16_t
__packet_fold_sum(uint32_t sum)1703 __packet_fold_sum(uint32_t sum)
1704 {
1705 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1706 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1707 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1708 return sum & 0xffff;
1709 }
1710
1711 __attribute__((always_inline))
1712 static inline uint16_t
__packet_fold_sum_final(uint32_t sum)1713 __packet_fold_sum_final(uint32_t sum)
1714 {
1715 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1716 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1717 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1718 return ~sum & 0xffff;
1719 }
1720
1721 __attribute__((always_inline))
1722 static inline uint32_t
__packet_cksum(const void * data,uint32_t len,uint32_t sum0)1723 __packet_cksum(const void *data, uint32_t len, uint32_t sum0)
1724 {
1725 return os_cpu_in_cksum(data, len, sum0);
1726 }
1727
1728 extern uint32_t os_cpu_copy_in_cksum(const void *, void *, uint32_t, uint32_t);
1729
1730 __attribute__((always_inline))
1731 static inline uint32_t
__packet_copy_and_sum(const void * src,void * dst,uint32_t len,uint32_t sum0)1732 __packet_copy_and_sum(const void *src, void *dst, uint32_t len, uint32_t sum0)
1733 {
1734 return os_cpu_copy_in_cksum(src, dst, len, sum0);
1735 }
1736
1737 __attribute__((always_inline))
1738 static inline uint16_t
__packet_fix_sum(uint16_t csum,uint16_t old,uint16_t new)1739 __packet_fix_sum(uint16_t csum, uint16_t old, uint16_t new)
1740 {
1741 uint32_t c = csum + old - new;
1742 c = (c >> 16) + (c & 0xffff); /* Only add carry once */
1743
1744 return c & 0xffff;
1745 }
1746
1747 /* MUST be used for uint32_t fields */
1748 __attribute__((always_inline))
1749 static inline void
__packet_fix_hdr_sum(uint8_t * field,uint16_t * csum,uint32_t new)1750 __packet_fix_hdr_sum(uint8_t *field, uint16_t *csum, uint32_t new)
1751 {
1752 uint32_t old;
1753 memcpy(&old, field, sizeof(old));
1754 memcpy(field, &new, sizeof(uint32_t));
1755 *csum = __packet_fix_sum(__packet_fix_sum(*csum, (uint16_t)(old >> 16),
1756 (uint16_t)(new >> 16)), (uint16_t)(old & 0xffff),
1757 (uint16_t)(new & 0xffff));
1758 }
1759
1760 __attribute__((always_inline))
1761 static inline void *
__buflet_get_data_address(const void * buf)1762 __buflet_get_data_address(const void *buf)
1763 {
1764 #if (defined(KERNEL) && (DEBUG || DEVELOPMENT))
1765 ASSERT(BLT_ADDR(buf)->buf_addr ==
1766 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr +
1767 BLT_ADDR(buf)->buf_boff);
1768 #endif /* KERNEL && (DEBUG || DEVELOPMENT) */
1769 return (void *)(BLT_ADDR(buf)->buf_addr);
1770 }
1771
1772 #ifdef KERNEL
1773 __attribute__((always_inline))
1774 static inline errno_t
__buflet_set_data_address(const void * buf,const void * addr)1775 __buflet_set_data_address(const void *buf, const void *addr)
1776 {
1777 /* buffer region is always marked as shareable */
1778 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1779
1780 /* full bounds checking will be performed during finalize */
1781 if (__probable((uintptr_t)addr >=
1782 (uintptr_t)BLT_ADDR(buf)->buf_objaddr)) {
1783 _CASSERT(sizeof(BLT_ADDR(buf)->buf_addr) ==
1784 sizeof(mach_vm_address_t));
1785 /* deconst */
1786 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1787 (mach_vm_address_t)addr;
1788
1789 /* compute the offset from objaddr for the case of shared buffer */
1790 _CASSERT(sizeof(BLT_ADDR(buf)->buf_boff) == sizeof(uint16_t));
1791 *(uint16_t *)(uintptr_t)&BLT_ADDR(buf)->buf_boff =
1792 (uint16_t)((mach_vm_address_t)addr -
1793 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr);
1794
1795 return 0;
1796 }
1797 return ERANGE;
1798 }
1799 #endif /* KERNEL */
1800
1801 __attribute__((always_inline))
1802 static inline int
__buflet_set_data_offset(const void * buf,const uint16_t doff)1803 __buflet_set_data_offset(const void *buf, const uint16_t doff)
1804 {
1805 #ifdef KERNEL
1806 /*
1807 * Kernel-specific assertion. For user space, the metadata
1808 * region gets redirected to anonymous zero-filled pages at
1809 * defunct time, so ignore it there.
1810 */
1811 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1812
1813 if (__probable(doff + BLT_ADDR(buf)->buf_boff <=
1814 BLT_ADDR(buf)->buf_objlim)) {
1815 BLT_ADDR(buf)->buf_doff = doff;
1816 return 0;
1817 }
1818 return ERANGE;
1819 #else /* !KERNEL */
1820 BLT_ADDR(buf)->buf_doff = doff;
1821 return 0;
1822 #endif /* KERNEL */
1823 }
1824
1825 __attribute__((always_inline))
1826 static inline int
__buflet_set_data_length(const void * buf,const uint16_t dlen)1827 __buflet_set_data_length(const void *buf, const uint16_t dlen)
1828 {
1829 #ifdef KERNEL
1830 /*
1831 * Kernel-specific assertion. For user space, the metadata
1832 * region gets redirected to anonymous zero-filled pages at
1833 * defunct time, so ignore it there.
1834 */
1835 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1836
1837 if (__probable((uint32_t)dlen <= BLT_ADDR(buf)->buf_objlim)) {
1838 BLT_ADDR(buf)->buf_dlen = dlen;
1839 return 0;
1840 }
1841 return ERANGE;
1842 #else /* !KERNEL */
1843 BLT_ADDR(buf)->buf_dlen = dlen;
1844 return 0;
1845 #endif /* KERNEL */
1846 }
1847
1848 __attribute__((always_inline))
1849 static inline uint16_t
__buflet_get_data_length(const void * buf)1850 __buflet_get_data_length(const void *buf)
1851 {
1852 return BLT_ADDR(buf)->buf_dlen;
1853 }
1854
1855 #ifdef KERNEL
1856 __attribute__((always_inline))
1857 static inline int
__buflet_set_buffer_offset(const void * buf,const uint16_t off)1858 __buflet_set_buffer_offset(const void *buf, const uint16_t off)
1859 {
1860 ASSERT(BLT_ADDR(buf)->buf_objlim != 0);
1861
1862 if (__probable(off <= BLT_ADDR(buf)->buf_objlim)) {
1863 _CASSERT(sizeof(BLT_ADDR(buf)->buf_boff) == sizeof(uint16_t));
1864 *(uint16_t *)(uintptr_t)&BLT_ADDR(buf)->buf_boff = off;
1865
1866 /* adjust dlim and buf_addr */
1867 if (BLT_ADDR(buf)->buf_dlim + off >= BLT_ADDR(buf)->buf_objlim) {
1868 _CASSERT(sizeof(BLT_ADDR(buf)->buf_dlim) == sizeof(uint16_t));
1869 *(uint16_t *)(uintptr_t)&BLT_ADDR(buf)->buf_dlim =
1870 (uint16_t)BLT_ADDR(buf)->buf_objlim - off;
1871 }
1872 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1873 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr + off;
1874 return 0;
1875 }
1876 return ERANGE;
1877 }
1878 #endif /* KERNEL */
1879
1880 __attribute__((always_inline))
1881 static inline uint16_t
__buflet_get_buffer_offset(const void * buf)1882 __buflet_get_buffer_offset(const void *buf)
1883 {
1884 #if (defined(KERNEL) && (DEBUG || DEVELOPMENT))
1885 ASSERT(BLT_ADDR(buf)->buf_addr ==
1886 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr +
1887 BLT_ADDR(buf)->buf_boff);
1888 #endif /* KERNEL && (DEBUG || DEVELOPMENT) */
1889 return BLT_ADDR(buf)->buf_boff;
1890 }
1891
1892 #ifdef KERNEL
1893 __attribute__((always_inline))
1894 static inline int
__buflet_set_gro_len(const void * buf,const uint16_t len)1895 __buflet_set_gro_len(const void *buf, const uint16_t len)
1896 {
1897 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1898
1899 if (__probable(len <= BLT_ADDR(buf)->buf_dlim)) {
1900 /* deconst */
1901 _CASSERT(sizeof(BLT_ADDR(buf)->buf_grolen) == sizeof(uint16_t));
1902 *(uint16_t *)(uintptr_t)&BLT_ADDR(buf)->buf_grolen = len;
1903 return 0;
1904 }
1905 return ERANGE;
1906 }
1907 #endif /* KERNEL */
1908
1909 __attribute__((always_inline))
1910 static inline uint16_t
__buflet_get_gro_len(const void * buf)1911 __buflet_get_gro_len(const void *buf)
1912 {
1913 return BLT_ADDR(buf)->buf_grolen;
1914 }
1915
1916 __attribute__((always_inline))
1917 static inline void *
__buflet_get_next_buf(const void * buflet,const void * prev_buf)1918 __buflet_get_next_buf(const void *buflet, const void *prev_buf)
1919 {
1920 uint16_t gro_len, dlen;
1921 mach_vm_address_t next_buf, baddr;
1922
1923 ASSERT(BLT_ADDR(buflet)->buf_dlen != 0);
1924 ASSERT(BLT_ADDR(buflet)->buf_grolen != 0);
1925
1926 gro_len = BLT_ADDR(buflet)->buf_grolen;
1927 dlen = BLT_ADDR(buflet)->buf_dlen;
1928 baddr = BLT_ADDR(buflet)->buf_addr;
1929 if (prev_buf != NULL) {
1930 ASSERT((mach_vm_address_t)prev_buf >= BLT_ADDR(buflet)->buf_addr);
1931 next_buf = (mach_vm_address_t)prev_buf + gro_len;
1932 } else {
1933 next_buf = BLT_ADDR(buflet)->buf_addr;
1934 }
1935
1936 if (__probable(next_buf < baddr + dlen)) {
1937 ASSERT(next_buf + gro_len <= baddr + dlen);
1938 return (void *)next_buf;
1939 }
1940
1941 return NULL;
1942 }
1943
1944 #ifdef KERNEL
1945 __attribute__((always_inline))
1946 static inline struct sksegment *
__buflet_get_object_segment(const void * buf,kern_obj_idx_seg_t * idx)1947 __buflet_get_object_segment(const void *buf, kern_obj_idx_seg_t *idx)
1948 {
1949 _CASSERT(sizeof(obj_idx_t) == sizeof(kern_obj_idx_seg_t));
1950
1951 if (idx != NULL) {
1952 *idx = BLT_ADDR(buf)->buf_ctl->bc_idx;
1953 }
1954
1955 return BLT_ADDR(buf)->buf_ctl->bc_slab->sl_seg;
1956 }
1957 #endif /* KERNEL */
1958
1959 __attribute__((always_inline))
1960 static inline void *
__buflet_get_object_address(const void * buf)1961 __buflet_get_object_address(const void *buf)
1962 {
1963 #ifdef KERNEL
1964 return (void *)(BLT_ADDR(buf)->buf_objaddr);
1965 #else /* !KERNEL */
1966 /*
1967 * For user space, shared buffer is not available and hence the data
1968 * address is immutable and is always the same as the underlying
1969 * buffer object address itself.
1970 */
1971 return __buflet_get_data_address(buf);
1972 #endif /* !KERNEL */
1973 }
1974
1975 __attribute__((always_inline))
1976 static inline uint32_t
__buflet_get_object_limit(const void * buf)1977 __buflet_get_object_limit(const void *buf)
1978 {
1979 #ifdef KERNEL
1980 return BLT_ADDR(buf)->buf_objlim;
1981 #else /* !KERNEL */
1982 /*
1983 * For user space, shared buffer is not available and hence the data
1984 * limit is immutable and is always the same as the underlying buffer
1985 * object limit itself.
1986 */
1987 return (uint32_t)__buflet_get_data_limit(buf);
1988 #endif /* !KERNEL */
1989 }
1990
1991 __attribute__((always_inline))
1992 static inline packet_trace_id_t
__packet_get_trace_id(const uint64_t ph)1993 __packet_get_trace_id(const uint64_t ph)
1994 {
1995 switch (SK_PTR_TYPE(ph)) {
1996 case NEXUS_META_TYPE_PACKET:
1997 return PKT_ADDR(ph)->pkt_trace_id;
1998 break;
1999 default:
2000 return 0;
2001 }
2002 }
2003
2004 __attribute__((always_inline))
2005 static inline void
__packet_set_trace_id(const uint64_t ph,packet_trace_id_t id)2006 __packet_set_trace_id(const uint64_t ph, packet_trace_id_t id)
2007 {
2008 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2009 PKT_ADDR(ph)->pkt_trace_id = id;
2010 }
2011
2012 __attribute__((always_inline))
2013 static inline void
__packet_trace_event(const uint64_t ph,uint32_t event)2014 __packet_trace_event(const uint64_t ph, uint32_t event)
2015 {
2016 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2017 #ifdef KERNEL
2018 #pragma unused(event, ph)
2019 KDBG(event, PKT_ADDR(ph)->pkt_trace_id);
2020 #else /* !KERNEL */
2021 kdebug_trace(event, PKT_ADDR(ph)->pkt_trace_id, 0, 0, 0);
2022 #endif /* !KERNEL */
2023 }
2024
2025 #ifdef KERNEL
2026 __attribute__((always_inline))
2027 static inline packet_trace_tag_t
__packet_get_trace_tag(const uint64_t ph)2028 __packet_get_trace_tag(const uint64_t ph)
2029 {
2030 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2031 return PKT_ADDR(ph)->pkt_trace_tag;
2032 }
2033
2034 __attribute__((always_inline))
2035 static inline void
__packet_set_trace_tag(const uint64_t ph,packet_trace_tag_t tag)2036 __packet_set_trace_tag(const uint64_t ph, packet_trace_tag_t tag)
2037 {
2038 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2039 PKT_ADDR(ph)->pkt_trace_tag = tag;
2040 }
2041
2042 static inline void
__packet_perform_tx_completion_callbacks(const kern_packet_t ph,ifnet_t ifp)2043 __packet_perform_tx_completion_callbacks(const kern_packet_t ph, ifnet_t ifp)
2044 {
2045 /*
2046 * NOTE: this function can be called with ifp as NULL.
2047 */
2048 uint64_t ts;
2049 kern_return_t tx_status;
2050 uintptr_t cb_arg, cb_data;
2051 struct __kern_packet *kpkt = SK_PTR_ADDR_KPKT(ph);
2052
2053 ASSERT((kpkt->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0);
2054 (void) __packet_get_tx_completion_status(ph, &tx_status);
2055 __packet_get_tx_completion_data(ph, &cb_arg, &cb_data);
2056 __packet_get_timestamp(ph, &ts, NULL);
2057 while (kpkt->pkt_tx_compl_callbacks != 0) {
2058 mbuf_tx_compl_func cb;
2059 uint32_t i;
2060
2061 i = ffs(kpkt->pkt_tx_compl_callbacks) - 1;
2062 kpkt->pkt_tx_compl_callbacks &= ~(1 << i);
2063 cb = m_get_tx_compl_callback(i);
2064 if (__probable(cb != NULL)) {
2065 cb(kpkt->pkt_tx_compl_context, ifp, ts, cb_arg, cb_data,
2066 tx_status);
2067 }
2068 }
2069 kpkt->pkt_pflags &= ~PKT_F_TX_COMPL_TS_REQ;
2070 }
2071
2072 static inline void *
__packet_get_priv(const kern_packet_t ph)2073 __packet_get_priv(const kern_packet_t ph)
2074 {
2075 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2076 return PKT_ADDR(ph)->pkt_priv;
2077 }
2078
2079 static inline void
__packet_set_priv(const uint64_t ph,void * priv)2080 __packet_set_priv(const uint64_t ph, void *priv)
2081 {
2082 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2083 PKT_ADDR(ph)->pkt_priv = priv;
2084 }
2085 #endif /* KERNEL */
2086
2087 #endif /* PRIVATE || BSD_KERNEL_PRIVATE */
2088 #endif /* !_SKYWALK_PACKET_COMMON_H_ */
2089