1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_PACKET_COMMON_H_
30 #define _SKYWALK_PACKET_COMMON_H_
31
32 #if defined(PRIVATE) || defined(BSD_KERNEL_PRIVATE)
33 /*
34 * Routines common to kernel and userland. This file is intended to
35 * be included by code implementing the packet APIs, in particular,
36 * the Skywalk kernel and libsyscall code.
37 */
38
39 #include <skywalk/os_packet_private.h>
40 #include <net/if_vlan_var.h>
41 #include <sys/errno.h>
42 #include <sys/kdebug.h>
43
44 #ifndef KERNEL
45 /*
46 * User.
47 */
48 #if !defined(LIBSYSCALL_INTERFACE)
49 #error "LIBSYSCALL_INTERFACE not defined"
50 #endif /* !LIBSYSCALL_INTERFACE */
51 #define QUM_ADDR(_ph) SK_PTR_ADDR_UQUM(_ph)
52 #define PKT_ADDR(_ph) SK_PTR_ADDR_UPKT(_ph)
53 #define BLT_ADDR(_bp) ((struct __user_buflet *)(uintptr_t)_bp)
54 #else /* KERNEL */
55 /*
56 * Kernel.
57 */
58 #include <skywalk/packet/packet_var.h>
59 #include <skywalk/packet/pbufpool_var.h>
60 #define QUM_ADDR(_ph) SK_PTR_ADDR_KQUM(_ph)
61 #define PKT_ADDR(_ph) SK_PTR_ADDR_KPKT(_ph)
62 #define BLT_ADDR(_bp) ((struct __kern_buflet *)(uintptr_t)_bp)
63 #define PKT_HAS_ATTACHED_MBUF(_ph) \
64 ((PKT_ADDR(_ph)->pkt_pflags & PKT_F_MBUF_DATA) != 0)
65 #endif /* KERNEL */
66
67 /*
68 * Common.
69 */
70 #if (DEBUG || DEVELOPMENT)
71 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) do { \
72 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type) || \
73 SK_PTR_SUBTYPE(_ph) != (uint64_t)(_subtype))) { \
74 pkt_subtype_assert_fail(_ph, _type, _subtype); \
75 /* NOTREACHED */ \
76 __builtin_unreachable(); \
77 } \
78 } while (0)
79
80 #define PKT_TYPE_ASSERT(_ph, _type) do { \
81 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type))) { \
82 pkt_type_assert_fail(_ph, _type); \
83 /* NOTREACHED */ \
84 __builtin_unreachable(); \
85 } \
86 } while (0)
87 #else /* !DEBUG && !DEVELOPMENT */
88 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) ((void)0)
89 #define PKT_TYPE_ASSERT(_ph, _type) ((void)0)
90 #endif /* !DEBUG && !DEVELOPMENT */
91
92 #define QUM_GET_NEXT_BUFLET(_qum, _pbuf, _buf) do { \
93 ASSERT((_pbuf) == NULL || (_pbuf) == (_qum)->qum_buf); \
94 (_buf) = (((_pbuf) == NULL) ? (_qum)->qum_buf : NULL); \
95 } while (0)
96
97 #define PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf) do { \
98 if (__improbable((_bcnt) == 0)) { \
99 (_buf) = NULL; \
100 break; \
101 } \
102 if (__probable((_pkt)->pkt_qum_buf.buf_addr != 0)) { \
103 (_buf) = &(_pkt)->pkt_qum_buf; \
104 } else { \
105 (_buf) = __unsafe_forge_single(struct __kern_buflet *, \
106 __DECONST(void *, (_pkt)->pkt_qum_buf.buf_nbft_addr));\
107 } \
108 } while (0)
109
110 #define _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
111 if ((_pbuf) == NULL) { \
112 PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf); \
113 } else { \
114 (_buf) = __unsafe_forge_single(struct __kern_buflet *, \
115 __DECONST(void *, (_pbuf)->buf_nbft_addr)); \
116 } \
117 } while (0)
118
119 #ifndef KERNEL
120 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
121 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
122 } while (0)
123 #else /* KERNEL */
124 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
125 ASSERT(((_bcnt) >= 1) || ((_pbuf) == NULL)); \
126 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
127 } while (0)
128 #endif /* KERNEL */
129
130 #ifdef KERNEL
131 #define PKT_COMPOSE_NX_PORT_ID(_nx_port, _gencnt) \
132 ((uint32_t)((_gencnt & 0xffff) << 16) | (_nx_port & 0xffff))
133
134 #define PKT_DECOMPOSE_NX_PORT_ID(_nx_port_id, _nx_port, _gencnt) do { \
135 _nx_port = _nx_port_id & 0xffff; \
136 _gencnt = (_nx_port_id >> 16) & 0xffff; \
137 } while (0)
138 #endif /* KERNEL */
139
140 __attribute__((always_inline))
141 static inline int
__packet_set_headroom(const uint64_t ph,const uint8_t headroom)142 __packet_set_headroom(const uint64_t ph, const uint8_t headroom)
143 {
144 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
145 if (__probable(headroom < PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
146 PKT_ADDR(ph)->pkt_headroom = headroom;
147 return 0;
148 }
149 return ERANGE;
150 }
151
152 __attribute__((always_inline))
153 static inline uint8_t
__packet_get_headroom(const uint64_t ph)154 __packet_get_headroom(const uint64_t ph)
155 {
156 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
157 return PKT_ADDR(ph)->pkt_headroom;
158 }
159
160 __attribute__((always_inline))
161 static inline int
__packet_set_link_header_length(const uint64_t ph,const uint8_t len)162 __packet_set_link_header_length(const uint64_t ph, const uint8_t len)
163 {
164 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
165 if (__probable(len <= PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
166 PKT_ADDR(ph)->pkt_l2_len = len;
167 return 0;
168 }
169 return ERANGE;
170 }
171
172 __attribute__((always_inline))
173 static inline uint8_t
__packet_get_link_header_length(const uint64_t ph)174 __packet_get_link_header_length(const uint64_t ph)
175 {
176 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
177 return PKT_ADDR(ph)->pkt_l2_len;
178 }
179
180 __attribute__((always_inline))
181 static inline int
__packet_set_link_broadcast(const uint64_t ph)182 __packet_set_link_broadcast(const uint64_t ph)
183 {
184 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
185 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_BCAST;
186 return 0;
187 }
188
189 __attribute__((always_inline))
190 static inline boolean_t
__packet_get_link_broadcast(const uint64_t ph)191 __packet_get_link_broadcast(const uint64_t ph)
192 {
193 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
194 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_BCAST) != 0;
195 }
196
197 __attribute__((always_inline))
198 static inline int
__packet_set_link_multicast(const uint64_t ph)199 __packet_set_link_multicast(const uint64_t ph)
200 {
201 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
202 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_MCAST;
203 return 0;
204 }
205
206 __attribute__((always_inline))
207 static inline boolean_t
__packet_get_link_multicast(const uint64_t ph)208 __packet_get_link_multicast(const uint64_t ph)
209 {
210 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
211 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_MCAST) != 0;
212 }
213
214 __attribute__((always_inline))
215 static inline int
__packet_set_link_ethfcs(const uint64_t ph)216 __packet_set_link_ethfcs(const uint64_t ph)
217 {
218 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
219 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_ETHFCS;
220 return 0;
221 }
222
223 __attribute__((always_inline))
224 static inline boolean_t
__packet_get_link_ethfcs(const uint64_t ph)225 __packet_get_link_ethfcs(const uint64_t ph)
226 {
227 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
228 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_ETHFCS) != 0;
229 }
230
231 __attribute__((always_inline))
232 static inline int
__packet_set_transport_traffic_background(const uint64_t ph)233 __packet_set_transport_traffic_background(const uint64_t ph)
234 {
235 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
236 PKT_ADDR(ph)->pkt_pflags |= PKT_F_BACKGROUND;
237 return 0;
238 }
239
240 __attribute__((always_inline))
241 static inline boolean_t
__packet_get_transport_traffic_background(const uint64_t ph)242 __packet_get_transport_traffic_background(const uint64_t ph)
243 {
244 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
245 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_BACKGROUND) != 0;
246 }
247
248 __attribute__((always_inline))
249 static inline int
__packet_set_transport_traffic_realtime(const uint64_t ph)250 __packet_set_transport_traffic_realtime(const uint64_t ph)
251 {
252 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
253 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REALTIME;
254 return 0;
255 }
256
257 __attribute__((always_inline))
258 static inline boolean_t
__packet_get_transport_traffic_realtime(const uint64_t ph)259 __packet_get_transport_traffic_realtime(const uint64_t ph)
260 {
261 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
262 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REALTIME) != 0;
263 }
264
265 __attribute__((always_inline))
266 static inline int
__packet_set_transport_retransmit(const uint64_t ph)267 __packet_set_transport_retransmit(const uint64_t ph)
268 {
269 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
270 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REXMT;
271 return 0;
272 }
273
274 __attribute__((always_inline))
275 static inline boolean_t
__packet_get_transport_retransmit(const uint64_t ph)276 __packet_get_transport_retransmit(const uint64_t ph)
277 {
278 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
279 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REXMT) != 0;
280 }
281
282 __attribute__((always_inline))
283 static inline int
__packet_set_transport_last_packet(const uint64_t ph)284 __packet_set_transport_last_packet(const uint64_t ph)
285 {
286 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
287 PKT_ADDR(ph)->pkt_pflags |= PKT_F_LAST_PKT;
288 return 0;
289 }
290
291 __attribute__((always_inline))
292 static inline int
__packet_set_group_start(const uint64_t ph)293 __packet_set_group_start(const uint64_t ph)
294 {
295 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
296 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_START;
297 return 0;
298 }
299
300 __attribute__((always_inline))
301 static inline boolean_t
__packet_get_group_start(const uint64_t ph)302 __packet_get_group_start(const uint64_t ph)
303 {
304 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
305 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_START) != 0;
306 }
307
308 __attribute__((always_inline))
309 static inline int
__packet_set_group_end(const uint64_t ph)310 __packet_set_group_end(const uint64_t ph)
311 {
312 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
313 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_END;
314 return 0;
315 }
316
317 __attribute__((always_inline))
318 static inline boolean_t
__packet_get_group_end(const uint64_t ph)319 __packet_get_group_end(const uint64_t ph)
320 {
321 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
322 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_END) != 0;
323 }
324
325 __attribute__((always_inline))
326 static inline errno_t
__packet_get_expire_time(const uint64_t ph,uint64_t * ts)327 __packet_get_expire_time(const uint64_t ph, uint64_t *ts)
328 {
329 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
330 #ifdef KERNEL
331 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
332 #else /* !KERNEL */
333 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
334 #endif /* !KERNEL */
335 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) == 0) {
336 return ENOENT;
337 }
338 if (ts == NULL) {
339 return EINVAL;
340 }
341 *ts = po->__po_expire_ts;
342 return 0;
343 }
344
345 __attribute__((always_inline))
346 static inline errno_t
__packet_set_expire_time(const uint64_t ph,const uint64_t ts)347 __packet_set_expire_time(const uint64_t ph, const uint64_t ts)
348 {
349 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
350 #ifdef KERNEL
351 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
352 #else /* !KERNEL */
353 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
354 #endif /* !KERNEL */
355 if (ts != 0) {
356 po->__po_expire_ts = ts;
357 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXPIRE_TS;
358 } else {
359 po->__po_expire_ts = 0;
360 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXPIRE_TS;
361 }
362 return 0;
363 }
364
365 __attribute__((always_inline))
366 static inline errno_t
__packet_get_expiry_action(const uint64_t ph,packet_expiry_action_t * pea)367 __packet_get_expiry_action(const uint64_t ph, packet_expiry_action_t *pea)
368 {
369 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
370 #ifdef KERNEL
371 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
372 #else /* !KERNEL */
373 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
374 #endif /* !KERNEL */
375 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXP_ACTION) == 0) {
376 return ENOENT;
377 }
378 if (pea == NULL) {
379 return EINVAL;
380 }
381 *pea = po->__po_expiry_action;
382 return 0;
383 }
384
385 __attribute__((always_inline))
386 static inline errno_t
__packet_set_expiry_action(const uint64_t ph,packet_expiry_action_t pea)387 __packet_set_expiry_action(const uint64_t ph, packet_expiry_action_t pea)
388 {
389 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
390 #ifdef KERNEL
391 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
392 #else /* !KERNEL */
393 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
394 #endif /* !KERNEL */
395 if (pea != PACKET_EXPIRY_ACTION_NONE) {
396 po->__po_expiry_action = (uint8_t)pea;
397 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXP_ACTION;
398 } else {
399 po->__po_expiry_action = 0;
400 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXP_ACTION;
401 }
402 return 0;
403 }
404
405 __attribute__((always_inline))
406 static inline errno_t
__packet_opt_get_token(const struct __packet_opt * po,void * __sized_by (* len)token,uint16_t * len,uint8_t * type)407 __packet_opt_get_token(const struct __packet_opt *po,
408 void *__sized_by(*len)token,
409 uint16_t *len, uint8_t *type)
410 {
411 uint16_t tlen = po->__po_token_len;
412 uint8_t ttype;
413
414 if (token == NULL || len == NULL || type == NULL || tlen > *len) {
415 return EINVAL;
416 }
417 ttype = (uint8_t)po->__po_token_type;
418
419 ASSERT(tlen <= PKT_OPT_MAX_TOKEN_SIZE);
420 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
421 bcopy(po->__po_token, token, tlen);
422 /*
423 * -fbounds-safety: Updating *len should be fine because at this point
424 * we know tlen is less than or equal to *len (check the first if
425 * statement in this function)
426 */
427 *len = tlen;
428 *type = ttype;
429 return 0;
430 }
431
432 __attribute__((always_inline))
433 static inline errno_t
__packet_get_token(const uint64_t ph,void * __sized_by (* len)token,uint16_t * len)434 __packet_get_token(const uint64_t ph,
435 void *__sized_by(*len)token, uint16_t *len)
436 {
437 #ifdef KERNEL
438 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
439 #else /* !KERNEL */
440 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
441 #endif /* !KERNEL */
442 uint8_t type;
443 errno_t err;
444
445 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
446 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
447 return ENOENT;
448 }
449 err = __packet_opt_get_token(po, token, len, &type);
450 if ((err == 0) && (type != PKT_OPT_TOKEN_TYPE_OPAQUE)) {
451 err = ENOENT;
452 }
453 return err;
454 }
455
456 __attribute__((always_inline))
457 static inline errno_t
__packet_opt_set_token(struct __packet_opt * po,const void * __sized_by (PKT_OPT_MAX_TOKEN_SIZE)token,const uint16_t len,const uint8_t type,volatile uint64_t * pflags)458 __packet_opt_set_token(struct __packet_opt *po,
459 const void *__sized_by(PKT_OPT_MAX_TOKEN_SIZE)token,
460 const uint16_t len, const uint8_t type, volatile uint64_t *pflags)
461 {
462 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
463 if (len != 0) {
464 if (token == NULL || len > PKT_OPT_MAX_TOKEN_SIZE ||
465 type == 0) {
466 return EINVAL;
467 }
468 if (__probable(IS_P2ALIGNED(token, 8))) {
469 uint64_t *token64 = __DECONST(void *, token);
470 po->__po_token_data[0] = *token64;
471 po->__po_token_data[1] = *(token64 + 1);
472 } else {
473 bcopy(token, po->__po_token, len);
474 }
475 po->__po_token_len = len;
476 po->__po_token_type = type;
477 *pflags |= PKT_F_OPT_TOKEN;
478 } else {
479 _CASSERT(sizeof(po->__po_token_data[0]) == 8);
480 _CASSERT(sizeof(po->__po_token_data[1]) == 8);
481 _CASSERT(sizeof(po->__po_token) == 16);
482 po->__po_token_data[0] = 0;
483 po->__po_token_data[1] = 0;
484 po->__po_token_len = 0;
485 po->__po_token_type = 0;
486 *pflags &= ~PKT_F_OPT_TOKEN;
487 }
488 return 0;
489 }
490
491 __attribute__((always_inline))
492 static inline void
__packet_set_tx_timestamp(const uint64_t ph,const uint64_t ts)493 __packet_set_tx_timestamp(const uint64_t ph, const uint64_t ts)
494 {
495 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
496 #ifdef KERNEL
497 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
498 #else /* !KERNEL */
499 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
500 #endif /* !KERNEL */
501
502 if (po != NULL) {
503 po->__po_pkt_tx_time = ts;
504 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_TX_TIMESTAMP;
505 }
506 }
507
508 __attribute__((always_inline))
509 static inline uint64_t
__packet_get_tx_timestamp(const uint64_t ph)510 __packet_get_tx_timestamp(const uint64_t ph)
511 {
512 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
513 #ifdef KERNEL
514 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
515 #else /* !KERNEL */
516 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
517 #endif /* !KERNEL */
518 if (po == NULL || (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TX_TIMESTAMP) == 0) {
519 return 0;
520 }
521
522 return po->__po_pkt_tx_time;
523 }
524
525 __attribute__((always_inline))
526 static inline errno_t
__packet_set_token(const uint64_t ph,const void * __sized_by (len)token,const uint16_t len)527 __packet_set_token(const uint64_t ph,
528 const void *__sized_by(len)token, const uint16_t len)
529 {
530 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
531 #ifdef KERNEL
532 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, token, len,
533 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
534 #else /* !KERNEL */
535 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, token, len,
536 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
537 #endif /* !KERNEL */
538 }
539
540 __attribute__((always_inline))
541 static inline errno_t
__packet_get_packetid(const uint64_t ph,packet_id_t * pktid)542 __packet_get_packetid(const uint64_t ph, packet_id_t *pktid)
543 {
544 #ifdef KERNEL
545 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
546 #else /* !KERNEL */
547 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
548 #endif /* !KERNEL */
549 uint16_t len = sizeof(packet_id_t);
550 uint8_t type;
551 errno_t err;
552
553
554 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
555 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
556 return ENOENT;
557 }
558 err = __packet_opt_get_token(po, (packet_id_t * __header_indexable)pktid,
559 &len, &type);
560 if ((err == 0) && ((type != PKT_OPT_TOKEN_TYPE_PACKET_ID) ||
561 (len != sizeof(packet_id_t)))) {
562 err = ENOENT;
563 }
564 return err;
565 }
566
567 __attribute__((always_inline))
568 static inline errno_t
__packet_set_packetid(const uint64_t ph,const packet_id_t * pktid)569 __packet_set_packetid(const uint64_t ph, const packet_id_t *pktid)
570 {
571 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
572 #ifdef KERNEL
573 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, pktid,
574 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
575 &PKT_ADDR(ph)->pkt_pflags);
576 #else /* !KERNEL */
577 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, pktid,
578 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
579 &PKT_ADDR(ph)->pkt_pflags);
580 #endif /* !KERNEL */
581 }
582
583 __attribute__((always_inline))
584 static inline errno_t
__packet_get_vlan_tag(const uint64_t ph,uint16_t * vlan_tag,boolean_t * tag_in_pkt)585 __packet_get_vlan_tag(const uint64_t ph, uint16_t *vlan_tag,
586 boolean_t *tag_in_pkt)
587 {
588 #ifdef KERNEL
589 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
590 #else /* !KERNEL */
591 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
592 #endif /* !KERNEL */
593 uint64_t pflags;
594
595 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
596 pflags = PKT_ADDR(ph)->pkt_pflags;
597 if ((pflags & PKT_F_OPT_VLTAG) == 0) {
598 return ENOENT;
599 }
600 if (vlan_tag != NULL) {
601 *vlan_tag = po->__po_vlan_tag;
602 }
603 if (tag_in_pkt != NULL) {
604 *tag_in_pkt = ((pflags & PKT_F_OPT_VLTAG_IN_PKT) != 0);
605 }
606 return 0;
607 }
608
609 __attribute__((always_inline))
610 static inline errno_t
__packet_set_vlan_tag(const uint64_t ph,const uint16_t vlan_tag,const boolean_t tag_in_pkt)611 __packet_set_vlan_tag(const uint64_t ph, const uint16_t vlan_tag,
612 const boolean_t tag_in_pkt)
613 {
614 #ifdef KERNEL
615 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
616 #else /* !KERNEL */
617 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
618 #endif /* !KERNEL */
619
620 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
621 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG;
622 po->__po_vlan_tag = vlan_tag;
623
624 if (tag_in_pkt) {
625 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG_IN_PKT;
626 }
627 return 0;
628 }
629
630 __attribute__((always_inline))
631 static inline uint16_t
__packet_get_vlan_id(const uint16_t vlan_tag)632 __packet_get_vlan_id(const uint16_t vlan_tag)
633 {
634 return EVL_VLANOFTAG(vlan_tag);
635 }
636
637 __attribute__((always_inline))
638 static inline uint8_t
__packet_get_vlan_priority(const uint16_t vlan_tag)639 __packet_get_vlan_priority(const uint16_t vlan_tag)
640 {
641 return EVL_PRIOFTAG(vlan_tag);
642 }
643
644 __attribute__((always_inline))
645 static inline errno_t
__packet_get_app_metadata(const uint64_t ph,packet_app_metadata_type_t * app_type,uint8_t * app_metadata)646 __packet_get_app_metadata(const uint64_t ph,
647 packet_app_metadata_type_t *app_type, uint8_t *app_metadata)
648 {
649 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
650 if (app_type == NULL || app_metadata == NULL) {
651 return EINVAL;
652 }
653 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_APP_METADATA) == 0) {
654 return ENOENT;
655 }
656 #ifdef KERNEL
657 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
658 #else /* !KERNEL */
659 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
660 #endif /* !KERNEL */
661 if (po->__po_app_type == PACKET_APP_METADATA_TYPE_UNSPECIFIED) {
662 return ENOENT;
663 }
664 *app_type = po->__po_app_type;
665 *app_metadata = po->__po_app_metadata;
666 return 0;
667 }
668
669 __attribute__((always_inline))
670 static inline errno_t
__packet_set_app_metadata(const uint64_t ph,const packet_app_metadata_type_t app_type,const uint8_t app_metadata)671 __packet_set_app_metadata(const uint64_t ph,
672 const packet_app_metadata_type_t app_type, const uint8_t app_metadata)
673 {
674 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
675 #ifdef KERNEL
676 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
677 #else /* !KERNEL */
678 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
679 #endif /* !KERNEL */
680 if (app_type < PACKET_APP_METADATA_TYPE_MIN ||
681 app_type > PACKET_APP_METADATA_TYPE_MAX) {
682 po->__po_app_type = PACKET_APP_METADATA_TYPE_UNSPECIFIED;
683 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_APP_METADATA;
684 return EINVAL;
685 }
686 po->__po_app_type = app_type;
687 po->__po_app_metadata = app_metadata;
688 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_APP_METADATA;
689 return 0;
690 }
691
692 #ifdef KERNEL
693 __attribute__((always_inline))
694 static inline void
__packet_set_wake_flag(const uint64_t ph)695 __packet_set_wake_flag(const uint64_t ph)
696 {
697 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
698 PKT_ADDR(ph)->pkt_pflags |= PKT_F_WAKE_PKT;
699 }
700 #endif
701
702 __attribute__((always_inline))
703 static inline boolean_t
__packet_get_wake_flag(const uint64_t ph)704 __packet_get_wake_flag(const uint64_t ph)
705 {
706 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_WAKE_PKT) != 0;
707 }
708
709 __attribute__((always_inline))
710 static inline void
__packet_set_keep_alive(const uint64_t ph,const boolean_t is_keep_alive)711 __packet_set_keep_alive(const uint64_t ph, const boolean_t is_keep_alive)
712 {
713 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
714 if (is_keep_alive) {
715 PKT_ADDR(ph)->pkt_pflags |= PKT_F_KEEPALIVE;
716 } else {
717 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_KEEPALIVE;
718 }
719 }
720
721 __attribute__((always_inline))
722 static inline boolean_t
__packet_get_keep_alive(const uint64_t ph)723 __packet_get_keep_alive(const uint64_t ph)
724 {
725 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_KEEPALIVE) != 0;
726 }
727
728 __attribute__((always_inline))
729 static inline boolean_t
__packet_get_truncated(const uint64_t ph)730 __packet_get_truncated(const uint64_t ph)
731 {
732 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
733 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_TRUNCATED) != 0;
734 }
735
736 #ifdef KERNEL
737 __attribute__((always_inline))
738 static inline boolean_t
__packet_get_transport_new_flow(const uint64_t ph)739 __packet_get_transport_new_flow(const uint64_t ph)
740 {
741 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
742 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_NEW_FLOW) != 0;
743 }
744
745 __attribute__((always_inline))
746 static inline boolean_t
__packet_get_transport_last_packet(const uint64_t ph)747 __packet_get_transport_last_packet(const uint64_t ph)
748 {
749 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
750 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_LAST_PKT) != 0;
751 }
752
753 __attribute__((always_inline))
754 static inline boolean_t
__packet_get_l4s_flag(const uint64_t ph)755 __packet_get_l4s_flag(const uint64_t ph)
756 {
757 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
758 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_L4S) != 0;
759 }
760 #endif /* KERNEL */
761
762 __attribute__((always_inline))
763 static inline void
__packet_set_l4s_flag(const uint64_t ph)764 __packet_set_l4s_flag(const uint64_t ph)
765 {
766 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
767 PKT_ADDR(ph)->pkt_pflags |= PKT_F_L4S;
768 }
769
770 __attribute__((always_inline))
771 static inline int
__packet_set_service_class(const uint64_t ph,const uint32_t sc)772 __packet_set_service_class(const uint64_t ph, const uint32_t sc)
773 {
774 int err = 0;
775
776 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
777
778 switch (sc) {
779 case PKT_SC_BE:
780 case PKT_SC_BK_SYS:
781 case PKT_SC_BK:
782 case PKT_SC_RD:
783 case PKT_SC_OAM:
784 case PKT_SC_AV:
785 case PKT_SC_RV:
786 case PKT_SC_VI:
787 case PKT_SC_SIG:
788 case PKT_SC_VO:
789 case PKT_SC_CTL:
790 QUM_ADDR(ph)->qum_svc_class = sc;
791 break;
792
793 default:
794 err = EINVAL;
795 break;
796 }
797
798 return err;
799 }
800
801 __attribute__((always_inline))
802 static inline uint32_t
__packet_get_service_class(const uint64_t ph)803 __packet_get_service_class(const uint64_t ph)
804 {
805 uint32_t sc;
806
807 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
808
809 switch (QUM_ADDR(ph)->qum_svc_class) {
810 case PKT_SC_BE: /* most likely best effort */
811 case PKT_SC_BK_SYS:
812 case PKT_SC_BK:
813 case PKT_SC_RD:
814 case PKT_SC_OAM:
815 case PKT_SC_AV:
816 case PKT_SC_RV:
817 case PKT_SC_VI:
818 case PKT_SC_SIG:
819 case PKT_SC_VO:
820 case PKT_SC_CTL:
821 sc = QUM_ADDR(ph)->qum_svc_class;
822 break;
823
824 default:
825 sc = PKT_SC_BE;
826 break;
827 }
828
829 return sc;
830 }
831
832 __attribute__((always_inline))
833 static inline errno_t
__packet_set_comp_gencnt(const uint64_t ph,const uint32_t gencnt)834 __packet_set_comp_gencnt(const uint64_t ph, const uint32_t gencnt)
835 {
836 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
837 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
838
839 PKT_ADDR(ph)->pkt_comp_gencnt = gencnt;
840
841 return 0;
842 }
843
844 __attribute__((always_inline))
845 static inline errno_t
__packet_get_comp_gencnt(const uint64_t ph,uint32_t * pgencnt)846 __packet_get_comp_gencnt(const uint64_t ph, uint32_t *pgencnt)
847 {
848 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
849 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
850
851 if (pgencnt == NULL) {
852 return EINVAL;
853 }
854
855 if (PKT_ADDR(ph)->pkt_comp_gencnt == 0) {
856 return ENOENT;
857 }
858
859 *pgencnt = PKT_ADDR(ph)->pkt_comp_gencnt;
860 return 0;
861 }
862
863 __attribute__((always_inline))
864 static inline int
__packet_set_traffic_class(const uint64_t ph,const uint32_t tc)865 __packet_set_traffic_class(const uint64_t ph, const uint32_t tc)
866 {
867 uint32_t val = PKT_TC2SCVAL(tc); /* just the val portion */
868 uint32_t sc;
869
870 switch (val) {
871 case PKT_SCVAL_BK_SYS:
872 sc = PKT_SC_BK_SYS;
873 break;
874 case PKT_SCVAL_BK:
875 sc = PKT_SC_BK;
876 break;
877 case PKT_SCVAL_BE:
878 sc = PKT_SC_BE;
879 break;
880 case PKT_SCVAL_RD:
881 sc = PKT_SC_RD;
882 break;
883 case PKT_SCVAL_OAM:
884 sc = PKT_SC_OAM;
885 break;
886 case PKT_SCVAL_AV:
887 sc = PKT_SC_AV;
888 break;
889 case PKT_SCVAL_RV:
890 sc = PKT_SC_RV;
891 break;
892 case PKT_SCVAL_VI:
893 sc = PKT_SC_VI;
894 break;
895 case PKT_SCVAL_SIG:
896 sc = PKT_SC_SIG;
897 break;
898 case PKT_SCVAL_VO:
899 sc = PKT_SC_VO;
900 break;
901 case PKT_SCVAL_CTL:
902 sc = PKT_SC_CTL;
903 break;
904 default:
905 sc = PKT_SC_BE;
906 break;
907 }
908
909 return __packet_set_service_class(ph, sc);
910 }
911
912 __attribute__((always_inline))
913 static inline uint32_t
__packet_get_traffic_class(const uint64_t ph)914 __packet_get_traffic_class(const uint64_t ph)
915 {
916 return PKT_SC2TC(__packet_get_service_class(ph));
917 }
918
919 __attribute__((always_inline))
920 static inline int
__packet_set_inet_checksum(const uint64_t ph,const packet_csum_flags_t flags,const uint16_t start,const uint16_t stuff_val,boolean_t tx)921 __packet_set_inet_checksum(const uint64_t ph, const packet_csum_flags_t flags,
922 const uint16_t start, const uint16_t stuff_val, boolean_t tx)
923 {
924 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
925
926 PKT_ADDR(ph)->pkt_csum_flags = flags & PACKET_CSUM_FLAGS;
927
928 if (tx) {
929 PKT_ADDR(ph)->pkt_csum_tx_start_off = start;
930 PKT_ADDR(ph)->pkt_csum_tx_stuff_off = stuff_val;
931 } else {
932 PKT_ADDR(ph)->pkt_csum_rx_start_off = start;
933 PKT_ADDR(ph)->pkt_csum_rx_value = stuff_val;
934 }
935 return 0;
936 }
937
938 __attribute__((always_inline))
939 static inline void
__packet_add_inet_csum_flags(const uint64_t ph,const packet_csum_flags_t flags)940 __packet_add_inet_csum_flags(const uint64_t ph, const packet_csum_flags_t flags)
941 {
942 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
943
944 PKT_ADDR(ph)->pkt_csum_flags |= flags & PACKET_CSUM_FLAGS;
945 }
946
947 __attribute__((always_inline))
948 static inline packet_csum_flags_t
__packet_get_inet_checksum(const uint64_t ph,uint16_t * start,uint16_t * stuff_val,boolean_t tx)949 __packet_get_inet_checksum(const uint64_t ph, uint16_t *start,
950 uint16_t *stuff_val, boolean_t tx)
951 {
952 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
953
954 if (tx) {
955 if (__probable(start != NULL)) {
956 *start = PKT_ADDR(ph)->pkt_csum_tx_start_off;
957 }
958 if (__probable(stuff_val != NULL)) {
959 *stuff_val = PKT_ADDR(ph)->pkt_csum_tx_stuff_off;
960 }
961 } else {
962 if (__probable(start != NULL)) {
963 *start = PKT_ADDR(ph)->pkt_csum_rx_start_off;
964 }
965 if (__probable(stuff_val != NULL)) {
966 *stuff_val = PKT_ADDR(ph)->pkt_csum_rx_value;
967 }
968 }
969 return PKT_ADDR(ph)->pkt_csum_flags & PACKET_CSUM_FLAGS;
970 }
971
972 __attribute__((always_inline))
973 static inline void
__packet_set_flow_uuid(const uint64_t ph,const uuid_t flow_uuid)974 __packet_set_flow_uuid(const uint64_t ph, const uuid_t flow_uuid)
975 {
976 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
977
978 /*
979 * Anticipate a nicely (8-bytes) aligned UUID from caller;
980 * the one in qum_flow_id is always 8-byte aligned.
981 */
982 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
983 const uint64_t *id_64 = (const uint64_t *)(const void *)flow_uuid;
984 q->__q_flow_id_val64[0] = id_64[0];
985 q->__q_flow_id_val64[1] = id_64[1];
986 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
987 const uint32_t *id_32 = (const uint32_t *)(const void *)flow_uuid;
988 q->__q_flow_id_val32[0] = id_32[0];
989 q->__q_flow_id_val32[1] = id_32[1];
990 q->__q_flow_id_val32[2] = id_32[2];
991 q->__q_flow_id_val32[3] = id_32[3];
992 } else {
993 bcopy(flow_uuid, q->__q_flow_id, sizeof(uuid_t));
994 }
995 }
996
997 __attribute__((always_inline))
998 static inline void
__packet_get_flow_uuid(const uint64_t ph,uuid_t flow_uuid)999 __packet_get_flow_uuid(const uint64_t ph, uuid_t flow_uuid)
1000 {
1001 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
1002
1003 /*
1004 * Anticipate a nicely (8-bytes) aligned UUID from caller;
1005 * the one in qum_flow_id is always 8-byte aligned.
1006 */
1007 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
1008 uint64_t *id_64 = (uint64_t *)(void *)flow_uuid;
1009 id_64[0] = q->__q_flow_id_val64[0];
1010 id_64[1] = q->__q_flow_id_val64[1];
1011 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
1012 uint32_t *id_32 = (uint32_t *)(void *)flow_uuid;
1013 id_32[0] = q->__q_flow_id_val32[0];
1014 id_32[1] = q->__q_flow_id_val32[1];
1015 id_32[2] = q->__q_flow_id_val32[2];
1016 id_32[3] = q->__q_flow_id_val32[3];
1017 } else {
1018 bcopy(q->__q_flow_id, flow_uuid, sizeof(uuid_t));
1019 }
1020 }
1021
1022 __attribute__((always_inline))
1023 static inline void
__packet_clear_flow_uuid(const uint64_t ph)1024 __packet_clear_flow_uuid(const uint64_t ph)
1025 {
1026 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
1027 q->__q_flow_id_val64[0] = 0;
1028 q->__q_flow_id_val64[1] = 0;
1029 }
1030
1031 __attribute__((always_inline))
1032 static inline uint8_t
__packet_get_aggregation_type(const uint64_t ph)1033 __packet_get_aggregation_type(const uint64_t ph)
1034 {
1035 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_aggr_type == sizeof(uint8_t)));
1036 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1037
1038 return PKT_ADDR(ph)->pkt_aggr_type;
1039 }
1040
1041 __attribute__((always_inline))
1042 static inline uint32_t
__packet_get_data_length(const uint64_t ph)1043 __packet_get_data_length(const uint64_t ph)
1044 {
1045 return QUM_ADDR(ph)->qum_len;
1046 }
1047
1048 #ifdef KERNEL
1049 /*
1050 * This handles truncated packets used in compat Tx and Rx classification.
1051 */
1052 __attribute__((always_inline))
1053 static inline uint32_t
__packet_get_real_data_length(const struct __kern_packet * pkt)1054 __packet_get_real_data_length(const struct __kern_packet *pkt)
1055 {
1056 uint32_t pkt_len;
1057
1058 if (pkt->pkt_pflags & PKT_F_TRUNCATED) {
1059 struct __kern_buflet *bft;
1060
1061 bft = kern_packet_get_next_buflet(SK_PKT2PH(pkt), NULL);
1062 pkt_len = kern_buflet_get_data_length(bft);
1063 } else {
1064 pkt_len = pkt->pkt_length;
1065 }
1066 return pkt_len;
1067 }
1068 #endif /* KERNEL */
1069
1070 __attribute__((always_inline))
1071 static inline uint16_t
__packet_get_buflet_count(const uint64_t ph)1072 __packet_get_buflet_count(const uint64_t ph)
1073 {
1074 uint16_t bcnt = 0;
1075
1076 switch (SK_PTR_TYPE(ph)) {
1077 case NEXUS_META_TYPE_PACKET:
1078 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1079 #ifdef KERNEL
1080 VERIFY(bcnt != 0 ||
1081 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1082 #else /* !KERNEL */
1083 /*
1084 * Handle the case where the metadata region gets
1085 * redirected to anonymous zero-filled pages at
1086 * defunct time. There's always 1 buflet in the
1087 * packet metadata, so pretend that's the count.
1088 */
1089 if (__improbable(bcnt == 0)) {
1090 bcnt = 1;
1091 }
1092 #endif /* !KERNEL */
1093 break;
1094 case NEXUS_META_TYPE_QUANTUM:
1095 bcnt = 1;
1096 break;
1097 default:
1098 #ifdef KERNEL
1099 VERIFY(0);
1100 /* NOTREACHED */
1101 __builtin_unreachable();
1102 #endif /* KERNEL */
1103 break;
1104 }
1105 return bcnt;
1106 }
1107
1108 __attribute__((always_inline))
1109 static inline int
__packet_add_buflet(const uint64_t ph,const void * bprev0,const void * bnew0)1110 __packet_add_buflet(const uint64_t ph, const void *bprev0, const void *bnew0)
1111 {
1112 uint16_t bcnt;
1113
1114 #ifdef KERNEL
1115 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1116 kern_buflet_t bnew = __DECONST(kern_buflet_t, bnew0);
1117
1118 VERIFY(PKT_ADDR(ph) && bnew && (bnew != bprev));
1119 VERIFY(PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1120 #else /* !KERNEL */
1121 buflet_t bprev = __DECONST(buflet_t, bprev0);
1122 buflet_t bnew = __DECONST(buflet_t, bnew0);
1123
1124 if (__improbable(!PKT_ADDR(ph) || !bnew || (bnew == bprev))) {
1125 return EINVAL;
1126 }
1127 #endif /* !KERNEL */
1128
1129 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1130 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1131
1132 #ifdef KERNEL
1133 VERIFY((bprev != NULL || bcnt == 0) &&
1134 (bcnt < PKT_ADDR(ph)->pkt_bufs_max));
1135 #else /* !KERNEL */
1136 if (__improbable(bcnt >= PKT_ADDR(ph)->pkt_bufs_max) ||
1137 (bprev == NULL && bcnt != 0)) {
1138 return EINVAL;
1139 }
1140 #endif /* !KERNEL */
1141
1142 #ifdef KERNEL
1143 #if DEVELOPMENT || DEBUG
1144 /* check if bprev is the last buflet in the chain */
1145 struct __kern_buflet *__single pbft, *__single kbft;
1146 int n = bcnt;
1147
1148 PKT_GET_FIRST_BUFLET(PKT_ADDR(ph), bcnt, pbft);
1149 kbft = pbft;
1150
1151 while ((kbft != NULL) && n--) {
1152 pbft = kbft;
1153 kbft = __unsafe_forge_single(struct __kern_buflet *,
1154 __DECONST(struct __kern_buflet *, kbft->buf_nbft_addr));
1155 }
1156 ASSERT(n == 0);
1157 ASSERT(bprev == pbft);
1158 #endif /* DEVELOPMENT || DEBUG */
1159 #endif /* KERNEL */
1160
1161 if (bprev == NULL) {
1162 bprev = &PKT_ADDR(ph)->pkt_qum_buf;
1163 }
1164 #ifdef KERNEL
1165 KBUF_LINK(bprev, bnew);
1166 #else /* !KERNEL */
1167 UBUF_LINK(bprev, bnew);
1168 #endif /* !KERNEL */
1169
1170 *(uint16_t *)(uintptr_t)&PKT_ADDR(ph)->pkt_bufs_cnt = ++bcnt;
1171 return 0;
1172 }
1173
1174 __attribute__((always_inline))
1175 #ifdef KERNEL
1176 static inline struct __kern_buflet *
1177 #else
1178 static inline struct __user_buflet *
1179 #endif
__packet_get_next_buflet(const uint64_t ph,const void * bprev0)1180 __packet_get_next_buflet(const uint64_t ph, const void *bprev0)
1181 {
1182 #ifdef KERNEL
1183 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1184 struct __kern_buflet *__single bcur = NULL;
1185 #else /* !KERNEL */
1186 buflet_t bprev = __DECONST(buflet_t, bprev0);
1187 void *bcur = NULL;
1188 #endif /* !KERNEL */
1189
1190 switch (SK_PTR_TYPE(ph)) {
1191 case NEXUS_META_TYPE_PACKET: {
1192 uint32_t bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1193 #ifdef KERNEL
1194 ASSERT(bcnt != 0 ||
1195 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1196 #else /* !KERNEL */
1197 /*
1198 * Handle the case where the metadata region gets
1199 * redirected to anonymous zero-filled pages at
1200 * defunct time. There's always 1 buflet in the
1201 * packet metadata, so pretend that's the count.
1202 */
1203 if (__improbable(bcnt == 0)) {
1204 bcnt = 1;
1205 bprev = NULL;
1206 }
1207 #endif /* !KERNEL */
1208 PKT_GET_NEXT_BUFLET(PKT_ADDR(ph), bcnt, BLT_ADDR(bprev), bcur);
1209 break;
1210 }
1211 case NEXUS_META_TYPE_QUANTUM:
1212 QUM_GET_NEXT_BUFLET(QUM_ADDR(ph), BLT_ADDR(bprev), bcur);
1213 break;
1214 default:
1215 #ifdef KERNEL
1216 VERIFY(0);
1217 /* NOTREACHED */
1218 __builtin_unreachable();
1219 #endif /* KERNEL */
1220 break;
1221 }
1222 return bcur;
1223 }
1224
1225 __attribute__((always_inline))
1226 static inline uint8_t
__packet_get_segment_count(const uint64_t ph)1227 __packet_get_segment_count(const uint64_t ph)
1228 {
1229 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1230 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1231
1232 return PKT_ADDR(ph)->pkt_seg_cnt;
1233 }
1234
1235 __attribute__((always_inline))
1236 static inline void
__packet_set_segment_count(const uint64_t ph,uint8_t segcount)1237 __packet_set_segment_count(const uint64_t ph, uint8_t segcount)
1238 {
1239 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1240 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1241
1242 PKT_ADDR(ph)->pkt_seg_cnt = segcount;
1243 }
1244
1245 __attribute__((always_inline))
1246 static inline uint16_t
__packet_get_protocol_segment_size(const uint64_t ph)1247 __packet_get_protocol_segment_size(const uint64_t ph)
1248 {
1249 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1250 return PKT_ADDR(ph)->pkt_proto_seg_sz;
1251 }
1252
1253 __attribute__((always_inline))
1254 static inline errno_t
__packet_set_protocol_segment_size(const uint64_t ph,uint16_t proto_seg_sz)1255 __packet_set_protocol_segment_size(const uint64_t ph, uint16_t proto_seg_sz)
1256 {
1257 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1258 PKT_ADDR(ph)->pkt_proto_seg_sz = proto_seg_sz;
1259 return 0;
1260 }
1261
1262 __attribute__((always_inline))
1263 static inline void
__packet_get_tso_flags(const uint64_t ph,packet_tso_flags_t * flags)1264 __packet_get_tso_flags(const uint64_t ph, packet_tso_flags_t *flags)
1265 {
1266 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_proto_seg_sz == sizeof(uint16_t)));
1267
1268 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1269 *flags = PKT_ADDR(ph)->pkt_csum_flags & (PACKET_CSUM_TSO_FLAGS);
1270 }
1271
1272 __attribute__((always_inline))
1273 static inline void
__packet_set_tso_flags(const uint64_t ph,packet_tso_flags_t flags)1274 __packet_set_tso_flags(const uint64_t ph, packet_tso_flags_t flags)
1275 {
1276 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1277
1278 PKT_ADDR(ph)->pkt_csum_flags |= flags & (PACKET_CSUM_TSO_FLAGS);
1279 }
1280
1281 __attribute__((always_inline))
1282 static inline uint32_t
__buflet_get_data_limit(const void * buf)1283 __buflet_get_data_limit(const void *buf)
1284 {
1285 return BLT_ADDR(buf)->buf_dlim;
1286 }
1287
1288 #ifdef KERNEL
1289 __attribute__((always_inline))
1290 static inline errno_t
__buflet_set_data_limit(const void * buf,const uint32_t dlim)1291 __buflet_set_data_limit(const void *buf, const uint32_t dlim)
1292 {
1293 /* buffer region is always marked as shareable */
1294 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1295
1296 /* full bounds checking will be performed during finalize */
1297 if (__probable((uint32_t)dlim <= BLT_ADDR(buf)->buf_objlim)) {
1298 _CASSERT(sizeof(BLT_ADDR(buf)->buf_dlim) == sizeof(uint32_t));
1299 /* deconst */
1300 *(uint32_t *)(uintptr_t)&BLT_ADDR(buf)->buf_dlim = dlim;
1301 return 0;
1302 }
1303 return ERANGE;
1304 }
1305 #endif /* KERNEL */
1306
1307 __attribute__((always_inline))
1308 static inline uint32_t
__buflet_get_data_offset(const void * buf)1309 __buflet_get_data_offset(const void *buf)
1310 {
1311 return BLT_ADDR(buf)->buf_doff;
1312 }
1313
1314 /*
1315 * ******************************************************************
1316 * Checks in __packet_finalize for packet finalized from userland
1317 * ******************************************************************
1318 * +-------+---------------------------+---------------------------+
1319 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1320 * |-------+---------------------------+---------------------------+
1321 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1322 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1323 * |l3_off | l3 = l2 | l3 == 0 |
1324 * |l4_off | l4 = l3 = l2 | l4 = l3 = 0 |
1325 * +-------+---------------------------+---------------------------+
1326 *
1327 * ******************************************************************
1328 * Checks in __packet_finalize for packet finalized from kernel
1329 * ******************************************************************
1330 * +-------+---------------------------+---------------------------+
1331 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1332 * |-------+---------------------------+---------------------------+
1333 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1334 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1335 * |l3_off | l3 >= l2 && l3 <bdlim | l3 == 0 |
1336 * |l4_off | l4 = l3 | l4 = l3 = 0 |
1337 * +-------+---------------------------+---------------------------+
1338 *
1339 */
1340 __attribute__((always_inline))
1341 static inline int
__packet_finalize(const uint64_t ph)1342 __packet_finalize(const uint64_t ph)
1343 {
1344 void *__single bcur = NULL, *__single bprev = NULL;
1345 uint32_t len, bcnt, bdoff0, bdlim0;
1346 int err = 0;
1347
1348 #ifdef KERNEL
1349 ASSERT(QUM_ADDR(ph)->qum_qflags & QUM_F_INTERNALIZED);
1350 #endif /* KERNEL */
1351 QUM_ADDR(ph)->qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1352
1353 bcnt = __packet_get_buflet_count(ph);
1354 len = QUM_ADDR(ph)->qum_len = 0;
1355
1356 while (bcnt--) {
1357 bcur = __packet_get_next_buflet(ph, bprev);
1358
1359 #ifdef KERNEL
1360 ASSERT(bcur != NULL);
1361 ASSERT(BLT_ADDR(bcur)->buf_addr != 0);
1362 #else /* !KERNEL */
1363 if (__improbable(bcur == NULL)) {
1364 err = ERANGE;
1365 break;
1366 }
1367 #endif /* KERNEL */
1368
1369 /* save data offset from the first buflet */
1370 if (bprev == NULL) {
1371 bdoff0 = __buflet_get_data_offset(bcur);
1372 bdlim0 = __buflet_get_data_limit(bcur);
1373 }
1374
1375 #ifndef KERNEL
1376 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)))) {
1377 err = ERANGE;
1378 break;
1379 }
1380 #else /* !KERNEL */
1381 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)) &&
1382 !PKT_HAS_ATTACHED_MBUF(ph))) {
1383 err = ERANGE;
1384 break;
1385 }
1386 #endif /* KERNEL */
1387 len += BLT_ADDR(bcur)->buf_dlen;
1388 bprev = bcur;
1389 }
1390
1391 if (__improbable(err != 0)) {
1392 goto done;
1393 }
1394
1395 switch (SK_PTR_TYPE(ph)) {
1396 case NEXUS_META_TYPE_PACKET:
1397 if (__improbable(bdoff0 > UINT8_MAX)) {
1398 err = ERANGE;
1399 goto done;
1400 }
1401 /* internalize headroom value from offset */
1402 PKT_ADDR(ph)->pkt_headroom = (uint8_t)bdoff0;
1403 /* validate header offsets in packet */
1404 switch (SK_PTR_SUBTYPE(ph)) {
1405 case NEXUS_META_SUBTYPE_RAW:
1406 #ifndef KERNEL
1407 /* Overwrite L2 len for raw packets from user space */
1408 PKT_ADDR(ph)->pkt_l2_len = 0;
1409 #else /* !KERNEL */
1410 /* ensure that L3 >= L2 && L3 < bdlim */
1411 if (__improbable((PKT_ADDR(ph)->pkt_headroom +
1412 PKT_ADDR(ph)->pkt_l2_len) >= bdlim0)) {
1413 err = ERANGE;
1414 goto done;
1415 }
1416 #endif /* KERNEL */
1417 break;
1418 case NEXUS_META_SUBTYPE_PAYLOAD:
1419 /*
1420 * For payload packet there is no concept of headroom
1421 * and L3 offset should always be 0
1422 */
1423 if (__improbable((PKT_ADDR(ph)->pkt_headroom != 0) ||
1424 (PKT_ADDR(ph)->pkt_l2_len != 0))) {
1425 err = ERANGE;
1426 goto done;
1427 }
1428 break;
1429 default:
1430 #ifdef KERNEL
1431 VERIFY(0);
1432 /* NOTREACHED */
1433 __builtin_unreachable();
1434 #endif /* KERNEL */
1435 break;
1436 }
1437
1438 if (__improbable(PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_DATA)) {
1439 #ifdef KERNEL
1440 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
1441 #else /* !KERNEL */
1442 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
1443 #endif /* !KERNEL */
1444 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1445 po->__po_expire_ts == 0) {
1446 err = EINVAL;
1447 goto done;
1448 }
1449 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) &&
1450 po->__po_token_len == 0) {
1451 err = EINVAL;
1452 goto done;
1453 }
1454 ASSERT(err == 0);
1455 }
1456
1457 /*
1458 * NOTE: we don't need the validation for total packet length
1459 * as checking if each buflet is in range and that
1460 * (pkt_headroom == bdoff0), should cover this check.
1461 */
1462 break;
1463
1464 default:
1465 /* nothing to do currently for quantum */
1466 break;
1467 }
1468
1469 done:
1470 if (__probable(err == 0)) {
1471 QUM_ADDR(ph)->qum_len = len;
1472 QUM_ADDR(ph)->qum_qflags |= QUM_F_FINALIZED;
1473 } else {
1474 QUM_ADDR(ph)->qum_len = 0;
1475 QUM_ADDR(ph)->qum_qflags |= QUM_F_DROPPED;
1476 }
1477
1478 return err;
1479 }
1480
1481 __attribute__((always_inline))
1482 static inline boolean_t
__packet_is_finalized(const uint64_t ph)1483 __packet_is_finalized(const uint64_t ph)
1484 {
1485 return QUM_ADDR(ph)->qum_qflags & QUM_F_FINALIZED;
1486 }
1487
1488 #ifdef KERNEL
1489 /*
1490 * function to initialize a packet with mbuf chain.
1491 * Apart from the attached mbuf, the packet can also be used to convey
1492 * additional metadata like the headroom and L2 header length.
1493 * For a packet with attached mbuf, the pkt_length conveys the length of
1494 * the attached mbuf. If the data copied is partial then PKT_F_TRUNCATED is
1495 * also set.
1496 */
1497 __attribute__((always_inline))
1498 static inline int
__packet_initialize_with_mbufchain(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1499 __packet_initialize_with_mbufchain(struct __kern_packet *pkt, struct mbuf *mbuf,
1500 uint8_t headroom, uint8_t l2len)
1501 {
1502 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1503 VERIFY(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1504 VERIFY((pkt->pkt_pflags & PKT_F_MBUF_MASK) == 0);
1505 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
1506 VERIFY(pkt->pkt_mbuf == NULL);
1507
1508 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1509 pkt->pkt_mbuf = mbuf;
1510 pkt->pkt_pflags |= (PKT_F_MBUF_DATA | PKT_F_TRUNCATED);
1511 pkt->pkt_headroom = headroom;
1512 pkt->pkt_l2_len = l2len;
1513 pkt->pkt_length = m_pktlen(mbuf);
1514 pkt->pkt_qum_buf.buf_dlen = 0;
1515 pkt->pkt_qum_buf.buf_doff = 0;
1516 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1517 return 0;
1518 }
1519
1520 __attribute__((always_inline))
1521 static inline int
__packet_initialize_with_mbuf(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1522 __packet_initialize_with_mbuf(struct __kern_packet *pkt, struct mbuf *mbuf,
1523 uint8_t headroom, uint8_t l2len)
1524 {
1525 __packet_initialize_with_mbufchain(pkt, mbuf, headroom, l2len);
1526 VERIFY(mbuf->m_nextpkt == NULL);
1527 return 0;
1528 }
1529
1530 /*
1531 * function to finalize a packet with attached mbuf.
1532 */
1533 __attribute__((always_inline))
1534 static inline int
__packet_finalize_with_mbuf(struct __kern_packet * pkt)1535 __packet_finalize_with_mbuf(struct __kern_packet *pkt)
1536 {
1537 uint32_t bdlen, bdoff, bdlim;
1538 struct __kern_buflet *buf;
1539 int err = 0;
1540
1541 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1542 VERIFY((pkt->pkt_pflags & (PKT_F_MBUF_DATA | PKT_F_PKT_DATA)) ==
1543 PKT_F_MBUF_DATA);
1544 VERIFY(pkt->pkt_mbuf != NULL);
1545 ASSERT(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1546 VERIFY(pkt->pkt_bufs_cnt == 1);
1547 PKT_GET_FIRST_BUFLET(pkt, pkt->pkt_bufs_cnt, buf);
1548 ASSERT(buf->buf_addr != 0);
1549
1550 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1551 pkt->pkt_pflags &= ~PKT_F_TRUNCATED;
1552 bdlen = buf->buf_dlen;
1553 bdlim = buf->buf_dlim;
1554 bdoff = buf->buf_doff;
1555 if (__improbable(!BUF_IN_RANGE(buf))) {
1556 err = ERANGE;
1557 goto done;
1558 }
1559
1560 /* validate header offsets in packet */
1561 switch (METADATA_SUBTYPE(pkt)) {
1562 case NEXUS_META_SUBTYPE_RAW:
1563 if (__improbable((pkt->pkt_headroom != bdoff) ||
1564 (pkt->pkt_headroom >= bdlim))) {
1565 err = ERANGE;
1566 goto done;
1567 }
1568 if (__improbable((pkt->pkt_headroom +
1569 pkt->pkt_l2_len) >= bdlim)) {
1570 err = ERANGE;
1571 goto done;
1572 }
1573 break;
1574
1575 case NEXUS_META_SUBTYPE_PAYLOAD:
1576 /*
1577 * For payload packet there is no concept of headroom.
1578 */
1579 if (__improbable((pkt->pkt_headroom != 0) || (bdoff != 0) ||
1580 (pkt->pkt_l2_len != 0))) {
1581 err = ERANGE;
1582 goto done;
1583 }
1584 break;
1585
1586 default:
1587 VERIFY(0);
1588 /* NOTREACHED */
1589 __builtin_unreachable();
1590 break;
1591 }
1592
1593
1594 if (__improbable(pkt->pkt_pflags & PKT_F_OPT_DATA)) {
1595 struct __packet_opt *po = pkt->pkt_com_opt;
1596
1597 if ((pkt->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1598 po->__po_expire_ts == 0) {
1599 err = EINVAL;
1600 goto done;
1601 }
1602 if ((pkt->pkt_pflags & PKT_F_OPT_TOKEN) &&
1603 po->__po_token_len == 0) {
1604 err = EINVAL;
1605 goto done;
1606 }
1607 }
1608 ASSERT(err == 0);
1609
1610 done:
1611 if (__probable(err == 0)) {
1612 pkt->pkt_length = (uint32_t)m_pktlen(pkt->pkt_mbuf);
1613 if (bdlen < pkt->pkt_length) {
1614 pkt->pkt_pflags |= PKT_F_TRUNCATED;
1615 }
1616 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1617 } else {
1618 pkt->pkt_length = 0;
1619 pkt->pkt_qum.qum_qflags |= QUM_F_DROPPED;
1620 }
1621
1622 return err;
1623 }
1624
1625 __attribute__((always_inline))
1626 static inline uint32_t
__packet_get_object_index(const uint64_t ph)1627 __packet_get_object_index(const uint64_t ph)
1628 {
1629 return METADATA_IDX(QUM_ADDR(ph));
1630 }
1631
1632 __attribute__((always_inline))
1633 static inline errno_t
__packet_get_timestamp(const uint64_t ph,uint64_t * ts,boolean_t * valid)1634 __packet_get_timestamp(const uint64_t ph, uint64_t *ts, boolean_t *valid)
1635 {
1636 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1637
1638 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TS_VALID) != 0) {
1639 if (valid != NULL) {
1640 *valid = TRUE;
1641 }
1642 *ts = PKT_ADDR(ph)->pkt_timestamp;
1643 } else {
1644 if (valid != NULL) {
1645 *valid = FALSE;
1646 }
1647 *ts = 0;
1648 }
1649
1650 return 0;
1651 }
1652
1653 __attribute__((always_inline))
1654 static inline errno_t
__packet_set_timestamp(const uint64_t ph,uint64_t ts,boolean_t valid)1655 __packet_set_timestamp(const uint64_t ph, uint64_t ts, boolean_t valid)
1656 {
1657 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1658
1659 if (valid) {
1660 PKT_ADDR(ph)->pkt_timestamp = ts;
1661 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TS_VALID;
1662 } else {
1663 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_TS_VALID;
1664 PKT_ADDR(ph)->pkt_timestamp = 0;
1665 }
1666
1667 return 0;
1668 }
1669
1670 __attribute__((always_inline))
1671 static inline errno_t
__packet_get_tx_completion_data(const uint64_t ph,uintptr_t * cb_arg,uintptr_t * cb_data)1672 __packet_get_tx_completion_data(const uint64_t ph, uintptr_t *cb_arg,
1673 uintptr_t *cb_data)
1674 {
1675 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1676 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1677 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1678 *cb_arg = PKT_ADDR(ph)->pkt_tx_compl_cb_arg;
1679 *cb_data = PKT_ADDR(ph)->pkt_tx_compl_cb_data;
1680 } else {
1681 *cb_arg = 0;
1682 *cb_data = 0;
1683 }
1684 return 0;
1685 }
1686
1687 __attribute__((always_inline))
1688 static inline errno_t
__packet_set_tx_completion_data(const uint64_t ph,uintptr_t cb_arg,uintptr_t cb_data)1689 __packet_set_tx_completion_data(const uint64_t ph, uintptr_t cb_arg,
1690 uintptr_t cb_data)
1691 {
1692 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1693 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1694 PKT_ADDR(ph)->pkt_tx_compl_cb_arg = cb_arg;
1695 PKT_ADDR(ph)->pkt_tx_compl_cb_data = cb_data;
1696 return 0;
1697 }
1698
1699 __attribute__((always_inline))
1700 static inline errno_t
__packet_get_timestamp_requested(const uint64_t ph,boolean_t * requested)1701 __packet_get_timestamp_requested(const uint64_t ph, boolean_t *requested)
1702 {
1703 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1704 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0) {
1705 *requested = TRUE;
1706 } else {
1707 *requested = FALSE;
1708 }
1709 return 0;
1710 }
1711
1712 __attribute__((always_inline))
1713 static inline errno_t
__packet_get_tx_completion_status(const uint64_t ph,kern_return_t * status)1714 __packet_get_tx_completion_status(const uint64_t ph, kern_return_t *status)
1715 {
1716 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1717 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1718 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1719 *status = (kern_return_t)PKT_ADDR(ph)->pkt_tx_compl_status;
1720 } else {
1721 *status = 0;
1722 }
1723 return 0;
1724 }
1725
1726 __attribute__((always_inline))
1727 static inline errno_t
__packet_set_tx_completion_status(const uint64_t ph,kern_return_t status)1728 __packet_set_tx_completion_status(const uint64_t ph, kern_return_t status)
1729 {
1730 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1731 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1732 PKT_ADDR(ph)->pkt_tx_compl_status = (uint32_t)status;
1733 return 0;
1734 }
1735
1736 __attribute__((always_inline))
1737 static inline errno_t
__packet_set_tx_nx_port(const uint64_t ph,nexus_port_t nx_port,uint16_t vpna_gencnt)1738 __packet_set_tx_nx_port(const uint64_t ph, nexus_port_t nx_port,
1739 uint16_t vpna_gencnt)
1740 {
1741 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1742 PKT_ADDR(ph)->pkt_nx_port = nx_port;
1743 PKT_ADDR(ph)->pkt_vpna_gencnt = vpna_gencnt;
1744 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TX_PORT_DATA;
1745 return 0;
1746 }
1747
1748 __attribute__((always_inline))
1749 static inline errno_t
__packet_get_tx_nx_port(const uint64_t ph,nexus_port_t * nx_port,uint16_t * vpna_gencnt)1750 __packet_get_tx_nx_port(const uint64_t ph, nexus_port_t *nx_port,
1751 uint16_t *vpna_gencnt)
1752 {
1753 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1754 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_PORT_DATA) == 0) {
1755 return ENOTSUP;
1756 }
1757
1758 *nx_port = PKT_ADDR(ph)->pkt_nx_port;
1759 *vpna_gencnt = PKT_ADDR(ph)->pkt_vpna_gencnt;
1760 return 0;
1761 }
1762
1763 __attribute__((always_inline))
1764 static inline errno_t
__packet_get_tx_nx_port_id(const uint64_t ph,uint32_t * nx_port_id)1765 __packet_get_tx_nx_port_id(const uint64_t ph, uint32_t *nx_port_id)
1766 {
1767 errno_t err;
1768 nexus_port_t nx_port;
1769 uint16_t vpna_gencnt;
1770
1771 _CASSERT(sizeof(nx_port) == sizeof(uint16_t));
1772
1773 err = __packet_get_tx_nx_port(ph, &nx_port, &vpna_gencnt);
1774 if (err == 0) {
1775 *nx_port_id = PKT_COMPOSE_NX_PORT_ID(nx_port, vpna_gencnt);
1776 }
1777 return err;
1778 }
1779
1780
1781 __attribute__((always_inline))
1782 static inline errno_t
__packet_get_flowid(const uint64_t ph,packet_flowid_t * pflowid)1783 __packet_get_flowid(const uint64_t ph, packet_flowid_t *pflowid)
1784 {
1785 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1786 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_FLOW_ID) == 0) {
1787 return ENOENT;
1788 }
1789 *pflowid = PKT_ADDR(ph)->pkt_flow_token;
1790 return 0;
1791 }
1792 #endif /* KERNEL */
1793
1794 extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t);
1795
1796 __attribute__((always_inline))
1797 static inline uint16_t
__packet_fold_sum(uint32_t sum)1798 __packet_fold_sum(uint32_t sum)
1799 {
1800 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1801 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1802 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1803 return sum & 0xffff;
1804 }
1805
1806 __attribute__((always_inline))
1807 static inline uint16_t
__packet_fold_sum_final(uint32_t sum)1808 __packet_fold_sum_final(uint32_t sum)
1809 {
1810 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1811 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1812 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1813 return ~sum & 0xffff;
1814 }
1815
1816 __attribute__((always_inline))
1817 static inline uint32_t
__packet_cksum(const void * data,uint32_t len,uint32_t sum0)1818 __packet_cksum(const void *data, uint32_t len, uint32_t sum0)
1819 {
1820 return os_cpu_in_cksum(data, len, sum0);
1821 }
1822
1823 extern uint32_t os_cpu_copy_in_cksum(const void *__sized_by(len), void *__sized_by(len),
1824 uint32_t len, uint32_t);
1825
1826 __attribute__((always_inline))
1827 static inline uint32_t
__packet_copy_and_sum(const void * __sized_by (len)src,void * __sized_by (len)dst,uint32_t len,uint32_t sum0)1828 __packet_copy_and_sum(const void *__sized_by(len) src, void *__sized_by(len) dst,
1829 uint32_t len, uint32_t sum0)
1830 {
1831 return os_cpu_copy_in_cksum(src, dst, len, sum0);
1832 }
1833
1834 __attribute__((always_inline))
1835 static inline uint16_t
__packet_fix_sum(uint16_t csum,uint16_t old,uint16_t new)1836 __packet_fix_sum(uint16_t csum, uint16_t old, uint16_t new)
1837 {
1838 uint32_t c = csum + old - new;
1839 c = (c >> 16) + (c & 0xffff); /* Only add carry once */
1840
1841 return c & 0xffff;
1842 }
1843
1844 /* MUST be used for uint32_t fields */
1845 __attribute__((always_inline))
1846 static inline void
1847 __packet_fix_hdr_sum(uint8_t *__sized_by(4)field, uint16_t *csum, uint32_t new)
1848 {
1849 uint32_t old;
1850 memcpy(&old, field, sizeof(old));
1851 memcpy(field, &new, sizeof(uint32_t));
1852 *csum = __packet_fix_sum(__packet_fix_sum(*csum, (uint16_t)(old >> 16),
1853 (uint16_t)(new >> 16)), (uint16_t)(old & 0xffff),
1854 (uint16_t)(new & 0xffff));
1855 }
1856
1857 __attribute__((always_inline))
1858 static inline void *__header_indexable
__buflet_get_data_address(const void * buf)1859 __buflet_get_data_address(const void *buf)
1860 {
1861 return __unsafe_forge_bidi_indexable(void *, (void *)(BLT_ADDR(buf)->buf_addr),
1862 BLT_ADDR(buf)->buf_dlim);
1863 }
1864
1865 #ifdef KERNEL
1866 __attribute__((always_inline))
1867 static inline errno_t
__buflet_set_data_address(const void * buf,const void * addr)1868 __buflet_set_data_address(const void *buf, const void *addr)
1869 {
1870 /* buffer region is always marked as shareable */
1871 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1872
1873 /* full bounds checking will be performed during finalize */
1874 if (__probable((uintptr_t)addr >=
1875 (uintptr_t)BLT_ADDR(buf)->buf_objaddr)) {
1876 _CASSERT(sizeof(BLT_ADDR(buf)->buf_addr) ==
1877 sizeof(mach_vm_address_t));
1878 /* deconst */
1879 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1880 (mach_vm_address_t)addr;
1881 return 0;
1882 }
1883 return ERANGE;
1884 }
1885
1886 /*
1887 * Equivalent to __buflet_set_data_address but based on offset, packets/buflets
1888 * set with this should not be directly passed to userspace, since shared buffer
1889 * is not yet supported by user facing pool.
1890 */
1891 __attribute__((always_inline))
1892 static inline int
__buflet_set_buffer_offset(const void * buf,const uint32_t off)1893 __buflet_set_buffer_offset(const void *buf, const uint32_t off)
1894 {
1895 ASSERT(BLT_ADDR(buf)->buf_objlim != 0);
1896
1897 if (__probable(off <= BLT_ADDR(buf)->buf_objlim)) {
1898 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1899 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr + off;
1900 return 0;
1901 }
1902 return ERANGE;
1903 }
1904 #endif /* KERNEL */
1905
1906 __attribute__((always_inline))
1907 static inline int
__buflet_set_data_offset(const void * buf,const uint32_t doff)1908 __buflet_set_data_offset(const void *buf, const uint32_t doff)
1909 {
1910 #ifdef KERNEL
1911 /*
1912 * Kernel-specific assertion. For user space, the metadata
1913 * region gets redirected to anonymous zero-filled pages at
1914 * defunct time, so ignore it there.
1915 */
1916 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1917
1918 if (__probable((uint32_t)doff <= BLT_ADDR(buf)->buf_objlim)) {
1919 BLT_ADDR(buf)->buf_doff = doff;
1920 return 0;
1921 }
1922 return ERANGE;
1923 #else /* !KERNEL */
1924 BLT_ADDR(buf)->buf_doff = doff;
1925 return 0;
1926 #endif /* KERNEL */
1927 }
1928
1929 __attribute__((always_inline))
1930 static inline int
__buflet_set_data_length(const void * buf,const uint32_t dlen)1931 __buflet_set_data_length(const void *buf, const uint32_t dlen)
1932 {
1933 #ifdef KERNEL
1934 /*
1935 * Kernel-specific assertion. For user space, the metadata
1936 * region gets redirected to anonymous zero-filled pages at
1937 * defunct time, so ignore it there.
1938 */
1939 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1940
1941 if (__probable((uint32_t)dlen <= BLT_ADDR(buf)->buf_objlim)) {
1942 BLT_ADDR(buf)->buf_dlen = dlen;
1943 return 0;
1944 }
1945 return ERANGE;
1946 #else /* !KERNEL */
1947 BLT_ADDR(buf)->buf_dlen = dlen;
1948 return 0;
1949 #endif /* KERNEL */
1950 }
1951
1952 __attribute__((always_inline))
1953 static inline uint32_t
__buflet_get_data_length(const void * buf)1954 __buflet_get_data_length(const void *buf)
1955 {
1956 return BLT_ADDR(buf)->buf_dlen;
1957 }
1958
1959 #ifdef KERNEL
1960 __attribute__((always_inline))
1961 static inline struct sksegment *
__buflet_get_object_segment(const void * buf,kern_obj_idx_seg_t * idx)1962 __buflet_get_object_segment(const void *buf, kern_obj_idx_seg_t *idx)
1963 {
1964 _CASSERT(sizeof(obj_idx_t) == sizeof(kern_obj_idx_seg_t));
1965
1966 if (idx != NULL) {
1967 *idx = BLT_ADDR(buf)->buf_ctl->bc_idx;
1968 }
1969
1970 return BLT_ADDR(buf)->buf_ctl->bc_slab->sl_seg;
1971 }
1972 #endif /* KERNEL */
1973
1974 __attribute__((always_inline))
1975 static inline void *
__buflet_get_object_address(const void * buf)1976 __buflet_get_object_address(const void *buf)
1977 {
1978 #ifdef KERNEL
1979 return (void *)(BLT_ADDR(buf)->buf_objaddr);
1980 #else /* !KERNEL */
1981 /*
1982 * For user space, shared buffer is not available and hence the data
1983 * address is immutable and is always the same as the underlying
1984 * buffer object address itself.
1985 */
1986 return __buflet_get_data_address(buf);
1987 #endif /* !KERNEL */
1988 }
1989
1990 __attribute__((always_inline))
1991 static inline uint32_t
__buflet_get_object_limit(const void * buf)1992 __buflet_get_object_limit(const void *buf)
1993 {
1994 #ifdef KERNEL
1995 return BLT_ADDR(buf)->buf_objlim;
1996 #else /* !KERNEL */
1997 /*
1998 * For user space, shared buffer is not available and hence the data
1999 * limit is immutable and is always the same as the underlying buffer
2000 * object limit itself.
2001 */
2002 return (uint32_t)__buflet_get_data_limit(buf);
2003 #endif /* !KERNEL */
2004 }
2005
2006 __attribute__((always_inline))
2007 static inline packet_trace_id_t
__packet_get_trace_id(const uint64_t ph)2008 __packet_get_trace_id(const uint64_t ph)
2009 {
2010 switch (SK_PTR_TYPE(ph)) {
2011 case NEXUS_META_TYPE_PACKET:
2012 return PKT_ADDR(ph)->pkt_trace_id;
2013 break;
2014 default:
2015 return 0;
2016 }
2017 }
2018
2019 __attribute__((always_inline))
2020 static inline void
__packet_set_trace_id(const uint64_t ph,packet_trace_id_t id)2021 __packet_set_trace_id(const uint64_t ph, packet_trace_id_t id)
2022 {
2023 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2024 PKT_ADDR(ph)->pkt_trace_id = id;
2025 }
2026
2027 __attribute__((always_inline))
2028 static inline void
__packet_trace_event(const uint64_t ph,uint32_t event)2029 __packet_trace_event(const uint64_t ph, uint32_t event)
2030 {
2031 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2032 #ifdef KERNEL
2033 #pragma unused(event, ph)
2034 KDBG(event, PKT_ADDR(ph)->pkt_trace_id);
2035 #else /* !KERNEL */
2036 kdebug_trace(event, PKT_ADDR(ph)->pkt_trace_id, 0, 0, 0);
2037 #endif /* !KERNEL */
2038 }
2039
2040 #ifdef KERNEL
2041 __attribute__((always_inline))
2042 static inline packet_trace_tag_t
__packet_get_trace_tag(const uint64_t ph)2043 __packet_get_trace_tag(const uint64_t ph)
2044 {
2045 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2046 return PKT_ADDR(ph)->pkt_trace_tag;
2047 }
2048
2049 __attribute__((always_inline))
2050 static inline void
__packet_set_trace_tag(const uint64_t ph,packet_trace_tag_t tag)2051 __packet_set_trace_tag(const uint64_t ph, packet_trace_tag_t tag)
2052 {
2053 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2054 PKT_ADDR(ph)->pkt_trace_tag = tag;
2055 }
2056
2057 static inline void
__packet_perform_tx_completion_callbacks(const kern_packet_t ph,ifnet_t ifp)2058 __packet_perform_tx_completion_callbacks(const kern_packet_t ph, ifnet_t ifp)
2059 {
2060 /*
2061 * NOTE: this function can be called with ifp as NULL.
2062 */
2063 uint64_t ts;
2064 kern_return_t tx_status;
2065 uintptr_t cb_arg, cb_data;
2066 struct __kern_packet *kpkt = SK_PTR_ADDR_KPKT(ph);
2067
2068 ASSERT((kpkt->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0);
2069 (void) __packet_get_tx_completion_status(ph, &tx_status);
2070 __packet_get_tx_completion_data(ph, &cb_arg, &cb_data);
2071 __packet_get_timestamp(ph, &ts, NULL);
2072 while (kpkt->pkt_tx_compl_callbacks != 0) {
2073 mbuf_tx_compl_func cb;
2074 uint32_t i;
2075
2076 i = ffs(kpkt->pkt_tx_compl_callbacks) - 1;
2077 kpkt->pkt_tx_compl_callbacks &= ~(1 << i);
2078 cb = m_get_tx_compl_callback(i);
2079 if (__probable(cb != NULL)) {
2080 cb(kpkt->pkt_tx_compl_context, ifp, ts, cb_arg, cb_data,
2081 tx_status);
2082 }
2083 }
2084 kpkt->pkt_pflags &= ~PKT_F_TX_COMPL_TS_REQ;
2085 }
2086
2087 static inline void *
__packet_get_priv(const kern_packet_t ph)2088 __packet_get_priv(const kern_packet_t ph)
2089 {
2090 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2091 return PKT_ADDR(ph)->pkt_priv;
2092 }
2093
2094 static inline void
__packet_set_priv(const uint64_t ph,void * priv)2095 __packet_set_priv(const uint64_t ph, void *priv)
2096 {
2097 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2098 PKT_ADDR(ph)->pkt_priv = priv;
2099 }
2100 #endif /* KERNEL */
2101
2102 #endif /* PRIVATE || BSD_KERNEL_PRIVATE */
2103 #endif /* !_SKYWALK_PACKET_COMMON_H_ */
2104