1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_PACKET_COMMON_H_
30 #define _SKYWALK_PACKET_COMMON_H_
31
32 #if defined(PRIVATE) || defined(BSD_KERNEL_PRIVATE)
33 /*
34 * Routines common to kernel and userland. This file is intended to
35 * be included by code implementing the packet APIs, in particular,
36 * the Skywalk kernel and libsyscall code.
37 */
38
39 #include <skywalk/os_packet_private.h>
40 #include <net/if_vlan_var.h>
41 #include <sys/errno.h>
42 #include <sys/kdebug.h>
43
44 #ifndef KERNEL
45 /*
46 * User.
47 */
48 #if !defined(LIBSYSCALL_INTERFACE)
49 #error "LIBSYSCALL_INTERFACE not defined"
50 #endif /* !LIBSYSCALL_INTERFACE */
51 #define QUM_ADDR(_ph) SK_PTR_ADDR_UQUM(_ph)
52 #define PKT_ADDR(_ph) SK_PTR_ADDR_UPKT(_ph)
53 #define BLT_ADDR(_bp) ((struct __user_buflet *)(uintptr_t)_bp)
54 #else /* KERNEL */
55 /*
56 * Kernel.
57 */
58 #include <skywalk/packet/packet_var.h>
59 #include <skywalk/packet/pbufpool_var.h>
60 #define QUM_ADDR(_ph) SK_PTR_ADDR_KQUM(_ph)
61 #define PKT_ADDR(_ph) SK_PTR_ADDR_KPKT(_ph)
62 #define BLT_ADDR(_bp) ((struct __kern_buflet *)(uintptr_t)_bp)
63 #define PKT_HAS_ATTACHED_MBUF(_ph) \
64 ((PKT_ADDR(_ph)->pkt_pflags & PKT_F_MBUF_DATA) != 0)
65 #endif /* KERNEL */
66
67 /*
68 * Common.
69 */
70 #if (DEBUG || DEVELOPMENT)
71 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) do { \
72 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type) || \
73 SK_PTR_SUBTYPE(_ph) != (uint64_t)(_subtype))) { \
74 pkt_subtype_assert_fail(_ph, _type, _subtype); \
75 /* NOTREACHED */ \
76 __builtin_unreachable(); \
77 } \
78 } while (0)
79
80 #define PKT_TYPE_ASSERT(_ph, _type) do { \
81 if (__improbable(SK_PTR_TYPE(_ph) != (uint64_t)(_type))) { \
82 pkt_type_assert_fail(_ph, _type); \
83 /* NOTREACHED */ \
84 __builtin_unreachable(); \
85 } \
86 } while (0)
87 #else /* !DEBUG && !DEVELOPMENT */
88 #define PKT_SUBTYPE_ASSERT(_ph, _type, _subtype) ((void)0)
89 #define PKT_TYPE_ASSERT(_ph, _type) ((void)0)
90 #endif /* !DEBUG && !DEVELOPMENT */
91
92 #define QUM_GET_NEXT_BUFLET(_qum, _pbuf, _buf) do { \
93 ASSERT((_pbuf) == NULL || (_pbuf) == (_qum)->qum_buf); \
94 (_buf) = (((_pbuf) == NULL) ? (_qum)->qum_buf : NULL); \
95 } while (0)
96
97 #define PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf) do { \
98 if (__improbable((_bcnt) == 0)) { \
99 (_buf) = NULL; \
100 break; \
101 } \
102 if (__probable((_pkt)->pkt_qum_buf.buf_addr != 0)) { \
103 (_buf) = &(_pkt)->pkt_qum_buf; \
104 } else { \
105 (_buf) = __unsafe_forge_single(struct __kern_buflet *, \
106 __DECONST(void *, (_pkt)->pkt_qum_buf.buf_nbft_addr));\
107 } \
108 } while (0)
109
110 #define _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
111 if ((_pbuf) == NULL) { \
112 PKT_GET_FIRST_BUFLET(_pkt, _bcnt, _buf); \
113 } else { \
114 (_buf) = __unsafe_forge_single(struct __kern_buflet *, \
115 __DECONST(void *, (_pbuf)->buf_nbft_addr)); \
116 } \
117 } while (0)
118
119 #ifndef KERNEL
120 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
121 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
122 } while (0)
123 #else /* KERNEL */
124 #define PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf) do { \
125 ASSERT(((_bcnt) >= 1) || ((_pbuf) == NULL)); \
126 _PKT_GET_NEXT_BUFLET(_pkt, _bcnt, _pbuf, _buf); \
127 } while (0)
128 #endif /* KERNEL */
129
130 #ifdef KERNEL
131 #define PKT_COMPOSE_NX_PORT_ID(_nx_port, _gencnt) \
132 ((uint32_t)((_gencnt & 0xffff) << 16) | (_nx_port & 0xffff))
133
134 #define PKT_DECOMPOSE_NX_PORT_ID(_nx_port_id, _nx_port, _gencnt) do { \
135 _nx_port = _nx_port_id & 0xffff; \
136 _gencnt = (_nx_port_id >> 16) & 0xffff; \
137 } while (0)
138 #endif /* KERNEL */
139
140 __attribute__((always_inline))
141 static inline int
__packet_set_headroom(const uint64_t ph,const uint8_t headroom)142 __packet_set_headroom(const uint64_t ph, const uint8_t headroom)
143 {
144 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
145 if (__probable(headroom < PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
146 PKT_ADDR(ph)->pkt_headroom = headroom;
147 return 0;
148 }
149 return ERANGE;
150 }
151
152 __attribute__((always_inline))
153 static inline uint8_t
__packet_get_headroom(const uint64_t ph)154 __packet_get_headroom(const uint64_t ph)
155 {
156 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
157 return PKT_ADDR(ph)->pkt_headroom;
158 }
159
160 __attribute__((always_inline))
161 static inline int
__packet_set_link_header_length(const uint64_t ph,const uint8_t len)162 __packet_set_link_header_length(const uint64_t ph, const uint8_t len)
163 {
164 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
165 if (__probable(len <= PKT_ADDR(ph)->pkt_qum_buf.buf_dlim)) {
166 PKT_ADDR(ph)->pkt_l2_len = len;
167 return 0;
168 }
169 return ERANGE;
170 }
171
172 __attribute__((always_inline))
173 static inline uint8_t
__packet_get_link_header_length(const uint64_t ph)174 __packet_get_link_header_length(const uint64_t ph)
175 {
176 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
177 return PKT_ADDR(ph)->pkt_l2_len;
178 }
179
180 __attribute__((always_inline))
181 static inline int
__packet_set_link_broadcast(const uint64_t ph)182 __packet_set_link_broadcast(const uint64_t ph)
183 {
184 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
185 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_BCAST;
186 return 0;
187 }
188
189 __attribute__((always_inline))
190 static inline boolean_t
__packet_get_link_broadcast(const uint64_t ph)191 __packet_get_link_broadcast(const uint64_t ph)
192 {
193 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
194 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_BCAST) != 0;
195 }
196
197 __attribute__((always_inline))
198 static inline int
__packet_set_link_multicast(const uint64_t ph)199 __packet_set_link_multicast(const uint64_t ph)
200 {
201 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
202 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_MCAST;
203 return 0;
204 }
205
206 __attribute__((always_inline))
207 static inline boolean_t
__packet_get_link_multicast(const uint64_t ph)208 __packet_get_link_multicast(const uint64_t ph)
209 {
210 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
211 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_MCAST) != 0;
212 }
213
214 __attribute__((always_inline))
215 static inline int
__packet_set_link_ethfcs(const uint64_t ph)216 __packet_set_link_ethfcs(const uint64_t ph)
217 {
218 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
219 PKT_ADDR(ph)->pkt_link_flags |= PKT_LINKF_ETHFCS;
220 return 0;
221 }
222
223 __attribute__((always_inline))
224 static inline boolean_t
__packet_get_link_ethfcs(const uint64_t ph)225 __packet_get_link_ethfcs(const uint64_t ph)
226 {
227 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
228 return (PKT_ADDR(ph)->pkt_link_flags & PKT_LINKF_ETHFCS) != 0;
229 }
230
231 __attribute__((always_inline))
232 static inline int
__packet_set_transport_traffic_background(const uint64_t ph)233 __packet_set_transport_traffic_background(const uint64_t ph)
234 {
235 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
236 PKT_ADDR(ph)->pkt_pflags |= PKT_F_BACKGROUND;
237 return 0;
238 }
239
240 __attribute__((always_inline))
241 static inline boolean_t
__packet_get_transport_traffic_background(const uint64_t ph)242 __packet_get_transport_traffic_background(const uint64_t ph)
243 {
244 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
245 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_BACKGROUND) != 0;
246 }
247
248 __attribute__((always_inline))
249 static inline int
__packet_set_transport_traffic_realtime(const uint64_t ph)250 __packet_set_transport_traffic_realtime(const uint64_t ph)
251 {
252 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
253 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REALTIME;
254 return 0;
255 }
256
257 __attribute__((always_inline))
258 static inline boolean_t
__packet_get_transport_traffic_realtime(const uint64_t ph)259 __packet_get_transport_traffic_realtime(const uint64_t ph)
260 {
261 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
262 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REALTIME) != 0;
263 }
264
265 __attribute__((always_inline))
266 static inline int
__packet_set_transport_retransmit(const uint64_t ph)267 __packet_set_transport_retransmit(const uint64_t ph)
268 {
269 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
270 PKT_ADDR(ph)->pkt_pflags |= PKT_F_REXMT;
271 return 0;
272 }
273
274 __attribute__((always_inline))
275 static inline boolean_t
__packet_get_transport_retransmit(const uint64_t ph)276 __packet_get_transport_retransmit(const uint64_t ph)
277 {
278 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
279 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_REXMT) != 0;
280 }
281
282 __attribute__((always_inline))
283 static inline int
__packet_set_transport_last_packet(const uint64_t ph)284 __packet_set_transport_last_packet(const uint64_t ph)
285 {
286 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
287 PKT_ADDR(ph)->pkt_pflags |= PKT_F_LAST_PKT;
288 return 0;
289 }
290
291 __attribute__((always_inline))
292 static inline int
__packet_set_group_start(const uint64_t ph)293 __packet_set_group_start(const uint64_t ph)
294 {
295 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
296 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_START;
297 return 0;
298 }
299
300 __attribute__((always_inline))
301 static inline boolean_t
__packet_get_group_start(const uint64_t ph)302 __packet_get_group_start(const uint64_t ph)
303 {
304 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
305 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_START) != 0;
306 }
307
308 __attribute__((always_inline))
309 static inline int
__packet_set_group_end(const uint64_t ph)310 __packet_set_group_end(const uint64_t ph)
311 {
312 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
313 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_GROUP_END;
314 return 0;
315 }
316
317 __attribute__((always_inline))
318 static inline boolean_t
__packet_get_group_end(const uint64_t ph)319 __packet_get_group_end(const uint64_t ph)
320 {
321 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
322 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_GROUP_END) != 0;
323 }
324
325 __attribute__((always_inline))
326 static inline errno_t
__packet_get_expire_time(const uint64_t ph,uint64_t * ts)327 __packet_get_expire_time(const uint64_t ph, uint64_t *ts)
328 {
329 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
330 #ifdef KERNEL
331 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
332 #else /* !KERNEL */
333 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
334 #endif /* !KERNEL */
335 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) == 0) {
336 return ENOENT;
337 }
338 if (ts == NULL) {
339 return EINVAL;
340 }
341 *ts = po->__po_expire_ts;
342 return 0;
343 }
344
345 __attribute__((always_inline))
346 static inline errno_t
__packet_set_expire_time(const uint64_t ph,const uint64_t ts)347 __packet_set_expire_time(const uint64_t ph, const uint64_t ts)
348 {
349 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
350 #ifdef KERNEL
351 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
352 #else /* !KERNEL */
353 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
354 #endif /* !KERNEL */
355 if (ts != 0) {
356 po->__po_expire_ts = ts;
357 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXPIRE_TS;
358 } else {
359 po->__po_expire_ts = 0;
360 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXPIRE_TS;
361 }
362 return 0;
363 }
364
365 __attribute__((always_inline))
366 static inline errno_t
__packet_get_expiry_action(const uint64_t ph,packet_expiry_action_t * pea)367 __packet_get_expiry_action(const uint64_t ph, packet_expiry_action_t *pea)
368 {
369 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
370 #ifdef KERNEL
371 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
372 #else /* !KERNEL */
373 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
374 #endif /* !KERNEL */
375 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXP_ACTION) == 0) {
376 return ENOENT;
377 }
378 if (pea == NULL) {
379 return EINVAL;
380 }
381 *pea = po->__po_expiry_action;
382 return 0;
383 }
384
385 __attribute__((always_inline))
386 static inline errno_t
__packet_set_expiry_action(const uint64_t ph,packet_expiry_action_t pea)387 __packet_set_expiry_action(const uint64_t ph, packet_expiry_action_t pea)
388 {
389 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
390 #ifdef KERNEL
391 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
392 #else /* !KERNEL */
393 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
394 #endif /* !KERNEL */
395 if (pea != PACKET_EXPIRY_ACTION_NONE) {
396 po->__po_expiry_action = (uint8_t)pea;
397 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_EXP_ACTION;
398 } else {
399 po->__po_expiry_action = 0;
400 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_EXP_ACTION;
401 }
402 return 0;
403 }
404
405 __attribute__((always_inline))
406 static inline errno_t
__packet_opt_get_token(const struct __packet_opt * po,void * __sized_by (* len)token,uint16_t * len,uint8_t * type)407 __packet_opt_get_token(const struct __packet_opt *po,
408 void *__sized_by(*len)token,
409 uint16_t *len, uint8_t *type)
410 {
411 uint16_t tlen = po->__po_token_len;
412 uint8_t ttype;
413
414 if (token == NULL || len == NULL || type == NULL || tlen > *len) {
415 return EINVAL;
416 }
417 ttype = (uint8_t)po->__po_token_type;
418
419 ASSERT(tlen <= PKT_OPT_MAX_TOKEN_SIZE);
420 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
421 bcopy(po->__po_token, token, tlen);
422 /*
423 * -fbounds-safety: Updating *len should be fine because at this point
424 * we know tlen is less than or equal to *len (check the first if
425 * statement in this function)
426 */
427 *len = tlen;
428 *type = ttype;
429 return 0;
430 }
431
432 __attribute__((always_inline))
433 static inline errno_t
__packet_get_token(const uint64_t ph,void * __sized_by (* len)token,uint16_t * len)434 __packet_get_token(const uint64_t ph,
435 void *__sized_by(*len)token, uint16_t *len)
436 {
437 #ifdef KERNEL
438 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
439 #else /* !KERNEL */
440 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
441 #endif /* !KERNEL */
442 uint8_t type;
443 errno_t err;
444
445 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
446 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
447 return ENOENT;
448 }
449 err = __packet_opt_get_token(po, token, len, &type);
450 if ((err == 0) && (type != PKT_OPT_TOKEN_TYPE_OPAQUE)) {
451 err = ENOENT;
452 }
453 return err;
454 }
455
456 __attribute__((always_inline))
457 static inline errno_t
__packet_opt_set_token(struct __packet_opt * po,const void * __sized_by (PKT_OPT_MAX_TOKEN_SIZE)token,const uint16_t len,const uint8_t type,volatile uint64_t * pflags)458 __packet_opt_set_token(struct __packet_opt *po,
459 const void *__sized_by(PKT_OPT_MAX_TOKEN_SIZE)token,
460 const uint16_t len, const uint8_t type, volatile uint64_t *pflags)
461 {
462 _CASSERT((__builtin_offsetof(struct __packet_opt, __po_token) % 8) == 0);
463 if (len != 0) {
464 if (token == NULL || len > PKT_OPT_MAX_TOKEN_SIZE ||
465 type == 0) {
466 return EINVAL;
467 }
468 if (__probable(IS_P2ALIGNED(token, 8))) {
469 uint64_t *token64 = __DECONST(void *, token);
470 po->__po_token_data[0] = *token64;
471 po->__po_token_data[1] = *(token64 + 1);
472 } else {
473 bcopy(token, po->__po_token, len);
474 }
475 po->__po_token_len = len;
476 po->__po_token_type = type;
477 *pflags |= PKT_F_OPT_TOKEN;
478 } else {
479 _CASSERT(sizeof(po->__po_token_data[0]) == 8);
480 _CASSERT(sizeof(po->__po_token_data[1]) == 8);
481 _CASSERT(sizeof(po->__po_token) == 16);
482 po->__po_token_data[0] = 0;
483 po->__po_token_data[1] = 0;
484 po->__po_token_len = 0;
485 po->__po_token_type = 0;
486 *pflags &= ~PKT_F_OPT_TOKEN;
487 }
488 return 0;
489 }
490
491 #ifndef KERNEL
492 __attribute__((always_inline))
493 static inline void
__packet_set_tx_timestamp(const uint64_t ph,const uint64_t ts)494 __packet_set_tx_timestamp(const uint64_t ph, const uint64_t ts)
495 {
496 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
497 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
498
499 po->__po_pkt_tx_time = ts;
500 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_TX_TIMESTAMP;
501 }
502 #endif /* !KERNEL */
503
504 __attribute__((always_inline))
505 static inline errno_t
__packet_set_token(const uint64_t ph,const void * __sized_by (len)token,const uint16_t len)506 __packet_set_token(const uint64_t ph,
507 const void *__sized_by(len)token, const uint16_t len)
508 {
509 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
510 #ifdef KERNEL
511 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, token, len,
512 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
513 #else /* !KERNEL */
514 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, token, len,
515 PKT_OPT_TOKEN_TYPE_OPAQUE, &PKT_ADDR(ph)->pkt_pflags);
516 #endif /* !KERNEL */
517 }
518
519 __attribute__((always_inline))
520 static inline errno_t
__packet_get_packetid(const uint64_t ph,packet_id_t * pktid)521 __packet_get_packetid(const uint64_t ph, packet_id_t *pktid)
522 {
523 #ifdef KERNEL
524 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
525 #else /* !KERNEL */
526 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
527 #endif /* !KERNEL */
528 uint16_t len = sizeof(packet_id_t);
529 uint8_t type;
530 errno_t err;
531
532
533 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
534 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) == 0) {
535 return ENOENT;
536 }
537 err = __packet_opt_get_token(po, (packet_id_t * __header_indexable)pktid,
538 &len, &type);
539 if ((err == 0) && ((type != PKT_OPT_TOKEN_TYPE_PACKET_ID) ||
540 (len != sizeof(packet_id_t)))) {
541 err = ENOENT;
542 }
543 return err;
544 }
545
546 __attribute__((always_inline))
547 static inline errno_t
__packet_set_packetid(const uint64_t ph,const packet_id_t * pktid)548 __packet_set_packetid(const uint64_t ph, const packet_id_t *pktid)
549 {
550 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
551 #ifdef KERNEL
552 return __packet_opt_set_token(PKT_ADDR(ph)->pkt_com_opt, pktid,
553 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
554 &PKT_ADDR(ph)->pkt_pflags);
555 #else /* !KERNEL */
556 return __packet_opt_set_token(&PKT_ADDR(ph)->pkt_com_opt, pktid,
557 sizeof(packet_id_t), PKT_OPT_TOKEN_TYPE_PACKET_ID,
558 &PKT_ADDR(ph)->pkt_pflags);
559 #endif /* !KERNEL */
560 }
561
562 __attribute__((always_inline))
563 static inline errno_t
__packet_get_vlan_tag(const uint64_t ph,uint16_t * vlan_tag,boolean_t * tag_in_pkt)564 __packet_get_vlan_tag(const uint64_t ph, uint16_t *vlan_tag,
565 boolean_t *tag_in_pkt)
566 {
567 #ifdef KERNEL
568 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
569 #else /* !KERNEL */
570 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
571 #endif /* !KERNEL */
572 uint64_t pflags;
573
574 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
575 pflags = PKT_ADDR(ph)->pkt_pflags;
576 if ((pflags & PKT_F_OPT_VLTAG) == 0) {
577 return ENOENT;
578 }
579 if (vlan_tag != NULL) {
580 *vlan_tag = po->__po_vlan_tag;
581 }
582 if (tag_in_pkt != NULL) {
583 *tag_in_pkt = ((pflags & PKT_F_OPT_VLTAG_IN_PKT) != 0);
584 }
585 return 0;
586 }
587
588 __attribute__((always_inline))
589 static inline errno_t
__packet_set_vlan_tag(const uint64_t ph,const uint16_t vlan_tag,const boolean_t tag_in_pkt)590 __packet_set_vlan_tag(const uint64_t ph, const uint16_t vlan_tag,
591 const boolean_t tag_in_pkt)
592 {
593 #ifdef KERNEL
594 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
595 #else /* !KERNEL */
596 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
597 #endif /* !KERNEL */
598
599 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
600 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG;
601 po->__po_vlan_tag = vlan_tag;
602
603 if (tag_in_pkt) {
604 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_VLTAG_IN_PKT;
605 }
606 return 0;
607 }
608
609 __attribute__((always_inline))
610 static inline uint16_t
__packet_get_vlan_id(const uint16_t vlan_tag)611 __packet_get_vlan_id(const uint16_t vlan_tag)
612 {
613 return EVL_VLANOFTAG(vlan_tag);
614 }
615
616 __attribute__((always_inline))
617 static inline uint8_t
__packet_get_vlan_priority(const uint16_t vlan_tag)618 __packet_get_vlan_priority(const uint16_t vlan_tag)
619 {
620 return EVL_PRIOFTAG(vlan_tag);
621 }
622
623 __attribute__((always_inline))
624 static inline errno_t
__packet_get_app_metadata(const uint64_t ph,packet_app_metadata_type_t * app_type,uint8_t * app_metadata)625 __packet_get_app_metadata(const uint64_t ph,
626 packet_app_metadata_type_t *app_type, uint8_t *app_metadata)
627 {
628 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
629 if (app_type == NULL || app_metadata == NULL) {
630 return EINVAL;
631 }
632 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_APP_METADATA) == 0) {
633 return ENOENT;
634 }
635 #ifdef KERNEL
636 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
637 #else /* !KERNEL */
638 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
639 #endif /* !KERNEL */
640 if (po->__po_app_type == PACKET_APP_METADATA_TYPE_UNSPECIFIED) {
641 return ENOENT;
642 }
643 *app_type = po->__po_app_type;
644 *app_metadata = po->__po_app_metadata;
645 return 0;
646 }
647
648 __attribute__((always_inline))
649 static inline errno_t
__packet_set_app_metadata(const uint64_t ph,const packet_app_metadata_type_t app_type,const uint8_t app_metadata)650 __packet_set_app_metadata(const uint64_t ph,
651 const packet_app_metadata_type_t app_type, const uint8_t app_metadata)
652 {
653 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
654 #ifdef KERNEL
655 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
656 #else /* !KERNEL */
657 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
658 #endif /* !KERNEL */
659 if (app_type < PACKET_APP_METADATA_TYPE_MIN ||
660 app_type > PACKET_APP_METADATA_TYPE_MAX) {
661 po->__po_app_type = PACKET_APP_METADATA_TYPE_UNSPECIFIED;
662 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_OPT_APP_METADATA;
663 return EINVAL;
664 }
665 po->__po_app_type = app_type;
666 po->__po_app_metadata = app_metadata;
667 PKT_ADDR(ph)->pkt_pflags |= PKT_F_OPT_APP_METADATA;
668 return 0;
669 }
670
671 #ifdef KERNEL
672 __attribute__((always_inline))
673 static inline void
__packet_set_wake_flag(const uint64_t ph)674 __packet_set_wake_flag(const uint64_t ph)
675 {
676 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
677 PKT_ADDR(ph)->pkt_pflags |= PKT_F_WAKE_PKT;
678 }
679 #endif
680
681 __attribute__((always_inline))
682 static inline boolean_t
__packet_get_wake_flag(const uint64_t ph)683 __packet_get_wake_flag(const uint64_t ph)
684 {
685 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_WAKE_PKT) != 0;
686 }
687
688 __attribute__((always_inline))
689 static inline void
__packet_set_keep_alive(const uint64_t ph,const boolean_t is_keep_alive)690 __packet_set_keep_alive(const uint64_t ph, const boolean_t is_keep_alive)
691 {
692 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
693 if (is_keep_alive) {
694 PKT_ADDR(ph)->pkt_pflags |= PKT_F_KEEPALIVE;
695 } else {
696 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_KEEPALIVE;
697 }
698 }
699
700 __attribute__((always_inline))
701 static inline boolean_t
__packet_get_keep_alive(const uint64_t ph)702 __packet_get_keep_alive(const uint64_t ph)
703 {
704 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_KEEPALIVE) != 0;
705 }
706
707 __attribute__((always_inline))
708 static inline boolean_t
__packet_get_truncated(const uint64_t ph)709 __packet_get_truncated(const uint64_t ph)
710 {
711 PKT_SUBTYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET, NEXUS_META_SUBTYPE_RAW);
712 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_TRUNCATED) != 0;
713 }
714
715 #ifdef KERNEL
716 __attribute__((always_inline))
717 static inline boolean_t
__packet_get_transport_new_flow(const uint64_t ph)718 __packet_get_transport_new_flow(const uint64_t ph)
719 {
720 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
721 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_NEW_FLOW) != 0;
722 }
723
724 __attribute__((always_inline))
725 static inline boolean_t
__packet_get_transport_last_packet(const uint64_t ph)726 __packet_get_transport_last_packet(const uint64_t ph)
727 {
728 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
729 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_LAST_PKT) != 0;
730 }
731
732 __attribute__((always_inline))
733 static inline boolean_t
__packet_get_l4s_flag(const uint64_t ph)734 __packet_get_l4s_flag(const uint64_t ph)
735 {
736 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
737 return (PKT_ADDR(ph)->pkt_pflags & PKT_F_L4S) != 0;
738 }
739 #endif /* KERNEL */
740
741 __attribute__((always_inline))
742 static inline void
__packet_set_l4s_flag(const uint64_t ph)743 __packet_set_l4s_flag(const uint64_t ph)
744 {
745 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
746 PKT_ADDR(ph)->pkt_pflags |= PKT_F_L4S;
747 }
748
749 __attribute__((always_inline))
750 static inline int
__packet_set_service_class(const uint64_t ph,const uint32_t sc)751 __packet_set_service_class(const uint64_t ph, const uint32_t sc)
752 {
753 int err = 0;
754
755 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
756
757 switch (sc) {
758 case PKT_SC_BE:
759 case PKT_SC_BK_SYS:
760 case PKT_SC_BK:
761 case PKT_SC_RD:
762 case PKT_SC_OAM:
763 case PKT_SC_AV:
764 case PKT_SC_RV:
765 case PKT_SC_VI:
766 case PKT_SC_SIG:
767 case PKT_SC_VO:
768 case PKT_SC_CTL:
769 QUM_ADDR(ph)->qum_svc_class = sc;
770 break;
771
772 default:
773 err = EINVAL;
774 break;
775 }
776
777 return err;
778 }
779
780 __attribute__((always_inline))
781 static inline uint32_t
__packet_get_service_class(const uint64_t ph)782 __packet_get_service_class(const uint64_t ph)
783 {
784 uint32_t sc;
785
786 _CASSERT(sizeof(QUM_ADDR(ph)->qum_svc_class == sizeof(uint32_t)));
787
788 switch (QUM_ADDR(ph)->qum_svc_class) {
789 case PKT_SC_BE: /* most likely best effort */
790 case PKT_SC_BK_SYS:
791 case PKT_SC_BK:
792 case PKT_SC_RD:
793 case PKT_SC_OAM:
794 case PKT_SC_AV:
795 case PKT_SC_RV:
796 case PKT_SC_VI:
797 case PKT_SC_SIG:
798 case PKT_SC_VO:
799 case PKT_SC_CTL:
800 sc = QUM_ADDR(ph)->qum_svc_class;
801 break;
802
803 default:
804 sc = PKT_SC_BE;
805 break;
806 }
807
808 return sc;
809 }
810
811 __attribute__((always_inline))
812 static inline errno_t
__packet_set_comp_gencnt(const uint64_t ph,const uint32_t gencnt)813 __packet_set_comp_gencnt(const uint64_t ph, const uint32_t gencnt)
814 {
815 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
816 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
817
818 PKT_ADDR(ph)->pkt_comp_gencnt = gencnt;
819
820 return 0;
821 }
822
823 __attribute__((always_inline))
824 static inline errno_t
__packet_get_comp_gencnt(const uint64_t ph,uint32_t * pgencnt)825 __packet_get_comp_gencnt(const uint64_t ph, uint32_t *pgencnt)
826 {
827 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_comp_gencnt == sizeof(uint32_t)));
828 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
829
830 if (pgencnt == NULL) {
831 return EINVAL;
832 }
833
834 if (PKT_ADDR(ph)->pkt_comp_gencnt == 0) {
835 return ENOENT;
836 }
837
838 *pgencnt = PKT_ADDR(ph)->pkt_comp_gencnt;
839 return 0;
840 }
841
842 __attribute__((always_inline))
843 static inline int
__packet_set_traffic_class(const uint64_t ph,const uint32_t tc)844 __packet_set_traffic_class(const uint64_t ph, const uint32_t tc)
845 {
846 uint32_t val = PKT_TC2SCVAL(tc); /* just the val portion */
847 uint32_t sc;
848
849 switch (val) {
850 case PKT_SCVAL_BK_SYS:
851 sc = PKT_SC_BK_SYS;
852 break;
853 case PKT_SCVAL_BK:
854 sc = PKT_SC_BK;
855 break;
856 case PKT_SCVAL_BE:
857 sc = PKT_SC_BE;
858 break;
859 case PKT_SCVAL_RD:
860 sc = PKT_SC_RD;
861 break;
862 case PKT_SCVAL_OAM:
863 sc = PKT_SC_OAM;
864 break;
865 case PKT_SCVAL_AV:
866 sc = PKT_SC_AV;
867 break;
868 case PKT_SCVAL_RV:
869 sc = PKT_SC_RV;
870 break;
871 case PKT_SCVAL_VI:
872 sc = PKT_SC_VI;
873 break;
874 case PKT_SCVAL_SIG:
875 sc = PKT_SC_SIG;
876 break;
877 case PKT_SCVAL_VO:
878 sc = PKT_SC_VO;
879 break;
880 case PKT_SCVAL_CTL:
881 sc = PKT_SC_CTL;
882 break;
883 default:
884 sc = PKT_SC_BE;
885 break;
886 }
887
888 return __packet_set_service_class(ph, sc);
889 }
890
891 __attribute__((always_inline))
892 static inline uint32_t
__packet_get_traffic_class(const uint64_t ph)893 __packet_get_traffic_class(const uint64_t ph)
894 {
895 return PKT_SC2TC(__packet_get_service_class(ph));
896 }
897
898 __attribute__((always_inline))
899 static inline int
__packet_set_inet_checksum(const uint64_t ph,const packet_csum_flags_t flags,const uint16_t start,const uint16_t stuff_val,boolean_t tx)900 __packet_set_inet_checksum(const uint64_t ph, const packet_csum_flags_t flags,
901 const uint16_t start, const uint16_t stuff_val, boolean_t tx)
902 {
903 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
904
905 PKT_ADDR(ph)->pkt_csum_flags = flags & PACKET_CSUM_FLAGS;
906
907 if (tx) {
908 PKT_ADDR(ph)->pkt_csum_tx_start_off = start;
909 PKT_ADDR(ph)->pkt_csum_tx_stuff_off = stuff_val;
910 } else {
911 PKT_ADDR(ph)->pkt_csum_rx_start_off = start;
912 PKT_ADDR(ph)->pkt_csum_rx_value = stuff_val;
913 }
914 return 0;
915 }
916
917 __attribute__((always_inline))
918 static inline void
__packet_add_inet_csum_flags(const uint64_t ph,const packet_csum_flags_t flags)919 __packet_add_inet_csum_flags(const uint64_t ph, const packet_csum_flags_t flags)
920 {
921 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
922
923 PKT_ADDR(ph)->pkt_csum_flags |= flags & PACKET_CSUM_FLAGS;
924 }
925
926 __attribute__((always_inline))
927 static inline packet_csum_flags_t
__packet_get_inet_checksum(const uint64_t ph,uint16_t * start,uint16_t * stuff_val,boolean_t tx)928 __packet_get_inet_checksum(const uint64_t ph, uint16_t *start,
929 uint16_t *stuff_val, boolean_t tx)
930 {
931 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
932
933 if (tx) {
934 if (__probable(start != NULL)) {
935 *start = PKT_ADDR(ph)->pkt_csum_tx_start_off;
936 }
937 if (__probable(stuff_val != NULL)) {
938 *stuff_val = PKT_ADDR(ph)->pkt_csum_tx_stuff_off;
939 }
940 } else {
941 if (__probable(start != NULL)) {
942 *start = PKT_ADDR(ph)->pkt_csum_rx_start_off;
943 }
944 if (__probable(stuff_val != NULL)) {
945 *stuff_val = PKT_ADDR(ph)->pkt_csum_rx_value;
946 }
947 }
948 return PKT_ADDR(ph)->pkt_csum_flags & PACKET_CSUM_FLAGS;
949 }
950
951 __attribute__((always_inline))
952 static inline void
__packet_set_flow_uuid(const uint64_t ph,const uuid_t flow_uuid)953 __packet_set_flow_uuid(const uint64_t ph, const uuid_t flow_uuid)
954 {
955 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
956
957 /*
958 * Anticipate a nicely (8-bytes) aligned UUID from caller;
959 * the one in qum_flow_id is always 8-byte aligned.
960 */
961 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
962 const uint64_t *id_64 = (const uint64_t *)(const void *)flow_uuid;
963 q->__q_flow_id_val64[0] = id_64[0];
964 q->__q_flow_id_val64[1] = id_64[1];
965 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
966 const uint32_t *id_32 = (const uint32_t *)(const void *)flow_uuid;
967 q->__q_flow_id_val32[0] = id_32[0];
968 q->__q_flow_id_val32[1] = id_32[1];
969 q->__q_flow_id_val32[2] = id_32[2];
970 q->__q_flow_id_val32[3] = id_32[3];
971 } else {
972 bcopy(flow_uuid, q->__q_flow_id, sizeof(uuid_t));
973 }
974 }
975
976 __attribute__((always_inline))
977 static inline void
__packet_get_flow_uuid(const uint64_t ph,uuid_t flow_uuid)978 __packet_get_flow_uuid(const uint64_t ph, uuid_t flow_uuid)
979 {
980 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
981
982 /*
983 * Anticipate a nicely (8-bytes) aligned UUID from caller;
984 * the one in qum_flow_id is always 8-byte aligned.
985 */
986 if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint64_t)))) {
987 uint64_t *id_64 = (uint64_t *)(void *)flow_uuid;
988 id_64[0] = q->__q_flow_id_val64[0];
989 id_64[1] = q->__q_flow_id_val64[1];
990 } else if (__probable(IS_P2ALIGNED(flow_uuid, sizeof(uint32_t)))) {
991 uint32_t *id_32 = (uint32_t *)(void *)flow_uuid;
992 id_32[0] = q->__q_flow_id_val32[0];
993 id_32[1] = q->__q_flow_id_val32[1];
994 id_32[2] = q->__q_flow_id_val32[2];
995 id_32[3] = q->__q_flow_id_val32[3];
996 } else {
997 bcopy(q->__q_flow_id, flow_uuid, sizeof(uuid_t));
998 }
999 }
1000
1001 __attribute__((always_inline))
1002 static inline void
__packet_clear_flow_uuid(const uint64_t ph)1003 __packet_clear_flow_uuid(const uint64_t ph)
1004 {
1005 struct __quantum *q = &QUM_ADDR(ph)->qum_com;
1006 q->__q_flow_id_val64[0] = 0;
1007 q->__q_flow_id_val64[1] = 0;
1008 }
1009
1010 __attribute__((always_inline))
1011 static inline uint8_t
__packet_get_aggregation_type(const uint64_t ph)1012 __packet_get_aggregation_type(const uint64_t ph)
1013 {
1014 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_aggr_type == sizeof(uint8_t)));
1015 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1016
1017 return PKT_ADDR(ph)->pkt_aggr_type;
1018 }
1019
1020 __attribute__((always_inline))
1021 static inline uint32_t
__packet_get_data_length(const uint64_t ph)1022 __packet_get_data_length(const uint64_t ph)
1023 {
1024 return QUM_ADDR(ph)->qum_len;
1025 }
1026
1027 #ifdef KERNEL
1028 /*
1029 * This handles truncated packets used in compat Tx and Rx classification.
1030 */
1031 __attribute__((always_inline))
1032 static inline uint32_t
__packet_get_real_data_length(const struct __kern_packet * pkt)1033 __packet_get_real_data_length(const struct __kern_packet *pkt)
1034 {
1035 uint32_t pkt_len;
1036
1037 if (pkt->pkt_pflags & PKT_F_TRUNCATED) {
1038 struct __kern_buflet *bft;
1039
1040 bft = kern_packet_get_next_buflet(SK_PKT2PH(pkt), NULL);
1041 pkt_len = kern_buflet_get_data_length(bft);
1042 } else {
1043 pkt_len = pkt->pkt_length;
1044 }
1045 return pkt_len;
1046 }
1047 #endif /* KERNEL */
1048
1049 __attribute__((always_inline))
1050 static inline uint16_t
__packet_get_buflet_count(const uint64_t ph)1051 __packet_get_buflet_count(const uint64_t ph)
1052 {
1053 uint16_t bcnt = 0;
1054
1055 switch (SK_PTR_TYPE(ph)) {
1056 case NEXUS_META_TYPE_PACKET:
1057 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1058 #ifdef KERNEL
1059 VERIFY(bcnt != 0 ||
1060 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1061 #else /* !KERNEL */
1062 /*
1063 * Handle the case where the metadata region gets
1064 * redirected to anonymous zero-filled pages at
1065 * defunct time. There's always 1 buflet in the
1066 * packet metadata, so pretend that's the count.
1067 */
1068 if (__improbable(bcnt == 0)) {
1069 bcnt = 1;
1070 }
1071 #endif /* !KERNEL */
1072 break;
1073 case NEXUS_META_TYPE_QUANTUM:
1074 bcnt = 1;
1075 break;
1076 default:
1077 #ifdef KERNEL
1078 VERIFY(0);
1079 /* NOTREACHED */
1080 __builtin_unreachable();
1081 #endif /* KERNEL */
1082 break;
1083 }
1084 return bcnt;
1085 }
1086
1087 __attribute__((always_inline))
1088 static inline int
__packet_add_buflet(const uint64_t ph,const void * bprev0,const void * bnew0)1089 __packet_add_buflet(const uint64_t ph, const void *bprev0, const void *bnew0)
1090 {
1091 uint16_t bcnt;
1092
1093 #ifdef KERNEL
1094 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1095 kern_buflet_t bnew = __DECONST(kern_buflet_t, bnew0);
1096
1097 VERIFY(PKT_ADDR(ph) && bnew && (bnew != bprev));
1098 VERIFY(PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1099 #else /* !KERNEL */
1100 buflet_t bprev = __DECONST(buflet_t, bprev0);
1101 buflet_t bnew = __DECONST(buflet_t, bnew0);
1102
1103 if (__improbable(!PKT_ADDR(ph) || !bnew || (bnew == bprev))) {
1104 return EINVAL;
1105 }
1106 #endif /* !KERNEL */
1107
1108 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1109 bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1110
1111 #ifdef KERNEL
1112 VERIFY((bprev != NULL || bcnt == 0) &&
1113 (bcnt < PKT_ADDR(ph)->pkt_bufs_max));
1114 #else /* !KERNEL */
1115 if (__improbable(bcnt >= PKT_ADDR(ph)->pkt_bufs_max) ||
1116 (bprev == NULL && bcnt != 0)) {
1117 return EINVAL;
1118 }
1119 #endif /* !KERNEL */
1120
1121 #ifdef KERNEL
1122 #if DEVELOPMENT || DEBUG
1123 /* check if bprev is the last buflet in the chain */
1124 struct __kern_buflet *__single pbft, *__single kbft;
1125 int n = bcnt;
1126
1127 PKT_GET_FIRST_BUFLET(PKT_ADDR(ph), bcnt, pbft);
1128 kbft = pbft;
1129
1130 while ((kbft != NULL) && n--) {
1131 pbft = kbft;
1132 kbft = __unsafe_forge_single(struct __kern_buflet *,
1133 __DECONST(struct __kern_buflet *, kbft->buf_nbft_addr));
1134 }
1135 ASSERT(n == 0);
1136 ASSERT(bprev == pbft);
1137 #endif /* DEVELOPMENT || DEBUG */
1138 #endif /* KERNEL */
1139
1140 if (bprev == NULL) {
1141 bprev = &PKT_ADDR(ph)->pkt_qum_buf;
1142 }
1143 #ifdef KERNEL
1144 KBUF_LINK(bprev, bnew);
1145 #else /* !KERNEL */
1146 UBUF_LINK(bprev, bnew);
1147 #endif /* !KERNEL */
1148
1149 *(uint16_t *)(uintptr_t)&PKT_ADDR(ph)->pkt_bufs_cnt = ++bcnt;
1150 return 0;
1151 }
1152
1153 __attribute__((always_inline))
1154 #ifdef KERNEL
1155 static inline struct __kern_buflet *
1156 #else
1157 static inline struct __user_buflet *
1158 #endif
__packet_get_next_buflet(const uint64_t ph,const void * bprev0)1159 __packet_get_next_buflet(const uint64_t ph, const void *bprev0)
1160 {
1161 #ifdef KERNEL
1162 kern_buflet_t bprev = __DECONST(kern_buflet_t, bprev0);
1163 struct __kern_buflet *__single bcur = NULL;
1164 #else /* !KERNEL */
1165 buflet_t bprev = __DECONST(buflet_t, bprev0);
1166 void *bcur = NULL;
1167 #endif /* !KERNEL */
1168
1169 switch (SK_PTR_TYPE(ph)) {
1170 case NEXUS_META_TYPE_PACKET: {
1171 uint32_t bcnt = PKT_ADDR(ph)->pkt_bufs_cnt;
1172 #ifdef KERNEL
1173 ASSERT(bcnt != 0 ||
1174 PP_HAS_BUFFER_ON_DEMAND(PKT_ADDR(ph)->pkt_qum.qum_pp));
1175 #else /* !KERNEL */
1176 /*
1177 * Handle the case where the metadata region gets
1178 * redirected to anonymous zero-filled pages at
1179 * defunct time. There's always 1 buflet in the
1180 * packet metadata, so pretend that's the count.
1181 */
1182 if (__improbable(bcnt == 0)) {
1183 bcnt = 1;
1184 bprev = NULL;
1185 }
1186 #endif /* !KERNEL */
1187 PKT_GET_NEXT_BUFLET(PKT_ADDR(ph), bcnt, BLT_ADDR(bprev), bcur);
1188 break;
1189 }
1190 case NEXUS_META_TYPE_QUANTUM:
1191 QUM_GET_NEXT_BUFLET(QUM_ADDR(ph), BLT_ADDR(bprev), bcur);
1192 break;
1193 default:
1194 #ifdef KERNEL
1195 VERIFY(0);
1196 /* NOTREACHED */
1197 __builtin_unreachable();
1198 #endif /* KERNEL */
1199 break;
1200 }
1201 return bcur;
1202 }
1203
1204 __attribute__((always_inline))
1205 static inline uint8_t
__packet_get_segment_count(const uint64_t ph)1206 __packet_get_segment_count(const uint64_t ph)
1207 {
1208 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1209 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1210
1211 return PKT_ADDR(ph)->pkt_seg_cnt;
1212 }
1213
1214 __attribute__((always_inline))
1215 static inline void
__packet_set_segment_count(const uint64_t ph,uint8_t segcount)1216 __packet_set_segment_count(const uint64_t ph, uint8_t segcount)
1217 {
1218 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_seg_cnt == sizeof(uint8_t)));
1219 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1220
1221 PKT_ADDR(ph)->pkt_seg_cnt = segcount;
1222 }
1223
1224 __attribute__((always_inline))
1225 static inline uint16_t
__packet_get_protocol_segment_size(const uint64_t ph)1226 __packet_get_protocol_segment_size(const uint64_t ph)
1227 {
1228 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1229 return PKT_ADDR(ph)->pkt_proto_seg_sz;
1230 }
1231
1232 __attribute__((always_inline))
1233 static inline errno_t
__packet_set_protocol_segment_size(const uint64_t ph,uint16_t proto_seg_sz)1234 __packet_set_protocol_segment_size(const uint64_t ph, uint16_t proto_seg_sz)
1235 {
1236 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1237 PKT_ADDR(ph)->pkt_proto_seg_sz = proto_seg_sz;
1238 return 0;
1239 }
1240
1241 __attribute__((always_inline))
1242 static inline void
__packet_get_tso_flags(const uint64_t ph,packet_tso_flags_t * flags)1243 __packet_get_tso_flags(const uint64_t ph, packet_tso_flags_t *flags)
1244 {
1245 _CASSERT(sizeof(PKT_ADDR(ph)->pkt_proto_seg_sz == sizeof(uint16_t)));
1246
1247 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1248 *flags = PKT_ADDR(ph)->pkt_csum_flags & (PACKET_CSUM_TSO_FLAGS);
1249 }
1250
1251 __attribute__((always_inline))
1252 static inline void
__packet_set_tso_flags(const uint64_t ph,packet_tso_flags_t flags)1253 __packet_set_tso_flags(const uint64_t ph, packet_tso_flags_t flags)
1254 {
1255 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1256
1257 PKT_ADDR(ph)->pkt_csum_flags |= flags & (PACKET_CSUM_TSO_FLAGS);
1258 }
1259
1260 __attribute__((always_inline))
1261 static inline uint32_t
__buflet_get_data_limit(const void * buf)1262 __buflet_get_data_limit(const void *buf)
1263 {
1264 return BLT_ADDR(buf)->buf_dlim;
1265 }
1266
1267 #ifdef KERNEL
1268 __attribute__((always_inline))
1269 static inline errno_t
__buflet_set_data_limit(const void * buf,const uint32_t dlim)1270 __buflet_set_data_limit(const void *buf, const uint32_t dlim)
1271 {
1272 /* buffer region is always marked as shareable */
1273 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1274
1275 /* full bounds checking will be performed during finalize */
1276 if (__probable((uint32_t)dlim <= BLT_ADDR(buf)->buf_objlim)) {
1277 _CASSERT(sizeof(BLT_ADDR(buf)->buf_dlim) == sizeof(uint32_t));
1278 /* deconst */
1279 *(uint32_t *)(uintptr_t)&BLT_ADDR(buf)->buf_dlim = dlim;
1280 return 0;
1281 }
1282 return ERANGE;
1283 }
1284 #endif /* KERNEL */
1285
1286 __attribute__((always_inline))
1287 static inline uint32_t
__buflet_get_data_offset(const void * buf)1288 __buflet_get_data_offset(const void *buf)
1289 {
1290 return BLT_ADDR(buf)->buf_doff;
1291 }
1292
1293 /*
1294 * ******************************************************************
1295 * Checks in __packet_finalize for packet finalized from userland
1296 * ******************************************************************
1297 * +-------+---------------------------+---------------------------+
1298 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1299 * |-------+---------------------------+---------------------------+
1300 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1301 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1302 * |l3_off | l3 = l2 | l3 == 0 |
1303 * |l4_off | l4 = l3 = l2 | l4 = l3 = 0 |
1304 * +-------+---------------------------+---------------------------+
1305 *
1306 * ******************************************************************
1307 * Checks in __packet_finalize for packet finalized from kernel
1308 * ******************************************************************
1309 * +-------+---------------------------+---------------------------+
1310 * | NEXUS_META_SUBTYPE_RAW | NEXUS_META_SUBTYPE_PAYLOAD|
1311 * |-------+---------------------------+---------------------------+
1312 * |buflet | (bdoff + len) <= dlim | (bdoff + len) <= dlim |
1313 * |l2_off | l2 == bdoff && l2 < bdlim | l2 = l3 = 0 && doff == 0 |
1314 * |l3_off | l3 >= l2 && l3 <bdlim | l3 == 0 |
1315 * |l4_off | l4 = l3 | l4 = l3 = 0 |
1316 * +-------+---------------------------+---------------------------+
1317 *
1318 */
1319 __attribute__((always_inline))
1320 static inline int
__packet_finalize(const uint64_t ph)1321 __packet_finalize(const uint64_t ph)
1322 {
1323 void *__single bcur = NULL, *__single bprev = NULL;
1324 uint32_t len, bcnt, bdoff0, bdlim0;
1325 int err = 0;
1326
1327 #ifdef KERNEL
1328 ASSERT(QUM_ADDR(ph)->qum_qflags & QUM_F_INTERNALIZED);
1329 #endif /* KERNEL */
1330 QUM_ADDR(ph)->qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1331
1332 bcnt = __packet_get_buflet_count(ph);
1333 len = QUM_ADDR(ph)->qum_len = 0;
1334
1335 while (bcnt--) {
1336 bcur = __packet_get_next_buflet(ph, bprev);
1337
1338 #ifdef KERNEL
1339 ASSERT(bcur != NULL);
1340 ASSERT(BLT_ADDR(bcur)->buf_addr != 0);
1341 #else /* !KERNEL */
1342 if (__improbable(bcur == NULL)) {
1343 err = ERANGE;
1344 break;
1345 }
1346 #endif /* KERNEL */
1347
1348 /* save data offset from the first buflet */
1349 if (bprev == NULL) {
1350 bdoff0 = __buflet_get_data_offset(bcur);
1351 bdlim0 = __buflet_get_data_limit(bcur);
1352 }
1353
1354 #ifndef KERNEL
1355 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)))) {
1356 err = ERANGE;
1357 break;
1358 }
1359 #else /* !KERNEL */
1360 if (__improbable(!BUF_IN_RANGE(BLT_ADDR(bcur)) &&
1361 !PKT_HAS_ATTACHED_MBUF(ph))) {
1362 err = ERANGE;
1363 break;
1364 }
1365 #endif /* KERNEL */
1366 len += BLT_ADDR(bcur)->buf_dlen;
1367 bprev = bcur;
1368 }
1369
1370 if (__improbable(err != 0)) {
1371 goto done;
1372 }
1373
1374 switch (SK_PTR_TYPE(ph)) {
1375 case NEXUS_META_TYPE_PACKET:
1376 if (__improbable(bdoff0 > UINT8_MAX)) {
1377 err = ERANGE;
1378 goto done;
1379 }
1380 /* internalize headroom value from offset */
1381 PKT_ADDR(ph)->pkt_headroom = (uint8_t)bdoff0;
1382 /* validate header offsets in packet */
1383 switch (SK_PTR_SUBTYPE(ph)) {
1384 case NEXUS_META_SUBTYPE_RAW:
1385 #ifndef KERNEL
1386 /* Overwrite L2 len for raw packets from user space */
1387 PKT_ADDR(ph)->pkt_l2_len = 0;
1388 #else /* !KERNEL */
1389 /* ensure that L3 >= L2 && L3 < bdlim */
1390 if (__improbable((PKT_ADDR(ph)->pkt_headroom +
1391 PKT_ADDR(ph)->pkt_l2_len) >= bdlim0)) {
1392 err = ERANGE;
1393 goto done;
1394 }
1395 #endif /* KERNEL */
1396 break;
1397 case NEXUS_META_SUBTYPE_PAYLOAD:
1398 /*
1399 * For payload packet there is no concept of headroom
1400 * and L3 offset should always be 0
1401 */
1402 if (__improbable((PKT_ADDR(ph)->pkt_headroom != 0) ||
1403 (PKT_ADDR(ph)->pkt_l2_len != 0))) {
1404 err = ERANGE;
1405 goto done;
1406 }
1407 break;
1408 default:
1409 #ifdef KERNEL
1410 VERIFY(0);
1411 /* NOTREACHED */
1412 __builtin_unreachable();
1413 #endif /* KERNEL */
1414 break;
1415 }
1416
1417 if (__improbable(PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_DATA)) {
1418 #ifdef KERNEL
1419 struct __packet_opt *po = PKT_ADDR(ph)->pkt_com_opt;
1420 #else /* !KERNEL */
1421 struct __packet_opt *po = &PKT_ADDR(ph)->pkt_com_opt;
1422 #endif /* !KERNEL */
1423 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1424 po->__po_expire_ts == 0) {
1425 err = EINVAL;
1426 goto done;
1427 }
1428 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_OPT_TOKEN) &&
1429 po->__po_token_len == 0) {
1430 err = EINVAL;
1431 goto done;
1432 }
1433 ASSERT(err == 0);
1434 }
1435
1436 /*
1437 * NOTE: we don't need the validation for total packet length
1438 * as checking if each buflet is in range and that
1439 * (pkt_headroom == bdoff0), should cover this check.
1440 */
1441 break;
1442
1443 default:
1444 /* nothing to do currently for quantum */
1445 break;
1446 }
1447
1448 done:
1449 if (__probable(err == 0)) {
1450 QUM_ADDR(ph)->qum_len = len;
1451 QUM_ADDR(ph)->qum_qflags |= QUM_F_FINALIZED;
1452 } else {
1453 QUM_ADDR(ph)->qum_len = 0;
1454 QUM_ADDR(ph)->qum_qflags |= QUM_F_DROPPED;
1455 }
1456
1457 return err;
1458 }
1459
1460 __attribute__((always_inline))
1461 static inline boolean_t
__packet_is_finalized(const uint64_t ph)1462 __packet_is_finalized(const uint64_t ph)
1463 {
1464 return QUM_ADDR(ph)->qum_qflags & QUM_F_FINALIZED;
1465 }
1466
1467 #ifdef KERNEL
1468 /*
1469 * function to initialize a packet with mbuf chain.
1470 * Apart from the attached mbuf, the packet can also be used to convey
1471 * additional metadata like the headroom and L2 header length.
1472 * For a packet with attached mbuf, the pkt_length conveys the length of
1473 * the attached mbuf. If the data copied is partial then PKT_F_TRUNCATED is
1474 * also set.
1475 */
1476 __attribute__((always_inline))
1477 static inline int
__packet_initialize_with_mbufchain(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1478 __packet_initialize_with_mbufchain(struct __kern_packet *pkt, struct mbuf *mbuf,
1479 uint8_t headroom, uint8_t l2len)
1480 {
1481 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1482 VERIFY(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1483 VERIFY((pkt->pkt_pflags & PKT_F_MBUF_MASK) == 0);
1484 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
1485 VERIFY(pkt->pkt_mbuf == NULL);
1486
1487 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1488 pkt->pkt_mbuf = mbuf;
1489 pkt->pkt_pflags |= (PKT_F_MBUF_DATA | PKT_F_TRUNCATED);
1490 pkt->pkt_headroom = headroom;
1491 pkt->pkt_l2_len = l2len;
1492 pkt->pkt_length = m_pktlen(mbuf);
1493 pkt->pkt_qum_buf.buf_dlen = 0;
1494 pkt->pkt_qum_buf.buf_doff = 0;
1495 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1496 return 0;
1497 }
1498
1499 __attribute__((always_inline))
1500 static inline int
__packet_initialize_with_mbuf(struct __kern_packet * pkt,struct mbuf * mbuf,uint8_t headroom,uint8_t l2len)1501 __packet_initialize_with_mbuf(struct __kern_packet *pkt, struct mbuf *mbuf,
1502 uint8_t headroom, uint8_t l2len)
1503 {
1504 __packet_initialize_with_mbufchain(pkt, mbuf, headroom, l2len);
1505 VERIFY(mbuf->m_nextpkt == NULL);
1506 return 0;
1507 }
1508
1509 /*
1510 * function to finalize a packet with attached mbuf.
1511 */
1512 __attribute__((always_inline))
1513 static inline int
__packet_finalize_with_mbuf(struct __kern_packet * pkt)1514 __packet_finalize_with_mbuf(struct __kern_packet *pkt)
1515 {
1516 uint32_t bdlen, bdoff, bdlim;
1517 struct __kern_buflet *buf;
1518 int err = 0;
1519
1520 VERIFY(METADATA_TYPE(pkt) == NEXUS_META_TYPE_PACKET);
1521 VERIFY((pkt->pkt_pflags & (PKT_F_MBUF_DATA | PKT_F_PKT_DATA)) ==
1522 PKT_F_MBUF_DATA);
1523 VERIFY(pkt->pkt_mbuf != NULL);
1524 ASSERT(pkt->pkt_qum.qum_qflags & QUM_F_INTERNALIZED);
1525 VERIFY(pkt->pkt_bufs_cnt == 1);
1526 PKT_GET_FIRST_BUFLET(pkt, pkt->pkt_bufs_cnt, buf);
1527 ASSERT(buf->buf_addr != 0);
1528
1529 pkt->pkt_qum.qum_qflags &= ~(QUM_F_DROPPED | QUM_F_FINALIZED);
1530 pkt->pkt_pflags &= ~PKT_F_TRUNCATED;
1531 bdlen = buf->buf_dlen;
1532 bdlim = buf->buf_dlim;
1533 bdoff = buf->buf_doff;
1534 if (__improbable(!BUF_IN_RANGE(buf))) {
1535 err = ERANGE;
1536 goto done;
1537 }
1538
1539 /* validate header offsets in packet */
1540 switch (METADATA_SUBTYPE(pkt)) {
1541 case NEXUS_META_SUBTYPE_RAW:
1542 if (__improbable((pkt->pkt_headroom != bdoff) ||
1543 (pkt->pkt_headroom >= bdlim))) {
1544 err = ERANGE;
1545 goto done;
1546 }
1547 if (__improbable((pkt->pkt_headroom +
1548 pkt->pkt_l2_len) >= bdlim)) {
1549 err = ERANGE;
1550 goto done;
1551 }
1552 break;
1553
1554 case NEXUS_META_SUBTYPE_PAYLOAD:
1555 /*
1556 * For payload packet there is no concept of headroom.
1557 */
1558 if (__improbable((pkt->pkt_headroom != 0) || (bdoff != 0) ||
1559 (pkt->pkt_l2_len != 0))) {
1560 err = ERANGE;
1561 goto done;
1562 }
1563 break;
1564
1565 default:
1566 VERIFY(0);
1567 /* NOTREACHED */
1568 __builtin_unreachable();
1569 break;
1570 }
1571
1572
1573 if (__improbable(pkt->pkt_pflags & PKT_F_OPT_DATA)) {
1574 struct __packet_opt *po = pkt->pkt_com_opt;
1575
1576 if ((pkt->pkt_pflags & PKT_F_OPT_EXPIRE_TS) &&
1577 po->__po_expire_ts == 0) {
1578 err = EINVAL;
1579 goto done;
1580 }
1581 if ((pkt->pkt_pflags & PKT_F_OPT_TOKEN) &&
1582 po->__po_token_len == 0) {
1583 err = EINVAL;
1584 goto done;
1585 }
1586 }
1587 ASSERT(err == 0);
1588
1589 done:
1590 if (__probable(err == 0)) {
1591 pkt->pkt_length = (uint32_t)m_pktlen(pkt->pkt_mbuf);
1592 if (bdlen < pkt->pkt_length) {
1593 pkt->pkt_pflags |= PKT_F_TRUNCATED;
1594 }
1595 pkt->pkt_qum.qum_qflags |= QUM_F_FINALIZED;
1596 } else {
1597 pkt->pkt_length = 0;
1598 pkt->pkt_qum.qum_qflags |= QUM_F_DROPPED;
1599 }
1600
1601 return err;
1602 }
1603
1604 __attribute__((always_inline))
1605 static inline uint32_t
__packet_get_object_index(const uint64_t ph)1606 __packet_get_object_index(const uint64_t ph)
1607 {
1608 return METADATA_IDX(QUM_ADDR(ph));
1609 }
1610
1611 __attribute__((always_inline))
1612 static inline errno_t
__packet_get_timestamp(const uint64_t ph,uint64_t * ts,boolean_t * valid)1613 __packet_get_timestamp(const uint64_t ph, uint64_t *ts, boolean_t *valid)
1614 {
1615 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1616
1617 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TS_VALID) != 0) {
1618 if (valid != NULL) {
1619 *valid = TRUE;
1620 }
1621 *ts = PKT_ADDR(ph)->pkt_timestamp;
1622 } else {
1623 if (valid != NULL) {
1624 *valid = FALSE;
1625 }
1626 *ts = 0;
1627 }
1628
1629 return 0;
1630 }
1631
1632 __attribute__((always_inline))
1633 static inline errno_t
__packet_set_timestamp(const uint64_t ph,uint64_t ts,boolean_t valid)1634 __packet_set_timestamp(const uint64_t ph, uint64_t ts, boolean_t valid)
1635 {
1636 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1637
1638 if (valid) {
1639 PKT_ADDR(ph)->pkt_timestamp = ts;
1640 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TS_VALID;
1641 } else {
1642 PKT_ADDR(ph)->pkt_pflags &= ~PKT_F_TS_VALID;
1643 PKT_ADDR(ph)->pkt_timestamp = 0;
1644 }
1645
1646 return 0;
1647 }
1648
1649 __attribute__((always_inline))
1650 static inline errno_t
__packet_get_tx_completion_data(const uint64_t ph,uintptr_t * cb_arg,uintptr_t * cb_data)1651 __packet_get_tx_completion_data(const uint64_t ph, uintptr_t *cb_arg,
1652 uintptr_t *cb_data)
1653 {
1654 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1655 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1656 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1657 *cb_arg = PKT_ADDR(ph)->pkt_tx_compl_cb_arg;
1658 *cb_data = PKT_ADDR(ph)->pkt_tx_compl_cb_data;
1659 } else {
1660 *cb_arg = 0;
1661 *cb_data = 0;
1662 }
1663 return 0;
1664 }
1665
1666 __attribute__((always_inline))
1667 static inline errno_t
__packet_set_tx_completion_data(const uint64_t ph,uintptr_t cb_arg,uintptr_t cb_data)1668 __packet_set_tx_completion_data(const uint64_t ph, uintptr_t cb_arg,
1669 uintptr_t cb_data)
1670 {
1671 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1672 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1673 PKT_ADDR(ph)->pkt_tx_compl_cb_arg = cb_arg;
1674 PKT_ADDR(ph)->pkt_tx_compl_cb_data = cb_data;
1675 return 0;
1676 }
1677
1678 __attribute__((always_inline))
1679 static inline errno_t
__packet_get_timestamp_requested(const uint64_t ph,boolean_t * requested)1680 __packet_get_timestamp_requested(const uint64_t ph, boolean_t *requested)
1681 {
1682 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1683 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0) {
1684 *requested = TRUE;
1685 } else {
1686 *requested = FALSE;
1687 }
1688 return 0;
1689 }
1690
1691 __attribute__((always_inline))
1692 static inline errno_t
__packet_get_tx_completion_status(const uint64_t ph,kern_return_t * status)1693 __packet_get_tx_completion_status(const uint64_t ph, kern_return_t *status)
1694 {
1695 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1696 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_DATA) != 0) {
1697 ASSERT((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_COMPL_ALLOC));
1698 *status = (kern_return_t)PKT_ADDR(ph)->pkt_tx_compl_status;
1699 } else {
1700 *status = 0;
1701 }
1702 return 0;
1703 }
1704
1705 __attribute__((always_inline))
1706 static inline errno_t
__packet_set_tx_completion_status(const uint64_t ph,kern_return_t status)1707 __packet_set_tx_completion_status(const uint64_t ph, kern_return_t status)
1708 {
1709 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1710 _KPKT_INIT_TX_COMPL_DATA(PKT_ADDR(ph));
1711 PKT_ADDR(ph)->pkt_tx_compl_status = (uint32_t)status;
1712 return 0;
1713 }
1714
1715 __attribute__((always_inline))
1716 static inline errno_t
__packet_set_tx_nx_port(const uint64_t ph,nexus_port_t nx_port,uint16_t vpna_gencnt)1717 __packet_set_tx_nx_port(const uint64_t ph, nexus_port_t nx_port,
1718 uint16_t vpna_gencnt)
1719 {
1720 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1721 PKT_ADDR(ph)->pkt_nx_port = nx_port;
1722 PKT_ADDR(ph)->pkt_vpna_gencnt = vpna_gencnt;
1723 PKT_ADDR(ph)->pkt_pflags |= PKT_F_TX_PORT_DATA;
1724 return 0;
1725 }
1726
1727 __attribute__((always_inline))
1728 static inline errno_t
__packet_get_tx_nx_port(const uint64_t ph,nexus_port_t * nx_port,uint16_t * vpna_gencnt)1729 __packet_get_tx_nx_port(const uint64_t ph, nexus_port_t *nx_port,
1730 uint16_t *vpna_gencnt)
1731 {
1732 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1733 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_TX_PORT_DATA) == 0) {
1734 return ENOTSUP;
1735 }
1736
1737 *nx_port = PKT_ADDR(ph)->pkt_nx_port;
1738 *vpna_gencnt = PKT_ADDR(ph)->pkt_vpna_gencnt;
1739 return 0;
1740 }
1741
1742 __attribute__((always_inline))
1743 static inline errno_t
__packet_get_tx_nx_port_id(const uint64_t ph,uint32_t * nx_port_id)1744 __packet_get_tx_nx_port_id(const uint64_t ph, uint32_t *nx_port_id)
1745 {
1746 errno_t err;
1747 nexus_port_t nx_port;
1748 uint16_t vpna_gencnt;
1749
1750 _CASSERT(sizeof(nx_port) == sizeof(uint16_t));
1751
1752 err = __packet_get_tx_nx_port(ph, &nx_port, &vpna_gencnt);
1753 if (err == 0) {
1754 *nx_port_id = PKT_COMPOSE_NX_PORT_ID(nx_port, vpna_gencnt);
1755 }
1756 return err;
1757 }
1758
1759
1760 __attribute__((always_inline))
1761 static inline errno_t
__packet_get_flowid(const uint64_t ph,packet_flowid_t * pflowid)1762 __packet_get_flowid(const uint64_t ph, packet_flowid_t *pflowid)
1763 {
1764 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
1765 if ((PKT_ADDR(ph)->pkt_pflags & PKT_F_FLOW_ID) == 0) {
1766 return ENOENT;
1767 }
1768 *pflowid = PKT_ADDR(ph)->pkt_flow_token;
1769 return 0;
1770 }
1771 #endif /* KERNEL */
1772
1773 extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t);
1774
1775 __attribute__((always_inline))
1776 static inline uint16_t
__packet_fold_sum(uint32_t sum)1777 __packet_fold_sum(uint32_t sum)
1778 {
1779 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1780 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1781 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1782 return sum & 0xffff;
1783 }
1784
1785 __attribute__((always_inline))
1786 static inline uint16_t
__packet_fold_sum_final(uint32_t sum)1787 __packet_fold_sum_final(uint32_t sum)
1788 {
1789 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1790 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1791 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1792 return ~sum & 0xffff;
1793 }
1794
1795 __attribute__((always_inline))
1796 static inline uint32_t
__packet_cksum(const void * data,uint32_t len,uint32_t sum0)1797 __packet_cksum(const void *data, uint32_t len, uint32_t sum0)
1798 {
1799 return os_cpu_in_cksum(data, len, sum0);
1800 }
1801
1802 extern uint32_t os_cpu_copy_in_cksum(const void *__sized_by(len), void *__sized_by(len),
1803 uint32_t len, uint32_t);
1804
1805 __attribute__((always_inline))
1806 static inline uint32_t
__packet_copy_and_sum(const void * __sized_by (len)src,void * __sized_by (len)dst,uint32_t len,uint32_t sum0)1807 __packet_copy_and_sum(const void *__sized_by(len) src, void *__sized_by(len) dst,
1808 uint32_t len, uint32_t sum0)
1809 {
1810 return os_cpu_copy_in_cksum(src, dst, len, sum0);
1811 }
1812
1813 __attribute__((always_inline))
1814 static inline uint16_t
__packet_fix_sum(uint16_t csum,uint16_t old,uint16_t new)1815 __packet_fix_sum(uint16_t csum, uint16_t old, uint16_t new)
1816 {
1817 uint32_t c = csum + old - new;
1818 c = (c >> 16) + (c & 0xffff); /* Only add carry once */
1819
1820 return c & 0xffff;
1821 }
1822
1823 /* MUST be used for uint32_t fields */
1824 __attribute__((always_inline))
1825 static inline void
1826 __packet_fix_hdr_sum(uint8_t *__sized_by(4)field, uint16_t *csum, uint32_t new)
1827 {
1828 uint32_t old;
1829 memcpy(&old, field, sizeof(old));
1830 memcpy(field, &new, sizeof(uint32_t));
1831 *csum = __packet_fix_sum(__packet_fix_sum(*csum, (uint16_t)(old >> 16),
1832 (uint16_t)(new >> 16)), (uint16_t)(old & 0xffff),
1833 (uint16_t)(new & 0xffff));
1834 }
1835
1836 __attribute__((always_inline))
1837 static inline void *__header_indexable
__buflet_get_data_address(const void * buf)1838 __buflet_get_data_address(const void *buf)
1839 {
1840 return __unsafe_forge_bidi_indexable(void *, (void *)(BLT_ADDR(buf)->buf_addr),
1841 BLT_ADDR(buf)->buf_dlim);
1842 }
1843
1844 #ifdef KERNEL
1845 __attribute__((always_inline))
1846 static inline errno_t
__buflet_set_data_address(const void * buf,const void * addr)1847 __buflet_set_data_address(const void *buf, const void *addr)
1848 {
1849 /* buffer region is always marked as shareable */
1850 ASSERT(BLT_ADDR(buf)->buf_ctl->bc_flags & SKMEM_BUFCTL_SHAREOK);
1851
1852 /* full bounds checking will be performed during finalize */
1853 if (__probable((uintptr_t)addr >=
1854 (uintptr_t)BLT_ADDR(buf)->buf_objaddr)) {
1855 _CASSERT(sizeof(BLT_ADDR(buf)->buf_addr) ==
1856 sizeof(mach_vm_address_t));
1857 /* deconst */
1858 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1859 (mach_vm_address_t)addr;
1860 return 0;
1861 }
1862 return ERANGE;
1863 }
1864
1865 /*
1866 * Equivalent to __buflet_set_data_address but based on offset, packets/buflets
1867 * set with this should not be directly passed to userspace, since shared buffer
1868 * is not yet supported by user facing pool.
1869 */
1870 __attribute__((always_inline))
1871 static inline int
__buflet_set_buffer_offset(const void * buf,const uint32_t off)1872 __buflet_set_buffer_offset(const void *buf, const uint32_t off)
1873 {
1874 ASSERT(BLT_ADDR(buf)->buf_objlim != 0);
1875
1876 if (__probable(off <= BLT_ADDR(buf)->buf_objlim)) {
1877 *(mach_vm_address_t *)(uintptr_t)&BLT_ADDR(buf)->buf_addr =
1878 (mach_vm_address_t)BLT_ADDR(buf)->buf_objaddr + off;
1879 return 0;
1880 }
1881 return ERANGE;
1882 }
1883 #endif /* KERNEL */
1884
1885 __attribute__((always_inline))
1886 static inline int
__buflet_set_data_offset(const void * buf,const uint32_t doff)1887 __buflet_set_data_offset(const void *buf, const uint32_t doff)
1888 {
1889 #ifdef KERNEL
1890 /*
1891 * Kernel-specific assertion. For user space, the metadata
1892 * region gets redirected to anonymous zero-filled pages at
1893 * defunct time, so ignore it there.
1894 */
1895 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1896
1897 if (__probable((uint32_t)doff <= BLT_ADDR(buf)->buf_objlim)) {
1898 BLT_ADDR(buf)->buf_doff = doff;
1899 return 0;
1900 }
1901 return ERANGE;
1902 #else /* !KERNEL */
1903 BLT_ADDR(buf)->buf_doff = doff;
1904 return 0;
1905 #endif /* KERNEL */
1906 }
1907
1908 __attribute__((always_inline))
1909 static inline int
__buflet_set_data_length(const void * buf,const uint32_t dlen)1910 __buflet_set_data_length(const void *buf, const uint32_t dlen)
1911 {
1912 #ifdef KERNEL
1913 /*
1914 * Kernel-specific assertion. For user space, the metadata
1915 * region gets redirected to anonymous zero-filled pages at
1916 * defunct time, so ignore it there.
1917 */
1918 ASSERT(BLT_ADDR(buf)->buf_dlim != 0);
1919
1920 if (__probable((uint32_t)dlen <= BLT_ADDR(buf)->buf_objlim)) {
1921 BLT_ADDR(buf)->buf_dlen = dlen;
1922 return 0;
1923 }
1924 return ERANGE;
1925 #else /* !KERNEL */
1926 BLT_ADDR(buf)->buf_dlen = dlen;
1927 return 0;
1928 #endif /* KERNEL */
1929 }
1930
1931 __attribute__((always_inline))
1932 static inline uint32_t
__buflet_get_data_length(const void * buf)1933 __buflet_get_data_length(const void *buf)
1934 {
1935 return BLT_ADDR(buf)->buf_dlen;
1936 }
1937
1938 #ifdef KERNEL
1939 __attribute__((always_inline))
1940 static inline struct sksegment *
__buflet_get_object_segment(const void * buf,kern_obj_idx_seg_t * idx)1941 __buflet_get_object_segment(const void *buf, kern_obj_idx_seg_t *idx)
1942 {
1943 _CASSERT(sizeof(obj_idx_t) == sizeof(kern_obj_idx_seg_t));
1944
1945 if (idx != NULL) {
1946 *idx = BLT_ADDR(buf)->buf_ctl->bc_idx;
1947 }
1948
1949 return BLT_ADDR(buf)->buf_ctl->bc_slab->sl_seg;
1950 }
1951 #endif /* KERNEL */
1952
1953 __attribute__((always_inline))
1954 static inline void *
__buflet_get_object_address(const void * buf)1955 __buflet_get_object_address(const void *buf)
1956 {
1957 #ifdef KERNEL
1958 return (void *)(BLT_ADDR(buf)->buf_objaddr);
1959 #else /* !KERNEL */
1960 /*
1961 * For user space, shared buffer is not available and hence the data
1962 * address is immutable and is always the same as the underlying
1963 * buffer object address itself.
1964 */
1965 return __buflet_get_data_address(buf);
1966 #endif /* !KERNEL */
1967 }
1968
1969 __attribute__((always_inline))
1970 static inline uint32_t
__buflet_get_object_limit(const void * buf)1971 __buflet_get_object_limit(const void *buf)
1972 {
1973 #ifdef KERNEL
1974 return BLT_ADDR(buf)->buf_objlim;
1975 #else /* !KERNEL */
1976 /*
1977 * For user space, shared buffer is not available and hence the data
1978 * limit is immutable and is always the same as the underlying buffer
1979 * object limit itself.
1980 */
1981 return (uint32_t)__buflet_get_data_limit(buf);
1982 #endif /* !KERNEL */
1983 }
1984
1985 __attribute__((always_inline))
1986 static inline packet_trace_id_t
__packet_get_trace_id(const uint64_t ph)1987 __packet_get_trace_id(const uint64_t ph)
1988 {
1989 switch (SK_PTR_TYPE(ph)) {
1990 case NEXUS_META_TYPE_PACKET:
1991 return PKT_ADDR(ph)->pkt_trace_id;
1992 break;
1993 default:
1994 return 0;
1995 }
1996 }
1997
1998 __attribute__((always_inline))
1999 static inline void
__packet_set_trace_id(const uint64_t ph,packet_trace_id_t id)2000 __packet_set_trace_id(const uint64_t ph, packet_trace_id_t id)
2001 {
2002 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2003 PKT_ADDR(ph)->pkt_trace_id = id;
2004 }
2005
2006 __attribute__((always_inline))
2007 static inline void
__packet_trace_event(const uint64_t ph,uint32_t event)2008 __packet_trace_event(const uint64_t ph, uint32_t event)
2009 {
2010 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2011 #ifdef KERNEL
2012 #pragma unused(event, ph)
2013 KDBG(event, PKT_ADDR(ph)->pkt_trace_id);
2014 #else /* !KERNEL */
2015 kdebug_trace(event, PKT_ADDR(ph)->pkt_trace_id, 0, 0, 0);
2016 #endif /* !KERNEL */
2017 }
2018
2019 #ifdef KERNEL
2020 __attribute__((always_inline))
2021 static inline packet_trace_tag_t
__packet_get_trace_tag(const uint64_t ph)2022 __packet_get_trace_tag(const uint64_t ph)
2023 {
2024 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2025 return PKT_ADDR(ph)->pkt_trace_tag;
2026 }
2027
2028 __attribute__((always_inline))
2029 static inline void
__packet_set_trace_tag(const uint64_t ph,packet_trace_tag_t tag)2030 __packet_set_trace_tag(const uint64_t ph, packet_trace_tag_t tag)
2031 {
2032 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2033 PKT_ADDR(ph)->pkt_trace_tag = tag;
2034 }
2035
2036 static inline void
__packet_perform_tx_completion_callbacks(const kern_packet_t ph,ifnet_t ifp)2037 __packet_perform_tx_completion_callbacks(const kern_packet_t ph, ifnet_t ifp)
2038 {
2039 /*
2040 * NOTE: this function can be called with ifp as NULL.
2041 */
2042 uint64_t ts;
2043 kern_return_t tx_status;
2044 uintptr_t cb_arg, cb_data;
2045 struct __kern_packet *kpkt = SK_PTR_ADDR_KPKT(ph);
2046
2047 ASSERT((kpkt->pkt_pflags & PKT_F_TX_COMPL_TS_REQ) != 0);
2048 (void) __packet_get_tx_completion_status(ph, &tx_status);
2049 __packet_get_tx_completion_data(ph, &cb_arg, &cb_data);
2050 __packet_get_timestamp(ph, &ts, NULL);
2051 while (kpkt->pkt_tx_compl_callbacks != 0) {
2052 mbuf_tx_compl_func cb;
2053 uint32_t i;
2054
2055 i = ffs(kpkt->pkt_tx_compl_callbacks) - 1;
2056 kpkt->pkt_tx_compl_callbacks &= ~(1 << i);
2057 cb = m_get_tx_compl_callback(i);
2058 if (__probable(cb != NULL)) {
2059 cb(kpkt->pkt_tx_compl_context, ifp, ts, cb_arg, cb_data,
2060 tx_status);
2061 }
2062 }
2063 kpkt->pkt_pflags &= ~PKT_F_TX_COMPL_TS_REQ;
2064 }
2065
2066 static inline void *
__packet_get_priv(const kern_packet_t ph)2067 __packet_get_priv(const kern_packet_t ph)
2068 {
2069 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2070 return PKT_ADDR(ph)->pkt_priv;
2071 }
2072
2073 static inline void
__packet_set_priv(const uint64_t ph,void * priv)2074 __packet_set_priv(const uint64_t ph, void *priv)
2075 {
2076 PKT_TYPE_ASSERT(ph, NEXUS_META_TYPE_PACKET);
2077 PKT_ADDR(ph)->pkt_priv = priv;
2078 }
2079 #endif /* KERNEL */
2080
2081 #endif /* PRIVATE || BSD_KERNEL_PRIVATE */
2082 #endif /* !_SKYWALK_PACKET_COMMON_H_ */
2083