1 /*
2 * Copyright (c) 2019-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <skywalk/os_skywalk_private.h>
29 #include <skywalk/nexus/netif/nx_netif.h>
30 #include <net/pktap.h>
31 #include <sys/sdt.h>
32
33 SK_NO_INLINE_ATTRIBUTE
34 struct __kern_packet *
nx_netif_alloc_packet(struct kern_pbufpool * pp,uint32_t sz,kern_packet_t * php)35 nx_netif_alloc_packet(struct kern_pbufpool *pp, uint32_t sz, kern_packet_t *php)
36 {
37 kern_packet_t ph;
38 ph = pp_alloc_packet_by_size(pp, sz, SKMEM_NOSLEEP);
39 if (__improbable(ph == 0)) {
40 DTRACE_SKYWALK2(alloc__fail, struct kern_pbufpool *,
41 pp, size_t, sz);
42 return NULL;
43 }
44 if (php != NULL) {
45 *php = ph;
46 }
47 return SK_PTR_ADDR_KPKT(ph);
48 }
49
50 SK_NO_INLINE_ATTRIBUTE
51 void
nx_netif_free_packet(struct __kern_packet * pkt)52 nx_netif_free_packet(struct __kern_packet *pkt)
53 {
54 pp_free_packet_single(pkt);
55 }
56
57 SK_NO_INLINE_ATTRIBUTE
58 void
nx_netif_free_packet_chain(struct __kern_packet * pkt_chain,int * cnt)59 nx_netif_free_packet_chain(struct __kern_packet *pkt_chain, int *cnt)
60 {
61 pp_free_packet_chain(pkt_chain, cnt);
62 }
63
64 static void
__check_convert_flags(uint32_t flags)65 __check_convert_flags(uint32_t flags)
66 {
67 VERIFY((flags & (NETIF_CONVERT_TX | NETIF_CONVERT_RX)) != 0);
68 VERIFY((flags & (NETIF_CONVERT_TX | NETIF_CONVERT_RX)) !=
69 (NETIF_CONVERT_TX | NETIF_CONVERT_RX));
70 }
71
72 SK_NO_INLINE_ATTRIBUTE
73 static void
fill_vlan_info(struct __kern_packet * fpkt)74 fill_vlan_info(struct __kern_packet *fpkt)
75 {
76 struct mbuf *m;
77 struct __kern_packet *pkt;
78 uint16_t tag;
79
80 /*
81 * A filter packet must always have an mbuf or a packet
82 * attached.
83 */
84 VERIFY((fpkt->pkt_pflags & PKT_F_MBUF_DATA) != 0 ||
85 (fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0);
86
87 if ((fpkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
88 m = fpkt->pkt_mbuf;
89 VERIFY(m != NULL);
90 if (mbuf_get_vlan_tag(m, &tag) != 0) {
91 return;
92 }
93 DTRACE_SKYWALK1(tag__from__mbuf, uint16_t, tag);
94 kern_packet_set_vlan_tag(SK_PKT2PH(fpkt), tag);
95 } else if ((fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0) {
96 pkt = fpkt->pkt_pkt;
97 VERIFY(pkt != NULL);
98
99 /*
100 * The attached packet could have an mbuf attached
101 * if it came from the compat path.
102 */
103 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
104 m = fpkt->pkt_mbuf;
105 VERIFY(m != NULL);
106 if (mbuf_get_vlan_tag(m, &tag) != 0) {
107 return;
108 }
109 DTRACE_SKYWALK1(tag__from__inner__mbuf,
110 uint16_t, tag);
111 } else {
112 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
113 if (__packet_get_vlan_tag(SK_PKT2PH(pkt), &tag) != 0) {
114 return;
115 }
116 DTRACE_SKYWALK1(tag__from__pkt, uint16_t, tag);
117 }
118 kern_packet_set_vlan_tag(SK_PKT2PH(fpkt), tag);
119 } else {
120 panic("filter packet has no mbuf or packet attached: "
121 "pkt_pflags 0x%llx\n", fpkt->pkt_pflags);
122 /* NOTREACHED */
123 __builtin_unreachable();
124 }
125 }
126
127 static struct __kern_packet *
nx_netif_mbuf_to_filter_pkt(struct nexus_netif_adapter * nifna,struct mbuf * m,uint32_t flags)128 nx_netif_mbuf_to_filter_pkt(struct nexus_netif_adapter *nifna,
129 struct mbuf *m, uint32_t flags)
130 {
131 struct __kern_packet *fpkt = NULL;
132 struct nx_netif *nif = nifna->nifna_netif;
133 struct netif_stats *nifs = &nif->nif_stats;
134 struct kern_pbufpool *pp = nif->nif_filter_pp;
135 ifnet_t ifp = nif->nif_ifp;
136 boolean_t is_l3, truncated = FALSE;
137 enum txrx type;
138 uint8_t off, hlen;
139 kern_packet_t fph;
140 int err, mlen;
141
142 __check_convert_flags(flags);
143 is_l3 = (ifp->if_family == IFNET_FAMILY_UTUN ||
144 ifp->if_family == IFNET_FAMILY_IPSEC);
145
146 off = ((flags & NETIF_CONVERT_TX) != 0) ?
147 (uint8_t)ifp->if_tx_headroom : 0;
148 hlen = is_l3 ? 0 : ifnet_hdrlen(ifp);
149 mlen = m_pktlen(m);
150
151 ASSERT(pp != NULL);
152 if (__improbable((off + mlen) > PP_BUF_SIZE_DEF(pp))) {
153 VERIFY(off < PP_BUF_SIZE_DEF(pp));
154 mlen = PP_BUF_SIZE_DEF(pp) - off;
155 truncated = TRUE;
156
157 DTRACE_SKYWALK5(mbuf__truncated,
158 struct nexus_netif_adapter *, nifna,
159 struct mbuf *, m, uint8_t, off, int, mlen,
160 uint32_t, PP_BUF_SIZE_DEF(pp));
161 STATS_INC(nifs, NETIF_STATS_FILTER_PKT_TRUNCATED);
162 }
163 fpkt = nx_netif_alloc_packet(pp, off + mlen, &fph);
164 if (__improbable(fpkt == NULL)) {
165 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
166 nifna, struct mbuf *, m);
167 STATS_INC(nifs, NETIF_STATS_FILTER_DROP_PKT_ALLOC_FAIL);
168 goto drop;
169 }
170 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
171
172 if (__improbable((m->m_flags & M_HASFCS) != 0)) {
173 if (type != NR_RX) {
174 /*
175 * There shouldn't be an FCS for TX packets
176 */
177 DTRACE_SKYWALK2(bad__flags,
178 struct nexus_netif_adapter *,
179 nifna, struct mbuf *, m);
180 goto drop;
181 }
182 if (mlen > ETHER_CRC_LEN) {
183 mlen -= ETHER_CRC_LEN;
184 } else {
185 DTRACE_SKYWALK3(bad__pkt__size,
186 struct nexus_netif_adapter *,
187 nifna, struct mbuf *, m, int, mlen);
188 goto drop;
189 }
190 }
191 /*
192 * XXX
193 * If the source packet has any checksum flags, the filter packet will
194 * not have valid checksums. To fill in the checksums, we need to do
195 * something similar to bridge_finalize_cksum() for packets.
196 */
197 err = __packet_initialize_with_mbuf(fpkt, m, off, hlen);
198 VERIFY(err == 0);
199 nif->nif_pkt_copy_from_mbuf(type, fph, off, m, 0,
200 mlen, FALSE, 0);
201
202 err = __packet_finalize_with_mbuf(fpkt);
203 VERIFY(err == 0);
204
205 /*
206 * XXX
207 * __packet_finalize_with_mbuf() sets pkt_length to the non-truncated
208 * length. We need to change it back to the truncated length.
209 */
210 fpkt->pkt_length = mlen;
211 if (!is_l3) {
212 fill_vlan_info(fpkt);
213 }
214
215 /*
216 * Verify that __packet_finalize_with_mbuf() is setting the truncated
217 * flag correctly.
218 */
219 if (truncated) {
220 VERIFY((fpkt->pkt_pflags & PKT_F_TRUNCATED) != 0);
221 } else {
222 VERIFY((fpkt->pkt_pflags & PKT_F_TRUNCATED) == 0);
223 }
224 return fpkt;
225 drop:
226 if (fpkt != NULL) {
227 /* ensure mbuf hasn't been attached */
228 ASSERT(fpkt->pkt_mbuf == NULL &&
229 (fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
230 nx_netif_free_packet(fpkt);
231 }
232 STATS_INC(nifs, NETIF_STATS_DROP);
233 m_freem(m);
234 return NULL;
235 }
236
237 struct __kern_packet *
nx_netif_mbuf_to_filter_pkt_chain(struct nexus_netif_adapter * nifna,struct mbuf * m_chain,uint32_t flags)238 nx_netif_mbuf_to_filter_pkt_chain(struct nexus_netif_adapter *nifna,
239 struct mbuf *m_chain, uint32_t flags)
240 {
241 struct mbuf *m = m_chain, *next;
242 struct __kern_packet *__single pkt_head = NULL, *pkt;
243 struct __kern_packet **pkt_tailp = &pkt_head;
244 int c = 0;
245
246 while (m != NULL) {
247 next = m->m_nextpkt;
248 m->m_nextpkt = NULL;
249
250 pkt = nx_netif_mbuf_to_filter_pkt(nifna, m, flags);
251 if (pkt != NULL) {
252 c++;
253 *pkt_tailp = pkt;
254 pkt_tailp = &pkt->pkt_nextpkt;
255 }
256 m = next;
257 }
258 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, pkt_head,
259 int, c);
260 return pkt_head;
261 }
262
263 static struct mbuf *
nx_netif_filter_pkt_to_mbuf(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)264 nx_netif_filter_pkt_to_mbuf(struct nexus_netif_adapter *nifna,
265 struct __kern_packet *pkt, uint32_t flags)
266 {
267 #pragma unused (nifna)
268 struct mbuf *m;
269
270 __check_convert_flags(flags);
271 ASSERT((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0);
272
273 m = pkt->pkt_mbuf;
274 ASSERT(m != NULL);
275 KPKT_CLEAR_MBUF_DATA(pkt);
276 nx_netif_free_packet(pkt);
277 return m;
278 }
279
280 struct mbuf *
nx_netif_filter_pkt_to_mbuf_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)281 nx_netif_filter_pkt_to_mbuf_chain(struct nexus_netif_adapter *nifna,
282 struct __kern_packet *pkt_chain, uint32_t flags)
283 {
284 struct __kern_packet *pkt = pkt_chain, *next;
285 struct mbuf *__single m_head = NULL, *m;
286 struct mbuf **m_tailp = &m_head;
287 int c = 0;
288
289 while (pkt != NULL) {
290 next = pkt->pkt_nextpkt;
291 pkt->pkt_nextpkt = NULL;
292
293 m = nx_netif_filter_pkt_to_mbuf(nifna, pkt, flags);
294 if (m != NULL) {
295 c++;
296 *m_tailp = m;
297 m_tailp = &m->m_nextpkt;
298 }
299 pkt = next;
300 }
301 DTRACE_SKYWALK2(mbuf__chain, struct mbuf *, m_head, int, c);
302 return m_head;
303 }
304
305 struct __kern_packet *
nx_netif_pkt_to_filter_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)306 nx_netif_pkt_to_filter_pkt(struct nexus_netif_adapter *nifna,
307 struct __kern_packet *pkt, uint32_t flags)
308 {
309 struct __kern_packet *fpkt = NULL;
310 struct nx_netif *nif = nifna->nifna_netif;
311 struct netif_stats *nifs = &nif->nif_stats;
312 struct kern_pbufpool *pp = nif->nif_filter_pp;
313 ifnet_t ifp = nif->nif_ifp;
314 boolean_t is_l3, truncated = FALSE;
315 enum txrx type;
316 uint8_t off, hlen;
317 struct mbuf *m = NULL;
318 kern_packet_t fph, ph;
319 int err, plen;
320
321 __check_convert_flags(flags);
322 ph = SK_PKT2PH(pkt);
323 is_l3 = (ifp->if_family == IFNET_FAMILY_UTUN ||
324 ifp->if_family == IFNET_FAMILY_IPSEC);
325
326 off = ((flags & NETIF_CONVERT_TX) != 0) ?
327 (uint8_t)ifp->if_tx_headroom : 0;
328 hlen = is_l3 ? 0 : ifnet_hdrlen(ifp);
329
330 /*
331 * The packet coming from the compat path could be empty or has
332 * truncated contents. We have to copy the contents from the
333 * attached mbuf. We also don't support attaching a filter
334 * packet (one that already has a packet attached) to another
335 * filter packet.
336 */
337 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
338 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
339 m = pkt->pkt_mbuf;
340 plen = m_pktlen(m);
341 } else {
342 plen = pkt->pkt_length;
343 }
344 ASSERT(pp != NULL);
345 if (__improbable((off + plen) > PP_BUF_SIZE_DEF(pp))) {
346 VERIFY(off < PP_BUF_SIZE_DEF(pp));
347 plen = PP_BUF_SIZE_DEF(pp) - off;
348 truncated = TRUE;
349
350 DTRACE_SKYWALK5(pkt__truncated,
351 struct nexus_netif_adapter *, nifna,
352 struct __kern_packet *, pkt, uint8_t, off,
353 int, plen, uint32_t, PP_BUF_SIZE_DEF(pp));
354 STATS_INC(nifs, NETIF_STATS_FILTER_PKT_TRUNCATED);
355 }
356 fpkt = nx_netif_alloc_packet(pp, off + plen, &fph);
357 if (__improbable(fpkt == NULL)) {
358 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
359 nifna, struct __kern_packet *, pkt);
360 STATS_INC(nifs, NETIF_STATS_FILTER_DROP_PKT_ALLOC_FAIL);
361 goto drop;
362 }
363 fpkt->pkt_link_flags = 0;
364 fpkt->pkt_headroom = off;
365 fpkt->pkt_l2_len = hlen;
366 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
367
368 if (__improbable((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) != 0 ||
369 (m != NULL && (m->m_flags & M_HASFCS) != 0))) {
370 if (type != NR_RX) {
371 /*
372 * There shouldn't be an FCS for TX packets
373 */
374 DTRACE_SKYWALK2(bad__flags,
375 struct nexus_netif_adapter *, nifna,
376 struct __kern_packet *, pkt);
377 goto drop;
378 }
379 if (plen > ETHER_CRC_LEN) {
380 plen -= ETHER_CRC_LEN;
381 } else {
382 DTRACE_SKYWALK3(bad__pkt__size,
383 struct nexus_netif_adapter *, nifna,
384 struct __kern_packet *, pkt, int, plen);
385 goto drop;
386 }
387 }
388 /*
389 * XXX
390 * If the source packet has any checksum flags, the filter packet will
391 * not have valid checksums. To fill in the checksums, we need to do
392 * something similar to bridge_finalize_cksum() for packets.
393 */
394 if (m != NULL) {
395 nif->nif_pkt_copy_from_mbuf(type, fph, off, m, 0,
396 plen, FALSE, 0);
397 } else {
398 nif->nif_pkt_copy_from_pkt(type, fph, off, ph,
399 pkt->pkt_headroom, plen, FALSE, 0, 0, FALSE);
400 }
401 ASSERT((fpkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
402 ASSERT((fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
403 ASSERT(fpkt->pkt_pkt == NULL);
404 ASSERT(pkt->pkt_nextpkt == NULL);
405 fpkt->pkt_pkt = pkt;
406 fpkt->pkt_pflags |= PKT_F_PKT_DATA;
407 if (truncated) {
408 fpkt->pkt_pflags |= PKT_F_TRUNCATED;
409 }
410 /*
411 * XXX
412 * Unlike the mbuf case, __packet_finalize below correctly sets
413 * pkt_length to the buflet length (possibly truncated). We set
414 * pkt_length here so that fill_vlan_info can use it.
415 */
416 fpkt->pkt_length = plen;
417 if (!is_l3) {
418 fill_vlan_info(fpkt);
419 }
420 err = __packet_finalize(fph);
421 VERIFY(err == 0);
422 return fpkt;
423 drop:
424 if (fpkt != NULL) {
425 /* ensure pkt hasn't been attached */
426 ASSERT(fpkt->pkt_pkt == NULL &&
427 (fpkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
428 nx_netif_free_packet(fpkt);
429 }
430 STATS_INC(nifs, NETIF_STATS_DROP);
431 nx_netif_free_packet(pkt);
432 return NULL;
433 }
434
435 struct __kern_packet *
nx_netif_pkt_to_filter_pkt_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)436 nx_netif_pkt_to_filter_pkt_chain(struct nexus_netif_adapter *nifna,
437 struct __kern_packet *pkt_chain, uint32_t flags)
438 {
439 struct __kern_packet *pkt = pkt_chain, *next;
440 struct __kern_packet *__single p_head = NULL, *p;
441 struct __kern_packet **p_tailp = &p_head;
442 int c = 0;
443
444 while (pkt != NULL) {
445 next = pkt->pkt_nextpkt;
446 pkt->pkt_nextpkt = NULL;
447
448 p = nx_netif_pkt_to_filter_pkt(nifna, pkt, flags);
449 if (p != NULL) {
450 c++;
451 *p_tailp = p;
452 p_tailp = &p->pkt_nextpkt;
453 }
454 pkt = next;
455 }
456 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, p_head, int, c);
457 return p_head;
458 }
459
460 static struct __kern_packet *
nx_netif_filter_pkt_to_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * fpkt,uint32_t flags)461 nx_netif_filter_pkt_to_pkt(struct nexus_netif_adapter *nifna,
462 struct __kern_packet *fpkt, uint32_t flags)
463 {
464 #pragma unused (nifna)
465 struct __kern_packet *pkt;
466
467 __check_convert_flags(flags);
468 ASSERT((fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0);
469 ASSERT((fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
470
471 pkt = fpkt->pkt_pkt;
472 ASSERT(pkt != NULL);
473 KPKT_CLEAR_PKT_DATA(fpkt);
474 nx_netif_free_packet(fpkt);
475 return pkt;
476 }
477
478 struct __kern_packet *
nx_netif_filter_pkt_to_pkt_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)479 nx_netif_filter_pkt_to_pkt_chain(struct nexus_netif_adapter *nifna,
480 struct __kern_packet *pkt_chain, uint32_t flags)
481 {
482 struct __kern_packet *pkt = pkt_chain, *next;
483 struct __kern_packet *__single p_head = NULL, *p;
484 struct __kern_packet **p_tailp = &p_head;
485 int c = 0;
486
487 while (pkt != NULL) {
488 next = pkt->pkt_nextpkt;
489 pkt->pkt_nextpkt = NULL;
490
491 p = nx_netif_filter_pkt_to_pkt(nifna, pkt, flags);
492 if (p != NULL) {
493 c++;
494 *p_tailp = p;
495 p_tailp = &p->pkt_nextpkt;
496 }
497 pkt = next;
498 }
499 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, p_head, int, c);
500 return p_head;
501 }
502
503 struct mbuf *
nx_netif_pkt_to_mbuf(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)504 nx_netif_pkt_to_mbuf(struct nexus_netif_adapter *nifna,
505 struct __kern_packet *pkt, uint32_t flags)
506 {
507 struct nx_netif *nif = nifna->nifna_netif;
508 ifnet_t ifp = nif->nif_ifp;
509 struct mbuf *__single m;
510 unsigned int one = 1;
511 size_t len;
512 uint16_t pad, hlen;
513 kern_packet_t ph;
514 enum txrx type;
515 int err;
516
517 __check_convert_flags(flags);
518 /* Compat packets or filter packets should never land here */
519 ASSERT((pkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
520 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
521
522 /* Outbound packets should not have this */
523 ASSERT((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) == 0);
524
525 /* This function is only meant to be used in the custom ether TX path */
526 ASSERT((flags & NETIF_CONVERT_TX) != 0);
527 type = NR_TX;
528
529 /* Packet must include L2 header */
530 hlen = ifnet_hdrlen(ifp);
531 pad = (uint16_t)P2ROUNDUP(hlen, sizeof(uint32_t)) - hlen;
532 len = pkt->pkt_length;
533
534 err = mbuf_allocpacket(MBUF_WAITOK, pad + len, &one, &m);
535 VERIFY(err == 0);
536 m->m_data += pad;
537 m->m_pkthdr.pkt_hdr = mtod(m, uint8_t *);
538 ph = SK_PTR_ENCODE(pkt, METADATA_TYPE(pkt), METADATA_SUBTYPE(pkt));
539
540 nif->nif_pkt_copy_to_mbuf(type, ph, pkt->pkt_headroom,
541 m, 0, (uint32_t)len, FALSE, 0);
542 m->m_pkthdr.pkt_flowsrc = pkt->pkt_flowsrc_type;
543 m->m_pkthdr.pkt_flowid = pkt->pkt_flow_token;
544 m->m_pkthdr.necp_mtag.necp_policy_id = pkt->pkt_policy_id;
545 m->m_pkthdr.necp_mtag.necp_skip_policy_id = pkt->pkt_skip_policy_id;
546
547 nx_netif_free_packet(pkt);
548 return m;
549 }
550
551 struct __kern_packet *
nx_netif_pkt_to_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)552 nx_netif_pkt_to_pkt(struct nexus_netif_adapter *nifna,
553 struct __kern_packet *pkt, uint32_t flags)
554 {
555 struct nx_netif *nif = nifna->nifna_netif;
556 struct nexus_adapter *na = &nifna->nifna_up;
557 struct netif_stats *nifs = &nif->nif_stats;
558 ifnet_t ifp = nif->nif_ifp;
559 struct kern_pbufpool *pp;
560 struct __kern_packet *dpkt = NULL;
561 struct mbuf *m = NULL;
562 uint8_t off, hlen;
563 int len;
564 kern_packet_t ph, dph;
565 enum txrx type;
566 int err;
567
568 __check_convert_flags(flags);
569 /* Filter packets should never land here */
570 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
571
572 /* Only support these target NAs for now */
573 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
574 if (type == NR_TX) {
575 ASSERT(na->na_type == NA_NETIF_DEV ||
576 na->na_type == NA_NETIF_COMPAT_DEV);
577 pp = skmem_arena_nexus(na->na_arena)->arn_tx_pp;
578 off = (uint8_t)ifp->if_tx_headroom;
579 } else {
580 ASSERT(na->na_type == NA_NETIF_VP ||
581 na->na_type == NA_NETIF_DEV);
582 pp = skmem_arena_nexus(na->na_arena)->arn_rx_pp;
583 off = 0;
584 }
585 /* Packet must include L2 header */
586 hlen = ifnet_hdrlen(ifp);
587
588 /*
589 * Source packet has no data. Need to copy from the attached mbuf.
590 */
591 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
592 /* An outbound packet shouldn't have an mbuf attached */
593 ASSERT(na->na_type == NA_NETIF_VP ||
594 na->na_type == NA_NETIF_DEV);
595 m = pkt->pkt_mbuf;
596 len = m_pktlen(m);
597 } else {
598 len = pkt->pkt_length;
599 }
600 ASSERT(pp != NULL);
601
602 ph = SK_PKT2PH(pkt);
603 dpkt = nx_netif_alloc_packet(pp, off + len, &dph);
604 if (__improbable(dpkt == NULL)) {
605 if (type == NR_TX) {
606 STATS_INC(nifs, NETIF_STATS_VP_DROP_TX_ALLOC_FAIL);
607 } else {
608 STATS_INC(nifs, NETIF_STATS_VP_DROP_RX_ALLOC_FAIL);
609 }
610 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
611 nifna, struct __kern_packet *, pkt);
612 goto drop;
613 }
614 if (__improbable((off + len) > PP_BUF_SIZE_DEF(pp))) {
615 STATS_INC(nifs, NETIF_STATS_VP_DROP_PKT_TOO_BIG);
616 DTRACE_SKYWALK5(pkt__too__large,
617 struct nexus_netif_adapter *, nifna,
618 struct __kern_packet *, pkt, uint8_t, off, int, len,
619 uint32_t, PP_BUF_SIZE_DEF(pp));
620 goto drop;
621 }
622 if (__improbable((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) != 0 ||
623 (m != NULL && (m->m_flags & M_HASFCS) != 0))) {
624 if (type != NR_RX) {
625 /*
626 * There shouldn't be an FCS for TX packets
627 */
628 DTRACE_SKYWALK2(bad__flags,
629 struct nexus_netif_adapter *, nifna,
630 struct __kern_packet *, pkt);
631 goto drop;
632 }
633 if (len > ETHER_CRC_LEN) {
634 len -= ETHER_CRC_LEN;
635 } else {
636 DTRACE_SKYWALK3(bad__pkt__size,
637 struct nexus_netif_adapter *, nifna,
638 struct __kern_packet *, pkt, int, len);
639 goto drop;
640 }
641 }
642 dpkt->pkt_link_flags = 0;
643 dpkt->pkt_headroom = off;
644 dpkt->pkt_l2_len = hlen;
645
646 /* Copy optional metadata */
647 dpkt->pkt_pflags = (pkt->pkt_pflags & PKT_F_COPY_MASK);
648 _PKT_COPY_OPT_DATA(pkt, dpkt);
649
650 /* Copy Transmit completion metadata */
651 _PKT_COPY_TX_PORT_DATA(pkt, dpkt);
652
653 /* Copy packet contents */
654 if (m != NULL) {
655 nif->nif_pkt_copy_from_mbuf(type, dph, off, m, 0,
656 len, FALSE, 0);
657 } else {
658 nif->nif_pkt_copy_from_pkt(type, dph, off, ph,
659 pkt->pkt_headroom, len, FALSE, 0, 0, FALSE);
660 }
661
662 if (type == NR_TX) {
663 dpkt->pkt_svc_class = pkt->pkt_svc_class;
664 dpkt->pkt_flowsrc_type = pkt->pkt_flowsrc_type;
665 dpkt->pkt_flow_token = pkt->pkt_flow_token;
666 dpkt->pkt_policy_id = pkt->pkt_policy_id;
667 dpkt->pkt_skip_policy_id = pkt->pkt_skip_policy_id;
668 _UUID_COPY(dpkt->pkt_policy_euuid, pkt->pkt_policy_euuid);
669 }
670
671 err = __packet_finalize(dph);
672 VERIFY(err == 0);
673 nx_netif_free_packet(pkt);
674 return dpkt;
675
676 drop:
677 if (dpkt != NULL) {
678 nx_netif_free_packet(dpkt);
679 }
680 STATS_INC(nifs, NETIF_STATS_DROP);
681 nx_netif_free_packet(pkt);
682 return NULL;
683 }
684
685 void
nx_netif_mbuf_chain_info(struct mbuf * m_head,struct mbuf ** m_tail,uint32_t * cnt,uint32_t * bytes)686 nx_netif_mbuf_chain_info(struct mbuf *m_head, struct mbuf **m_tail,
687 uint32_t *cnt, uint32_t *bytes)
688 {
689 struct mbuf *m = m_head, *tail = NULL;
690 uint32_t c = 0, b = 0;
691
692 while (m != NULL) {
693 c++;
694 b += m_pktlen(m);
695 tail = m;
696 m = m->m_nextpkt;
697 }
698 if (m_tail != NULL) {
699 *m_tail = tail;
700 }
701 if (cnt != NULL) {
702 *cnt = c;
703 }
704 if (bytes != NULL) {
705 *bytes = b;
706 }
707 }
708
709 void
nx_netif_pkt_chain_info(struct __kern_packet * p_head,struct __kern_packet ** p_tail,uint32_t * cnt,uint32_t * bytes)710 nx_netif_pkt_chain_info(struct __kern_packet *p_head,
711 struct __kern_packet **p_tail, uint32_t *cnt, uint32_t *bytes)
712 {
713 struct __kern_packet *p = p_head, *tail = NULL;
714 uint32_t c = 0, b = 0;
715
716 while (p != NULL) {
717 c++;
718 b += p->pkt_length;
719 tail = p;
720 p = p->pkt_nextpkt;
721 }
722 if (p_tail != NULL) {
723 *p_tail = tail;
724 }
725 if (cnt != NULL) {
726 *cnt = c;
727 }
728 if (bytes != NULL) {
729 *bytes = b;
730 }
731 }
732
733 int
nx_netif_get_max_mtu(ifnet_t ifp,uint32_t * max_mtu)734 nx_netif_get_max_mtu(ifnet_t ifp, uint32_t *max_mtu)
735 {
736 struct ifreq ifr;
737 int err;
738
739 bzero(&ifr, sizeof(ifr));
740 err = ifnet_ioctl(ifp, 0, SIOCGIFDEVMTU, &ifr);
741 if (err != 0) {
742 SK_ERR("SIOCGIFDEVMTU failed for %s\n", if_name(ifp));
743 return err;
744 }
745 *max_mtu = MAX(ifr.ifr_devmtu.ifdm_max, ifr.ifr_devmtu.ifdm_current);
746 return 0;
747 }
748
749 void
nx_netif_pktap_output(ifnet_t ifp,int af,struct __kern_packet * pkt)750 nx_netif_pktap_output(ifnet_t ifp, int af, struct __kern_packet *pkt)
751 {
752 uint32_t dlt;
753 uint32_t flags = PTH_FLAG_SOCKET;
754
755 switch (ifp->if_family) {
756 case IFNET_FAMILY_ETHERNET:
757 dlt = DLT_EN10MB;
758 break;
759 case IFNET_FAMILY_CELLULAR:
760 case IFNET_FAMILY_UTUN:
761 case IFNET_FAMILY_IPSEC:
762 dlt = DLT_RAW;
763 break;
764 default:
765 DTRACE_SKYWALK1(invalid__family, ifnet_t, ifp);
766 return;
767 }
768 if ((pkt->pkt_pflags & PKT_F_KEEPALIVE) != 0) {
769 flags |= PTH_FLAG_KEEP_ALIVE;
770 }
771 if ((pkt->pkt_pflags & PKT_F_REXMT) != 0) {
772 flags |= PTH_FLAG_REXMIT;
773 }
774 pktap_output_packet(ifp, af, dlt, -1, NULL, -1, NULL, SK_PKT2PH(pkt),
775 NULL, 0, pkt->pkt_flow_ip_proto, pkt->pkt_flow_token, flags);
776 }
777
778 __attribute__((always_inline))
779 inline void
netif_ifp_inc_traffic_class_out_pkt(struct ifnet * ifp,uint32_t svc,uint32_t cnt,uint32_t len)780 netif_ifp_inc_traffic_class_out_pkt(struct ifnet *ifp, uint32_t svc,
781 uint32_t cnt, uint32_t len)
782 {
783 switch (svc) {
784 case PKT_TC_BE:
785 ifp->if_tc.ifi_obepackets += cnt;
786 ifp->if_tc.ifi_obebytes += len;
787 break;
788 case PKT_TC_BK:
789 ifp->if_tc.ifi_obkpackets += cnt;
790 ifp->if_tc.ifi_obkbytes += len;
791 break;
792 case PKT_TC_VI:
793 ifp->if_tc.ifi_ovipackets += cnt;
794 ifp->if_tc.ifi_ovibytes += len;
795 break;
796 case PKT_TC_VO:
797 ifp->if_tc.ifi_ovopackets += cnt;
798 ifp->if_tc.ifi_ovobytes += len;
799 break;
800 default:
801 break;
802 }
803 }
804