1 /*
2 * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <skywalk/os_skywalk_private.h>
29 #include <skywalk/nexus/netif/nx_netif.h>
30 #include <net/pktap.h>
31 #include <sys/sdt.h>
32
33 SK_NO_INLINE_ATTRIBUTE
34 struct __kern_packet *
nx_netif_alloc_packet(struct kern_pbufpool * pp,uint32_t sz,kern_packet_t * php)35 nx_netif_alloc_packet(struct kern_pbufpool *pp, uint32_t sz, kern_packet_t *php)
36 {
37 kern_packet_t ph;
38 ph = pp_alloc_packet_by_size(pp, sz, SKMEM_NOSLEEP);
39 if (__improbable(ph == 0)) {
40 DTRACE_SKYWALK2(alloc__fail, struct kern_pbufpool *,
41 pp, size_t, sz);
42 return NULL;
43 }
44 if (php != NULL) {
45 *php = ph;
46 }
47 return SK_PTR_ADDR_KPKT(ph);
48 }
49
50 SK_NO_INLINE_ATTRIBUTE
51 void
nx_netif_free_packet(struct __kern_packet * pkt)52 nx_netif_free_packet(struct __kern_packet *pkt)
53 {
54 pp_free_packet_single(pkt);
55 }
56
57 SK_NO_INLINE_ATTRIBUTE
58 void
nx_netif_free_packet_chain(struct __kern_packet * pkt_chain,int * cnt)59 nx_netif_free_packet_chain(struct __kern_packet *pkt_chain, int *cnt)
60 {
61 pp_free_packet_chain(pkt_chain, cnt);
62 }
63
64 static void
__check_convert_flags(uint32_t flags)65 __check_convert_flags(uint32_t flags)
66 {
67 VERIFY((flags & (NETIF_CONVERT_TX | NETIF_CONVERT_RX)) != 0);
68 VERIFY((flags & (NETIF_CONVERT_TX | NETIF_CONVERT_RX)) !=
69 (NETIF_CONVERT_TX | NETIF_CONVERT_RX));
70 }
71
72 SK_NO_INLINE_ATTRIBUTE
73 static void
fill_vlan_info(struct __kern_packet * fpkt)74 fill_vlan_info(struct __kern_packet *fpkt)
75 {
76 uint8_t *buf;
77 struct ether_vlan_header *evl;
78 uint16_t tag;
79 boolean_t tag_in_pkt = FALSE;
80
81 if (fpkt->pkt_length < sizeof(*evl)) {
82 DTRACE_SKYWALK2(bad__len, struct __kern_packet *, fpkt,
83 uint32_t, fpkt->pkt_length);
84 return;
85 }
86 MD_BUFLET_ADDR_ABS(fpkt, buf);
87 buf += fpkt->pkt_headroom;
88 evl = (struct ether_vlan_header *)(void *)buf;
89 if (ntohs(evl->evl_encap_proto) == ETHERTYPE_VLAN) {
90 tag = ntohs(evl->evl_tag);
91 tag_in_pkt = TRUE;
92 DTRACE_SKYWALK1(tag__in__pkt, uint16_t, tag);
93 } else {
94 struct mbuf *m;
95 struct __kern_packet *pkt;
96
97 /*
98 * A filter packet must always have an mbuf or a packet
99 * attached.
100 */
101 VERIFY((fpkt->pkt_pflags & PKT_F_MBUF_DATA) != 0 ||
102 (fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0);
103
104 if ((fpkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
105 m = fpkt->pkt_mbuf;
106 VERIFY(m != NULL);
107 if (mbuf_get_vlan_tag(m, &tag) != 0) {
108 return;
109 }
110 DTRACE_SKYWALK1(tag__from__mbuf, uint16_t, tag);
111 } else if ((fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0) {
112 pkt = fpkt->pkt_pkt;
113 VERIFY(pkt != NULL);
114
115 /*
116 * The attached packet could have an mbuf attached
117 * if it came from the compat path.
118 */
119 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
120 m = fpkt->pkt_mbuf;
121 VERIFY(m != NULL);
122 if (mbuf_get_vlan_tag(m, &tag) != 0) {
123 return;
124 }
125 DTRACE_SKYWALK1(tag__from__inner__mbuf,
126 uint16_t, tag);
127 } else {
128 /*
129 * XXX
130 * No native driver today fills in the vlan tag
131 * metadata. This code will work when the driver
132 * adds support for this.
133 */
134 VERIFY((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
135 if (__packet_get_vlan_tag(SK_PKT2PH(pkt), &tag,
136 NULL) != 0) {
137 return;
138 }
139 DTRACE_SKYWALK1(tag__from__pkt, uint16_t, tag);
140 }
141 } else {
142 panic("filter packet has no mbuf or packet attached: "
143 "pkt_pflags 0x%llx\n", fpkt->pkt_pflags);
144 /* NOTREACHED */
145 __builtin_unreachable();
146 }
147 }
148 kern_packet_set_vlan_tag(SK_PKT2PH(fpkt), tag, tag_in_pkt);
149 }
150
151 static struct __kern_packet *
nx_netif_mbuf_to_filter_pkt(struct nexus_netif_adapter * nifna,struct mbuf * m,uint32_t flags)152 nx_netif_mbuf_to_filter_pkt(struct nexus_netif_adapter *nifna,
153 struct mbuf *m, uint32_t flags)
154 {
155 struct __kern_packet *fpkt = NULL;
156 struct nx_netif *nif = nifna->nifna_netif;
157 struct netif_stats *nifs = &nif->nif_stats;
158 struct kern_pbufpool *pp = nif->nif_filter_pp;
159 ifnet_t ifp = nif->nif_ifp;
160 boolean_t is_l3, truncated = FALSE;
161 enum txrx type;
162 uint8_t off, hlen;
163 kern_packet_t fph;
164 int err, mlen;
165
166 __check_convert_flags(flags);
167 is_l3 = (ifp->if_family == IFNET_FAMILY_UTUN ||
168 ifp->if_family == IFNET_FAMILY_IPSEC);
169
170 off = ((flags & NETIF_CONVERT_TX) != 0) ?
171 (uint8_t)ifp->if_tx_headroom : 0;
172 hlen = is_l3 ? 0 : ifnet_hdrlen(ifp);
173 mlen = m_pktlen(m);
174
175 ASSERT(pp != NULL);
176 if (__improbable((off + mlen) > PP_BUF_SIZE_DEF(pp))) {
177 VERIFY(off < PP_BUF_SIZE_DEF(pp));
178 mlen = PP_BUF_SIZE_DEF(pp) - off;
179 truncated = TRUE;
180
181 DTRACE_SKYWALK5(mbuf__truncated,
182 struct nexus_netif_adapter *, nifna,
183 struct mbuf *, m, uint8_t, off, int, mlen,
184 uint32_t, PP_BUF_SIZE_DEF(pp));
185 STATS_INC(nifs, NETIF_STATS_FILTER_PKT_TRUNCATED);
186 }
187 fpkt = nx_netif_alloc_packet(pp, off + mlen, &fph);
188 if (__improbable(fpkt == NULL)) {
189 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
190 nifna, struct mbuf *, m);
191 STATS_INC(nifs, NETIF_STATS_FILTER_DROP_PKT_ALLOC_FAIL);
192 goto drop;
193 }
194 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
195
196 if (__improbable((m->m_flags & M_HASFCS) != 0)) {
197 if (type != NR_RX) {
198 /*
199 * There shouldn't be an FCS for TX packets
200 */
201 DTRACE_SKYWALK2(bad__flags,
202 struct nexus_netif_adapter *,
203 nifna, struct mbuf *, m);
204 goto drop;
205 }
206 if (mlen > ETHER_CRC_LEN) {
207 mlen -= ETHER_CRC_LEN;
208 } else {
209 DTRACE_SKYWALK3(bad__pkt__size,
210 struct nexus_netif_adapter *,
211 nifna, struct mbuf *, m, int, mlen);
212 goto drop;
213 }
214 }
215 /*
216 * XXX
217 * If the source packet has any checksum flags, the filter packet will
218 * not have valid checksums. To fill in the checksums, we need to do
219 * something similar to bridge_finalize_cksum() for packets.
220 */
221 err = __packet_initialize_with_mbuf(fpkt, m, off, hlen);
222 VERIFY(err == 0);
223 nif->nif_pkt_copy_from_mbuf(type, fph, off, m, 0,
224 mlen, FALSE, 0);
225
226 err = __packet_finalize_with_mbuf(fpkt);
227 VERIFY(err == 0);
228
229 /*
230 * XXX
231 * __packet_finalize_with_mbuf() sets pkt_length to the non-truncated
232 * length. We need to change it back to the truncated length.
233 */
234 fpkt->pkt_length = mlen;
235 if (!is_l3) {
236 fill_vlan_info(fpkt);
237 }
238
239 /*
240 * Verify that __packet_finalize_with_mbuf() is setting the truncated
241 * flag correctly.
242 */
243 if (truncated) {
244 VERIFY((fpkt->pkt_pflags & PKT_F_TRUNCATED) != 0);
245 } else {
246 VERIFY((fpkt->pkt_pflags & PKT_F_TRUNCATED) == 0);
247 }
248 return fpkt;
249 drop:
250 if (fpkt != NULL) {
251 /* ensure mbuf hasn't been attached */
252 ASSERT(fpkt->pkt_mbuf == NULL &&
253 (fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
254 nx_netif_free_packet(fpkt);
255 }
256 STATS_INC(nifs, NETIF_STATS_DROP);
257 m_freem(m);
258 return NULL;
259 }
260
261 struct __kern_packet *
nx_netif_mbuf_to_filter_pkt_chain(struct nexus_netif_adapter * nifna,struct mbuf * m_chain,uint32_t flags)262 nx_netif_mbuf_to_filter_pkt_chain(struct nexus_netif_adapter *nifna,
263 struct mbuf *m_chain, uint32_t flags)
264 {
265 struct mbuf *m = m_chain, *next;
266 struct __kern_packet *pkt_head = NULL, *pkt;
267 struct __kern_packet **pkt_tailp = &pkt_head;
268 int c = 0;
269
270 while (m != NULL) {
271 next = m->m_nextpkt;
272 m->m_nextpkt = NULL;
273
274 pkt = nx_netif_mbuf_to_filter_pkt(nifna, m, flags);
275 if (pkt != NULL) {
276 c++;
277 *pkt_tailp = pkt;
278 pkt_tailp = &pkt->pkt_nextpkt;
279 }
280 m = next;
281 }
282 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, pkt_head,
283 int, c);
284 return pkt_head;
285 }
286
287 static struct mbuf *
nx_netif_filter_pkt_to_mbuf(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)288 nx_netif_filter_pkt_to_mbuf(struct nexus_netif_adapter *nifna,
289 struct __kern_packet *pkt, uint32_t flags)
290 {
291 #pragma unused (nifna)
292 struct mbuf *m;
293
294 __check_convert_flags(flags);
295 ASSERT((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0);
296
297 m = pkt->pkt_mbuf;
298 ASSERT(m != NULL);
299 KPKT_CLEAR_MBUF_DATA(pkt);
300 nx_netif_free_packet(pkt);
301 return m;
302 }
303
304 struct mbuf *
nx_netif_filter_pkt_to_mbuf_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)305 nx_netif_filter_pkt_to_mbuf_chain(struct nexus_netif_adapter *nifna,
306 struct __kern_packet *pkt_chain, uint32_t flags)
307 {
308 struct __kern_packet *pkt = pkt_chain, *next;
309 struct mbuf *m_head = NULL, *m;
310 struct mbuf **m_tailp = &m_head;
311 int c = 0;
312
313 while (pkt != NULL) {
314 next = pkt->pkt_nextpkt;
315 pkt->pkt_nextpkt = NULL;
316
317 m = nx_netif_filter_pkt_to_mbuf(nifna, pkt, flags);
318 if (m != NULL) {
319 c++;
320 *m_tailp = m;
321 m_tailp = &m->m_nextpkt;
322 }
323 pkt = next;
324 }
325 DTRACE_SKYWALK2(mbuf__chain, struct mbuf *, m_head, int, c);
326 return m_head;
327 }
328
329 struct __kern_packet *
nx_netif_pkt_to_filter_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)330 nx_netif_pkt_to_filter_pkt(struct nexus_netif_adapter *nifna,
331 struct __kern_packet *pkt, uint32_t flags)
332 {
333 struct __kern_packet *fpkt = NULL;
334 struct nx_netif *nif = nifna->nifna_netif;
335 struct netif_stats *nifs = &nif->nif_stats;
336 struct kern_pbufpool *pp = nif->nif_filter_pp;
337 ifnet_t ifp = nif->nif_ifp;
338 boolean_t is_l3, truncated = FALSE;
339 enum txrx type;
340 uint8_t off, hlen;
341 struct mbuf *m = NULL;
342 kern_packet_t fph, ph;
343 int err, plen;
344
345 __check_convert_flags(flags);
346 ph = SK_PKT2PH(pkt);
347 is_l3 = (ifp->if_family == IFNET_FAMILY_UTUN ||
348 ifp->if_family == IFNET_FAMILY_IPSEC);
349
350 off = ((flags & NETIF_CONVERT_TX) != 0) ?
351 (uint8_t)ifp->if_tx_headroom : 0;
352 hlen = is_l3 ? 0 : ifnet_hdrlen(ifp);
353
354 /*
355 * The packet coming from the compat path could be empty or has
356 * truncated contents. We have to copy the contents from the
357 * attached mbuf. We also don't support attaching a filter
358 * packet (one that already has a packet attached) to another
359 * filter packet.
360 */
361 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
362 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
363 m = pkt->pkt_mbuf;
364 plen = m_pktlen(m);
365 } else {
366 plen = pkt->pkt_length;
367 }
368 ASSERT(pp != NULL);
369 if (__improbable((off + plen) > PP_BUF_SIZE_DEF(pp))) {
370 VERIFY(off < PP_BUF_SIZE_DEF(pp));
371 plen = PP_BUF_SIZE_DEF(pp) - off;
372 truncated = TRUE;
373
374 DTRACE_SKYWALK5(pkt__truncated,
375 struct nexus_netif_adapter *, nifna,
376 struct __kern_packet *, pkt, uint8_t, off,
377 int, plen, uint32_t, PP_BUF_SIZE_DEF(pp));
378 STATS_INC(nifs, NETIF_STATS_FILTER_PKT_TRUNCATED);
379 }
380 fpkt = nx_netif_alloc_packet(pp, off + plen, &fph);
381 if (__improbable(fpkt == NULL)) {
382 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
383 nifna, struct __kern_packet *, pkt);
384 STATS_INC(nifs, NETIF_STATS_FILTER_DROP_PKT_ALLOC_FAIL);
385 goto drop;
386 }
387 fpkt->pkt_link_flags = 0;
388 fpkt->pkt_headroom = off;
389 fpkt->pkt_l2_len = hlen;
390 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
391
392 if (__improbable((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) != 0 ||
393 (m != NULL && (m->m_flags & M_HASFCS) != 0))) {
394 if (type != NR_RX) {
395 /*
396 * There shouldn't be an FCS for TX packets
397 */
398 DTRACE_SKYWALK2(bad__flags,
399 struct nexus_netif_adapter *, nifna,
400 struct __kern_packet *, pkt);
401 goto drop;
402 }
403 if (plen > ETHER_CRC_LEN) {
404 plen -= ETHER_CRC_LEN;
405 } else {
406 DTRACE_SKYWALK3(bad__pkt__size,
407 struct nexus_netif_adapter *, nifna,
408 struct __kern_packet *, pkt, int, plen);
409 goto drop;
410 }
411 }
412 /*
413 * XXX
414 * If the source packet has any checksum flags, the filter packet will
415 * not have valid checksums. To fill in the checksums, we need to do
416 * something similar to bridge_finalize_cksum() for packets.
417 */
418 if (m != NULL) {
419 nif->nif_pkt_copy_from_mbuf(type, fph, off, m, 0,
420 plen, FALSE, 0);
421 } else {
422 nif->nif_pkt_copy_from_pkt(type, fph, off, ph,
423 pkt->pkt_headroom, plen, FALSE, 0, 0, FALSE);
424 }
425 ASSERT((fpkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
426 ASSERT((fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
427 ASSERT(fpkt->pkt_pkt == NULL);
428 ASSERT(pkt->pkt_nextpkt == NULL);
429 fpkt->pkt_pkt = pkt;
430 fpkt->pkt_pflags |= PKT_F_PKT_DATA;
431 if (truncated) {
432 fpkt->pkt_pflags |= PKT_F_TRUNCATED;
433 }
434 /*
435 * XXX
436 * Unlike the mbuf case, __packet_finalize below correctly sets
437 * pkt_length to the buflet length (possibly truncated). We set
438 * pkt_length here so that fill_vlan_info can use it.
439 */
440 fpkt->pkt_length = plen;
441 if (!is_l3) {
442 fill_vlan_info(fpkt);
443 }
444 err = __packet_finalize(fph);
445 VERIFY(err == 0);
446 return fpkt;
447 drop:
448 if (fpkt != NULL) {
449 /* ensure pkt hasn't been attached */
450 ASSERT(fpkt->pkt_pkt == NULL &&
451 (fpkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
452 nx_netif_free_packet(fpkt);
453 }
454 STATS_INC(nifs, NETIF_STATS_DROP);
455 nx_netif_free_packet(pkt);
456 return NULL;
457 }
458
459 struct __kern_packet *
nx_netif_pkt_to_filter_pkt_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)460 nx_netif_pkt_to_filter_pkt_chain(struct nexus_netif_adapter *nifna,
461 struct __kern_packet *pkt_chain, uint32_t flags)
462 {
463 struct __kern_packet *pkt = pkt_chain, *next;
464 struct __kern_packet *p_head = NULL, *p;
465 struct __kern_packet **p_tailp = &p_head;
466 int c = 0;
467
468 while (pkt != NULL) {
469 next = pkt->pkt_nextpkt;
470 pkt->pkt_nextpkt = NULL;
471
472 p = nx_netif_pkt_to_filter_pkt(nifna, pkt, flags);
473 if (p != NULL) {
474 c++;
475 *p_tailp = p;
476 p_tailp = &p->pkt_nextpkt;
477 }
478 pkt = next;
479 }
480 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, p_head, int, c);
481 return p_head;
482 }
483
484 static struct __kern_packet *
nx_netif_filter_pkt_to_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * fpkt,uint32_t flags)485 nx_netif_filter_pkt_to_pkt(struct nexus_netif_adapter *nifna,
486 struct __kern_packet *fpkt, uint32_t flags)
487 {
488 #pragma unused (nifna)
489 struct __kern_packet *pkt;
490
491 __check_convert_flags(flags);
492 ASSERT((fpkt->pkt_pflags & PKT_F_PKT_DATA) != 0);
493 ASSERT((fpkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
494
495 pkt = fpkt->pkt_pkt;
496 ASSERT(pkt != NULL);
497 KPKT_CLEAR_PKT_DATA(fpkt);
498 nx_netif_free_packet(fpkt);
499 return pkt;
500 }
501
502 struct __kern_packet *
nx_netif_filter_pkt_to_pkt_chain(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt_chain,uint32_t flags)503 nx_netif_filter_pkt_to_pkt_chain(struct nexus_netif_adapter *nifna,
504 struct __kern_packet *pkt_chain, uint32_t flags)
505 {
506 struct __kern_packet *pkt = pkt_chain, *next;
507 struct __kern_packet *p_head = NULL, *p;
508 struct __kern_packet **p_tailp = &p_head;
509 int c = 0;
510
511 while (pkt != NULL) {
512 next = pkt->pkt_nextpkt;
513 pkt->pkt_nextpkt = NULL;
514
515 p = nx_netif_filter_pkt_to_pkt(nifna, pkt, flags);
516 if (p != NULL) {
517 c++;
518 *p_tailp = p;
519 p_tailp = &p->pkt_nextpkt;
520 }
521 pkt = next;
522 }
523 DTRACE_SKYWALK2(pkt__chain, struct __kern_packet *, p_head, int, c);
524 return p_head;
525 }
526
527 struct mbuf *
nx_netif_pkt_to_mbuf(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)528 nx_netif_pkt_to_mbuf(struct nexus_netif_adapter *nifna,
529 struct __kern_packet *pkt, uint32_t flags)
530 {
531 struct nx_netif *nif = nifna->nifna_netif;
532 ifnet_t ifp = nif->nif_ifp;
533 struct mbuf *m;
534 unsigned int one = 1;
535 size_t len;
536 uint16_t pad, hlen;
537 kern_packet_t ph;
538 enum txrx type;
539 int err;
540
541 __check_convert_flags(flags);
542 /* Compat packets or filter packets should never land here */
543 ASSERT((pkt->pkt_pflags & PKT_F_MBUF_DATA) == 0);
544 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
545
546 /* Outbound packets should not have this */
547 ASSERT((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) == 0);
548
549 /* This function is only meant to be used in the custom ether TX path */
550 ASSERT((flags & NETIF_CONVERT_TX) != 0);
551 type = NR_TX;
552
553 /* Packet must include L2 header */
554 hlen = ifnet_hdrlen(ifp);
555 pad = (uint16_t)P2ROUNDUP(hlen, sizeof(uint32_t)) - hlen;
556 len = pkt->pkt_length;
557
558 err = mbuf_allocpacket(MBUF_WAITOK, pad + len, &one, &m);
559 VERIFY(err == 0);
560 m->m_data += pad;
561 m->m_pkthdr.pkt_hdr = mtod(m, uint8_t *);
562 ph = SK_PTR_ENCODE(pkt, METADATA_TYPE(pkt), METADATA_SUBTYPE(pkt));
563
564 nif->nif_pkt_copy_to_mbuf(type, ph, pkt->pkt_headroom,
565 m, 0, (uint32_t)len, FALSE, 0);
566 nx_netif_free_packet(pkt);
567 return m;
568 }
569
570 struct __kern_packet *
nx_netif_pkt_to_pkt(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt,uint32_t flags)571 nx_netif_pkt_to_pkt(struct nexus_netif_adapter *nifna,
572 struct __kern_packet *pkt, uint32_t flags)
573 {
574 struct nx_netif *nif = nifna->nifna_netif;
575 struct nexus_adapter *na = &nifna->nifna_up;
576 struct netif_stats *nifs = &nif->nif_stats;
577 ifnet_t ifp = nif->nif_ifp;
578 struct kern_pbufpool *pp;
579 struct __kern_packet *dpkt = NULL;
580 struct mbuf *m = NULL;
581 uint8_t off, hlen;
582 int len;
583 kern_packet_t ph, dph;
584 enum txrx type;
585 int err;
586
587 __check_convert_flags(flags);
588 /* Filter packets should never land here */
589 ASSERT((pkt->pkt_pflags & PKT_F_PKT_DATA) == 0);
590
591 /* Only support these target NAs for now */
592 type = ((flags & NETIF_CONVERT_TX) != 0) ? NR_TX : NR_RX;
593 if (type == NR_TX) {
594 ASSERT(na->na_type == NA_NETIF_DEV ||
595 na->na_type == NA_NETIF_COMPAT_DEV);
596 pp = skmem_arena_nexus(na->na_arena)->arn_tx_pp;
597 off = (uint8_t)ifp->if_tx_headroom;
598 } else {
599 ASSERT(na->na_type == NA_NETIF_VP ||
600 na->na_type == NA_NETIF_DEV);
601 pp = skmem_arena_nexus(na->na_arena)->arn_rx_pp;
602 off = 0;
603 }
604 /* Packet must include L2 header */
605 hlen = ifnet_hdrlen(ifp);
606
607 /*
608 * Source packet has no data. Need to copy from the attached mbuf.
609 */
610 if ((pkt->pkt_pflags & PKT_F_MBUF_DATA) != 0) {
611 /* An outbound packet shouldn't have an mbuf attached */
612 ASSERT(na->na_type == NA_NETIF_VP ||
613 na->na_type == NA_NETIF_DEV);
614 m = pkt->pkt_mbuf;
615 len = m_pktlen(m);
616 } else {
617 len = pkt->pkt_length;
618 }
619 ASSERT(pp != NULL);
620
621 ph = SK_PKT2PH(pkt);
622 dpkt = nx_netif_alloc_packet(pp, off + len, &dph);
623 if (__improbable(dpkt == NULL)) {
624 if (type == NR_TX) {
625 STATS_INC(nifs, NETIF_STATS_VP_DROP_TX_ALLOC_FAIL);
626 } else {
627 STATS_INC(nifs, NETIF_STATS_VP_DROP_RX_ALLOC_FAIL);
628 }
629 DTRACE_SKYWALK2(alloc__fail, struct nexus_netif_adapter *,
630 nifna, struct __kern_packet *, pkt);
631 goto drop;
632 }
633 if (__improbable((off + len) > PP_BUF_SIZE_DEF(pp))) {
634 STATS_INC(nifs, NETIF_STATS_VP_DROP_PKT_TOO_BIG);
635 DTRACE_SKYWALK5(pkt__too__large,
636 struct nexus_netif_adapter *, nifna,
637 struct __kern_packet *, pkt, uint8_t, off, int, len,
638 uint32_t, PP_BUF_SIZE_DEF(pp));
639 goto drop;
640 }
641 if (__improbable((pkt->pkt_link_flags & PKT_LINKF_ETHFCS) != 0 ||
642 (m != NULL && (m->m_flags & M_HASFCS) != 0))) {
643 if (type != NR_RX) {
644 /*
645 * There shouldn't be an FCS for TX packets
646 */
647 DTRACE_SKYWALK2(bad__flags,
648 struct nexus_netif_adapter *, nifna,
649 struct __kern_packet *, pkt);
650 goto drop;
651 }
652 if (len > ETHER_CRC_LEN) {
653 len -= ETHER_CRC_LEN;
654 } else {
655 DTRACE_SKYWALK3(bad__pkt__size,
656 struct nexus_netif_adapter *, nifna,
657 struct __kern_packet *, pkt, int, len);
658 goto drop;
659 }
660 }
661 dpkt->pkt_link_flags = 0;
662 dpkt->pkt_headroom = off;
663 dpkt->pkt_l2_len = hlen;
664
665 /* Copy optional metadata */
666 dpkt->pkt_pflags = (pkt->pkt_pflags & PKT_F_COPY_MASK);
667 _PKT_COPY_OPT_DATA(pkt, dpkt);
668
669 /* Copy Transmit completion metadata */
670 _PKT_COPY_TX_PORT_DATA(pkt, dpkt);
671
672 /* Copy packet contents */
673 if (m != NULL) {
674 nif->nif_pkt_copy_from_mbuf(type, dph, off, m, 0,
675 len, FALSE, 0);
676 } else {
677 nif->nif_pkt_copy_from_pkt(type, dph, off, ph,
678 pkt->pkt_headroom, len, FALSE, 0, 0, FALSE);
679 }
680 err = __packet_finalize(dph);
681 VERIFY(err == 0);
682 nx_netif_free_packet(pkt);
683 return dpkt;
684
685 drop:
686 if (dpkt != NULL) {
687 nx_netif_free_packet(dpkt);
688 }
689 STATS_INC(nifs, NETIF_STATS_DROP);
690 nx_netif_free_packet(pkt);
691 return NULL;
692 }
693
694 void
nx_netif_mbuf_chain_info(struct mbuf * m_head,struct mbuf ** m_tail,uint32_t * cnt,uint32_t * bytes)695 nx_netif_mbuf_chain_info(struct mbuf *m_head, struct mbuf **m_tail,
696 uint32_t *cnt, uint32_t *bytes)
697 {
698 struct mbuf *m = m_head, *tail = NULL;
699 uint32_t c = 0, b = 0;
700
701 while (m != NULL) {
702 c++;
703 b += m_pktlen(m);
704 tail = m;
705 m = m->m_nextpkt;
706 }
707 if (m_tail != NULL) {
708 *m_tail = tail;
709 }
710 if (cnt != NULL) {
711 *cnt = c;
712 }
713 if (bytes != NULL) {
714 *bytes = b;
715 }
716 }
717
718 void
nx_netif_pkt_chain_info(struct __kern_packet * p_head,struct __kern_packet ** p_tail,uint32_t * cnt,uint32_t * bytes)719 nx_netif_pkt_chain_info(struct __kern_packet *p_head,
720 struct __kern_packet **p_tail, uint32_t *cnt, uint32_t *bytes)
721 {
722 struct __kern_packet *p = p_head, *tail = NULL;
723 uint32_t c = 0, b = 0;
724
725 while (p != NULL) {
726 c++;
727 b += p->pkt_length;
728 tail = p;
729 p = p->pkt_nextpkt;
730 }
731 if (p_tail != NULL) {
732 *p_tail = tail;
733 }
734 if (cnt != NULL) {
735 *cnt = c;
736 }
737 if (bytes != NULL) {
738 *bytes = b;
739 }
740 }
741
742 int
nx_netif_get_max_mtu(ifnet_t ifp,uint32_t * max_mtu)743 nx_netif_get_max_mtu(ifnet_t ifp, uint32_t *max_mtu)
744 {
745 struct ifreq ifr;
746 int err;
747
748 bzero(&ifr, sizeof(ifr));
749 err = ifnet_ioctl(ifp, 0, SIOCGIFDEVMTU, &ifr);
750 if (err != 0) {
751 SK_ERR("SIOCGIFDEVMTU failed for %s\n", if_name(ifp));
752 return err;
753 }
754 *max_mtu = MAX(ifr.ifr_devmtu.ifdm_max, ifr.ifr_devmtu.ifdm_current);
755 return 0;
756 }
757
758 void
nx_netif_pktap_output(ifnet_t ifp,int af,struct __kern_packet * pkt)759 nx_netif_pktap_output(ifnet_t ifp, int af, struct __kern_packet *pkt)
760 {
761 uint32_t dlt;
762 uint32_t flags = PTH_FLAG_SOCKET;
763
764 switch (ifp->if_family) {
765 case IFNET_FAMILY_ETHERNET:
766 dlt = DLT_EN10MB;
767 break;
768 case IFNET_FAMILY_CELLULAR:
769 case IFNET_FAMILY_UTUN:
770 case IFNET_FAMILY_IPSEC:
771 dlt = DLT_RAW;
772 break;
773 default:
774 DTRACE_SKYWALK1(invalid__family, ifnet_t, ifp);
775 return;
776 }
777 if ((pkt->pkt_pflags & PKT_F_KEEPALIVE) != 0) {
778 flags |= PTH_FLAG_KEEP_ALIVE;
779 }
780 if ((pkt->pkt_pflags & PKT_F_REXMT) != 0) {
781 flags |= PTH_FLAG_REXMIT;
782 }
783 pktap_output_packet(ifp, af, dlt, -1, NULL, -1, NULL, SK_PKT2PH(pkt),
784 NULL, 0, pkt->pkt_flow_ip_proto, pkt->pkt_flow_token, flags);
785 }
786
787 __attribute__((always_inline))
788 inline void
netif_ifp_inc_traffic_class_out_pkt(struct ifnet * ifp,uint32_t svc,uint32_t cnt,uint32_t len)789 netif_ifp_inc_traffic_class_out_pkt(struct ifnet *ifp, uint32_t svc,
790 uint32_t cnt, uint32_t len)
791 {
792 switch (svc) {
793 case PKT_TC_BE:
794 ifp->if_tc.ifi_obepackets += cnt;
795 ifp->if_tc.ifi_obebytes += len;
796 break;
797 case PKT_TC_BK:
798 ifp->if_tc.ifi_obkpackets += cnt;
799 ifp->if_tc.ifi_obkbytes += len;
800 break;
801 case PKT_TC_VI:
802 ifp->if_tc.ifi_ovipackets += cnt;
803 ifp->if_tc.ifi_ovibytes += len;
804 break;
805 case PKT_TC_VO:
806 ifp->if_tc.ifi_ovopackets += cnt;
807 ifp->if_tc.ifi_ovobytes += len;
808 break;
809 default:
810 break;
811 }
812 }
813