1 /*
2 * Copyright (c) 1999-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <net/if_var.h>
30 #include <net/dlil_var_private.h>
31 #include <net/dlil.h>
32 #include <net/dlil_sysctl.h>
33
34
35 static void dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls);
36
37 static int dlil_interface_filters_output(struct ifnet *ifp, struct mbuf **m_p,
38 protocol_family_t protocol_family);
39
40 static void dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff,
41 protocol_family_t pf);
42
43 #if CONFIG_DTRACE
44 static void dlil_output_dtrace(ifnet_t ifp, protocol_family_t proto_family, mbuf_t m);
45 #endif /* CONFIG_DTRACE */
46
47 /*
48 * dlil_output
49 *
50 * Caller should have a lock on the protocol domain if the protocol
51 * doesn't support finer grained locking. In most cases, the lock
52 * will be held from the socket layer and won't be released until
53 * we return back to the socket layer.
54 *
55 * This does mean that we must take a protocol lock before we take
56 * an interface lock if we're going to take both. This makes sense
57 * because a protocol is likely to interact with an ifp while it
58 * is under the protocol lock.
59 *
60 * An advisory code will be returned if adv is not null. This
61 * can be used to provide feedback about interface queues to the
62 * application.
63 */
64 errno_t
dlil_output(ifnet_t ifp,protocol_family_t proto_family,mbuf_t packetlist,void * route,const struct sockaddr * dest,int flags,struct flowadv * adv)65 dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist,
66 void *route, const struct sockaddr *dest, int flags, struct flowadv *adv)
67 {
68 char *frame_type = NULL;
69 char *dst_linkaddr = NULL;
70 int retval = 0;
71 char frame_type_buffer[IFNET_MAX_FRAME_TYPE_BUFFER_SIZE];
72 char dst_linkaddr_buffer[IFNET_MAX_LINKADDR_BUFFER_SIZE];
73 if_proto_ref_t proto = NULL;
74 mbuf_ref_t m = NULL;
75 mbuf_ref_t send_head = NULL;
76 mbuf_ref_ptr_t send_tail = &send_head;
77 int iorefcnt = 0;
78 u_int32_t pre = 0, post = 0;
79 u_int32_t fpkts = 0, fbytes = 0;
80 int32_t flen = 0;
81 struct timespec now;
82 u_int64_t now_nsec;
83 boolean_t did_clat46 = FALSE;
84 protocol_family_t old_proto_family = proto_family;
85 struct sockaddr_in6 dest6;
86 rtentry_ref_t rt = NULL;
87 u_int16_t m_loop_set = 0;
88 bool raw = (flags & DLIL_OUTPUT_FLAGS_RAW) != 0;
89
90 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
91
92 /*
93 * Get an io refcnt if the interface is attached to prevent ifnet_detach
94 * from happening while this operation is in progress
95 */
96 if (!ifnet_datamov_begin(ifp)) {
97 retval = ENXIO;
98 goto cleanup;
99 }
100 iorefcnt = 1;
101
102 VERIFY(ifp->if_output_dlil != NULL);
103
104 /* update the driver's multicast filter, if needed */
105 if (ifp->if_updatemcasts > 0) {
106 if_mcasts_update_async(ifp);
107 ifp->if_updatemcasts = 0;
108 }
109
110 frame_type = frame_type_buffer;
111 dst_linkaddr = dst_linkaddr_buffer;
112
113 if (flags == DLIL_OUTPUT_FLAGS_NONE) {
114 ifnet_lock_shared(ifp);
115 /* callee holds a proto refcnt upon success */
116 proto = find_attached_proto(ifp, proto_family);
117 if (proto == NULL) {
118 ifnet_lock_done(ifp);
119 retval = ENXIO;
120 goto cleanup;
121 }
122 ifnet_lock_done(ifp);
123 }
124
125 preout_again:
126 if (packetlist == NULL) {
127 goto cleanup;
128 }
129
130 m = packetlist;
131 packetlist = packetlist->m_nextpkt;
132 m->m_nextpkt = NULL;
133
134 m_add_crumb(m, PKT_CRUMB_DLIL_OUTPUT);
135
136 /*
137 * Perform address family translation for the first
138 * packet outside the loop in order to perform address
139 * lookup for the translated proto family.
140 */
141 if (proto_family == PF_INET && IS_INTF_CLAT46(ifp) &&
142 (ifp->if_type == IFT_CELLULAR ||
143 dlil_is_clat_needed(proto_family, m))) {
144 retval = dlil_clat46(ifp, &proto_family, &m);
145 /*
146 * Go to the next packet if translation fails
147 */
148 if (retval != 0) {
149 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_CLAT64, NULL, 0);
150 m = NULL;
151 ip6stat.ip6s_clat464_out_drop++;
152 /* Make sure that the proto family is PF_INET */
153 ASSERT(proto_family == PF_INET);
154 goto preout_again;
155 }
156 /*
157 * Free the old one and make it point to the IPv6 proto structure.
158 *
159 * Change proto for the first time we have successfully
160 * performed address family translation.
161 */
162 if (!did_clat46 && proto_family == PF_INET6) {
163 did_clat46 = TRUE;
164
165 if (proto != NULL) {
166 if_proto_free(proto);
167 }
168 ifnet_lock_shared(ifp);
169 /* callee holds a proto refcnt upon success */
170 proto = find_attached_proto(ifp, proto_family);
171 if (proto == NULL) {
172 ifnet_lock_done(ifp);
173 retval = ENXIO;
174 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_CLAT64, NULL, 0);
175 m = NULL;
176 goto cleanup;
177 }
178 ifnet_lock_done(ifp);
179 if (ifp->if_type == IFT_ETHER) {
180 /* Update the dest to translated v6 address */
181 dest6.sin6_len = sizeof(struct sockaddr_in6);
182 dest6.sin6_family = AF_INET6;
183 dest6.sin6_addr = (mtod(m, struct ip6_hdr *))->ip6_dst;
184 dest = SA(&dest6);
185
186 /*
187 * Lookup route to the translated destination
188 * Free this route ref during cleanup
189 */
190 rt = rtalloc1_scoped(SA(&dest6),
191 0, 0, ifp->if_index);
192
193 route = rt;
194 }
195 }
196 }
197
198 /*
199 * This path gets packet chain going to the same destination.
200 * The pre output routine is used to either trigger resolution of
201 * the next hop or retrieve the next hop's link layer addressing.
202 * For ex: ether_inet(6)_pre_output routine.
203 *
204 * If the routine returns EJUSTRETURN, it implies that packet has
205 * been queued, and therefore we have to call preout_again for the
206 * following packet in the chain.
207 *
208 * For errors other than EJUSTRETURN, the current packet is freed
209 * and the rest of the chain (pointed by packetlist is freed as
210 * part of clean up.
211 *
212 * Else if there is no error the retrieved information is used for
213 * all the packets in the chain.
214 */
215 if (flags == DLIL_OUTPUT_FLAGS_NONE) {
216 proto_media_preout preoutp = (proto->proto_kpi == kProtoKPI_v1 ?
217 proto->kpi.v1.pre_output : proto->kpi.v2.pre_output);
218 retval = 0;
219 if (preoutp != NULL) {
220 retval = preoutp(ifp, proto_family, &m, dest, route,
221 frame_type, dst_linkaddr);
222
223 if (retval != 0) {
224 if (retval == EJUSTRETURN) {
225 goto preout_again;
226 }
227 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_PRE_OUTPUT, NULL, 0);
228 m = NULL;
229 goto cleanup;
230 }
231 }
232 }
233
234 nanouptime(&now);
235 net_timernsec(&now, &now_nsec);
236
237 do {
238 m_add_hdr_crumb_interface_output(m, ifp->if_index, false);
239 /*
240 * pkt_hdr is set here to point to m_data prior to
241 * calling into the framer. This value of pkt_hdr is
242 * used by the netif gso logic to retrieve the ip header
243 * for the TCP packets, offloaded for TSO processing.
244 */
245 if (raw && (ifp->if_family == IFNET_FAMILY_ETHERNET)) {
246 uint8_t vlan_encap_len = 0;
247
248 if ((m->m_pkthdr.csum_flags & CSUM_VLAN_ENCAP_PRESENT) != 0) {
249 vlan_encap_len = ETHER_VLAN_ENCAP_LEN;
250 }
251 m->m_pkthdr.pkt_hdr = mtod(m, char *) + ETHER_HDR_LEN + vlan_encap_len;
252 } else {
253 m->m_pkthdr.pkt_hdr = mtod(m, void *);
254 }
255
256 /*
257 * Perform address family translation if needed.
258 * For now we only support stateless 4 to 6 translation
259 * on the out path.
260 *
261 * The routine below translates IP header, updates protocol
262 * checksum and also translates ICMP.
263 *
264 * We skip the first packet as it is already translated and
265 * the proto family is set to PF_INET6.
266 */
267 if (proto_family == PF_INET && IS_INTF_CLAT46(ifp) &&
268 (ifp->if_type == IFT_CELLULAR ||
269 dlil_is_clat_needed(proto_family, m))) {
270 retval = dlil_clat46(ifp, &proto_family, &m);
271 /* Goto the next packet if the translation fails */
272 if (retval != 0) {
273 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_CLAT64, NULL, 0);
274 m = NULL;
275 ip6stat.ip6s_clat464_out_drop++;
276 goto next;
277 }
278 }
279
280 #if CONFIG_DTRACE
281 if (flags == DLIL_OUTPUT_FLAGS_NONE) {
282 dlil_output_dtrace(ifp, proto_family, m);
283 }
284 #endif /* CONFIG_DTRACE */
285
286 if (flags == DLIL_OUTPUT_FLAGS_NONE && ifp->if_framer != NULL) {
287 int rcvif_set = 0;
288
289 /*
290 * If this is a broadcast packet that needs to be
291 * looped back into the system, set the inbound ifp
292 * to that of the outbound ifp. This will allow
293 * us to determine that it is a legitimate packet
294 * for the system. Only set the ifp if it's not
295 * already set, just to be safe.
296 */
297 if ((m->m_flags & (M_BCAST | M_LOOP)) &&
298 m->m_pkthdr.rcvif == NULL) {
299 m->m_pkthdr.rcvif = ifp;
300 rcvif_set = 1;
301 }
302 m_loop_set = m->m_flags & M_LOOP;
303 retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr,
304 frame_type, &pre, &post);
305 if (retval != 0) {
306 if (retval != EJUSTRETURN) {
307 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_IF_FRAMER, NULL, 0);
308 }
309 goto next;
310 }
311
312 /*
313 * For partial checksum offload, adjust the start
314 * and stuff offsets based on the prepended header.
315 */
316 if ((m->m_pkthdr.csum_flags &
317 (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
318 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
319 m->m_pkthdr.csum_tx_stuff += pre;
320 m->m_pkthdr.csum_tx_start += pre;
321 }
322
323 if (hwcksum_dbg != 0 && !(ifp->if_flags & IFF_LOOPBACK)) {
324 dlil_output_cksum_dbg(ifp, m, pre,
325 proto_family);
326 }
327
328 /*
329 * Clear the ifp if it was set above, and to be
330 * safe, only if it is still the same as the
331 * outbound ifp we have in context. If it was
332 * looped back, then a copy of it was sent to the
333 * loopback interface with the rcvif set, and we
334 * are clearing the one that will go down to the
335 * layer below.
336 */
337 if (rcvif_set && m->m_pkthdr.rcvif == ifp) {
338 m->m_pkthdr.rcvif = NULL;
339 }
340 }
341
342 /*
343 * Let interface filters (if any) do their thing ...
344 */
345 if ((flags & DLIL_OUTPUT_FLAGS_SKIP_IF_FILTERS) == 0) {
346 retval = dlil_interface_filters_output(ifp, &m, proto_family);
347 if (retval != 0) {
348 if (retval != EJUSTRETURN) {
349 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_IF_FILTER, NULL, 0);
350 }
351 goto next;
352 }
353 }
354 /*
355 * Strip away M_PROTO1 bit prior to sending packet
356 * to the driver as this field may be used by the driver
357 */
358 m->m_flags &= ~M_PROTO1;
359
360 /*
361 * If the underlying interface is not capable of handling a
362 * packet whose data portion spans across physically disjoint
363 * pages, we need to "normalize" the packet so that we pass
364 * down a chain of mbufs where each mbuf points to a span that
365 * resides in the system page boundary. If the packet does
366 * not cross page(s), the following is a no-op.
367 */
368 if (!(ifp->if_hwassist & IFNET_MULTIPAGES)) {
369 if ((m = m_normalize(m)) == NULL) {
370 goto next;
371 }
372 }
373
374 /*
375 * If this is a TSO packet, make sure the interface still
376 * advertise TSO capability.
377 */
378 if (TSO_IPV4_NOTOK(ifp, m) || TSO_IPV6_NOTOK(ifp, m)) {
379 retval = EMSGSIZE;
380 m_drop(m, DROPTAP_FLAG_DIR_OUT, DROP_REASON_DLIL_TSO_NOT_OK, NULL, 0);
381 goto cleanup;
382 }
383
384 ifp_inc_traffic_class_out(ifp, m);
385
386 #if SKYWALK
387 /*
388 * For native skywalk devices, packets will be passed to pktap
389 * after GSO or after the mbuf to packet conversion.
390 * This is done for IPv4/IPv6 packets only because there is no
391 * space in the mbuf to pass down the proto family.
392 */
393 if (dlil_is_native_netif_nexus(ifp)) {
394 if (raw || m->m_pkthdr.pkt_proto == 0) {
395 pktap_output(ifp, proto_family, m, pre, post);
396 m->m_pkthdr.pkt_flags |= PKTF_SKIP_PKTAP;
397 }
398 } else {
399 pktap_output(ifp, proto_family, m, pre, post);
400 }
401 #else /* SKYWALK */
402 pktap_output(ifp, proto_family, m, pre, post);
403 #endif /* SKYWALK */
404
405 /*
406 * Count the number of elements in the mbuf chain
407 */
408 if (tx_chain_len_count) {
409 dlil_count_chain_len(m, &tx_chain_len_stats);
410 }
411
412 /*
413 * Discard partial sum information if this packet originated
414 * from another interface; the packet would already have the
415 * final checksum and we shouldn't recompute it.
416 */
417 if ((m->m_pkthdr.pkt_flags & PKTF_FORWARDED) &&
418 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
419 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
420 m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS;
421 m->m_pkthdr.csum_data = 0;
422 }
423
424 /*
425 * Finally, call the driver.
426 */
427 if (ifp->if_eflags & (IFEF_SENDLIST | IFEF_ENQUEUE_MULTI)) {
428 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
429 flen += (m_pktlen(m) - (pre + post));
430 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
431 }
432 (void) mbuf_set_timestamp(m, now_nsec, TRUE);
433
434 *send_tail = m;
435 send_tail = &m->m_nextpkt;
436 } else {
437 /*
438 * Record timestamp; ifnet_enqueue() will use this info
439 * rather than redoing the work.
440 */
441 nanouptime(&now);
442 net_timernsec(&now, &now_nsec);
443 (void) mbuf_set_timestamp(m, now_nsec, TRUE);
444
445 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
446 flen = (m_pktlen(m) - (pre + post));
447 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
448 } else {
449 flen = 0;
450 }
451 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
452 0, 0, 0, 0, 0);
453 retval = (*ifp->if_output_dlil)(ifp, m);
454 if (retval == EQFULL || retval == EQSUSPENDED) {
455 if (adv != NULL && adv->code == FADV_SUCCESS) {
456 adv->code = (retval == EQFULL ?
457 FADV_FLOW_CONTROLLED :
458 FADV_SUSPENDED);
459 }
460 retval = 0;
461 }
462 if (retval == 0 && flen > 0) {
463 fbytes += flen;
464 fpkts++;
465 }
466 if (retval != 0 && dlil_verbose) {
467 DLIL_PRINTF("%s: output error on %s retval = %d\n",
468 __func__, if_name(ifp),
469 retval);
470 }
471 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END,
472 0, 0, 0, 0, 0);
473 }
474 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
475
476 next:
477 m = packetlist;
478 if (m != NULL) {
479 m->m_flags |= m_loop_set;
480 packetlist = packetlist->m_nextpkt;
481 m->m_nextpkt = NULL;
482 }
483 /* Reset the proto family to old proto family for CLAT */
484 if (did_clat46) {
485 proto_family = old_proto_family;
486 }
487 } while (m != NULL);
488
489 if (send_head != NULL) {
490 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
491 0, 0, 0, 0, 0);
492 if (ifp->if_eflags & IFEF_SENDLIST) {
493 retval = (*ifp->if_output_dlil)(ifp, send_head);
494 if (retval == EQFULL || retval == EQSUSPENDED) {
495 if (adv != NULL) {
496 adv->code = (retval == EQFULL ?
497 FADV_FLOW_CONTROLLED :
498 FADV_SUSPENDED);
499 }
500 retval = 0;
501 }
502 if (retval == 0 && flen > 0) {
503 fbytes += flen;
504 fpkts++;
505 }
506 if (retval != 0 && dlil_verbose) {
507 DLIL_PRINTF("%s: output error on %s retval = %d\n",
508 __func__, if_name(ifp), retval);
509 }
510 } else {
511 struct mbuf *send_m;
512 int enq_cnt = 0;
513 VERIFY(ifp->if_eflags & IFEF_ENQUEUE_MULTI);
514 while (send_head != NULL) {
515 send_m = send_head;
516 send_head = send_m->m_nextpkt;
517 send_m->m_nextpkt = NULL;
518 retval = (*ifp->if_output_dlil)(ifp, send_m);
519 if (retval == EQFULL || retval == EQSUSPENDED) {
520 if (adv != NULL) {
521 adv->code = (retval == EQFULL ?
522 FADV_FLOW_CONTROLLED :
523 FADV_SUSPENDED);
524 }
525 retval = 0;
526 }
527 if (retval == 0) {
528 enq_cnt++;
529 if (flen > 0) {
530 fpkts++;
531 }
532 }
533 if (retval != 0 && dlil_verbose) {
534 DLIL_PRINTF("%s: output error on %s "
535 "retval = %d\n",
536 __func__, if_name(ifp), retval);
537 }
538 }
539 if (enq_cnt > 0) {
540 fbytes += flen;
541 ifnet_start(ifp);
542 }
543 }
544 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
545 }
546
547 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
548
549 cleanup:
550 if (fbytes > 0) {
551 ifp->if_fbytes += fbytes;
552 }
553 if (fpkts > 0) {
554 ifp->if_fpackets += fpkts;
555 }
556 if (proto != NULL) {
557 if_proto_free(proto);
558 }
559 if (packetlist) { /* if any packets are left, clean up */
560 mbuf_freem_list(packetlist);
561 }
562 if (retval == EJUSTRETURN) {
563 retval = 0;
564 }
565 if (iorefcnt == 1) {
566 ifnet_datamov_end(ifp);
567 }
568 if (rt != NULL) {
569 rtfree(rt);
570 rt = NULL;
571 }
572
573 return retval;
574 }
575
576
577 /*
578 * Static function implementations.
579 */
580 static void
dlil_count_chain_len(mbuf_t m,struct chain_len_stats * cls)581 dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls)
582 {
583 mbuf_t n = m;
584 int chainlen = 0;
585
586 while (n != NULL) {
587 chainlen++;
588 n = n->m_next;
589 }
590 switch (chainlen) {
591 case 0:
592 break;
593 case 1:
594 os_atomic_inc(&cls->cls_one, relaxed);
595 break;
596 case 2:
597 os_atomic_inc(&cls->cls_two, relaxed);
598 break;
599 case 3:
600 os_atomic_inc(&cls->cls_three, relaxed);
601 break;
602 case 4:
603 os_atomic_inc(&cls->cls_four, relaxed);
604 break;
605 case 5:
606 default:
607 os_atomic_inc(&cls->cls_five_or_more, relaxed);
608 break;
609 }
610 }
611
612
613 __attribute__((noinline))
614 static int
dlil_interface_filters_output(struct ifnet * ifp,struct mbuf ** m_p,protocol_family_t protocol_family)615 dlil_interface_filters_output(struct ifnet *ifp, struct mbuf **m_p,
616 protocol_family_t protocol_family)
617 {
618 boolean_t is_vlan_packet;
619 struct ifnet_filter *filter;
620 struct mbuf *m = *m_p;
621
622 if (TAILQ_EMPTY(&ifp->if_flt_head)) {
623 return 0;
624 }
625 is_vlan_packet = packet_has_vlan_tag(m);
626
627 /*
628 * Pass the outbound packet to the interface filters
629 */
630 lck_mtx_lock_spin(&ifp->if_flt_lock);
631 /* prevent filter list from changing in case we drop the lock */
632 if_flt_monitor_busy(ifp);
633 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
634 int result;
635
636 /* exclude VLAN packets from external filters PR-3586856 */
637 if (is_vlan_packet &&
638 (filter->filt_flags & DLIL_IFF_INTERNAL) == 0) {
639 continue;
640 }
641
642 if (!filter->filt_skip && filter->filt_output != NULL &&
643 (filter->filt_protocol == 0 ||
644 filter->filt_protocol == protocol_family)) {
645 lck_mtx_unlock(&ifp->if_flt_lock);
646
647 result = filter->filt_output(filter->filt_cookie, ifp,
648 protocol_family, m_p);
649
650 lck_mtx_lock_spin(&ifp->if_flt_lock);
651 if (result != 0) {
652 /* we're done with the filter list */
653 if_flt_monitor_unbusy(ifp);
654 lck_mtx_unlock(&ifp->if_flt_lock);
655 return result;
656 }
657 }
658 }
659 /* we're done with the filter list */
660 if_flt_monitor_unbusy(ifp);
661 lck_mtx_unlock(&ifp->if_flt_lock);
662
663 return 0;
664 }
665
666 __attribute__((noinline))
667 static void
dlil_output_cksum_dbg(struct ifnet * ifp,struct mbuf * m,uint32_t hoff,protocol_family_t pf)668 dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff,
669 protocol_family_t pf)
670 {
671 #pragma unused(ifp)
672 uint32_t did_sw;
673
674 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_FINALIZE_FORCED) ||
675 (m->m_pkthdr.csum_flags & (CSUM_TSO_IPV4 | CSUM_TSO_IPV6))) {
676 return;
677 }
678
679 switch (pf) {
680 case PF_INET:
681 did_sw = in_finalize_cksum(m, hoff, m->m_pkthdr.csum_flags);
682 if (did_sw & CSUM_DELAY_IP) {
683 hwcksum_dbg_finalized_hdr++;
684 }
685 if (did_sw & CSUM_DELAY_DATA) {
686 hwcksum_dbg_finalized_data++;
687 }
688 break;
689 case PF_INET6:
690 /*
691 * Checksum offload should not have been enabled when
692 * extension headers exist; that also means that we
693 * cannot force-finalize packets with extension headers.
694 * Indicate to the callee should it skip such case by
695 * setting optlen to -1.
696 */
697 did_sw = in6_finalize_cksum(m, hoff, -1, -1,
698 m->m_pkthdr.csum_flags);
699 if (did_sw & CSUM_DELAY_IPV6_DATA) {
700 hwcksum_dbg_finalized_data++;
701 }
702 break;
703 default:
704 return;
705 }
706 }
707
708 #if CONFIG_DTRACE
709 __attribute__((noinline))
710 static void
dlil_output_dtrace(ifnet_t ifp,protocol_family_t proto_family,mbuf_t m)711 dlil_output_dtrace(ifnet_t ifp, protocol_family_t proto_family, mbuf_t m)
712 {
713 if (proto_family == PF_INET) {
714 struct ip *ip = mtod(m, struct ip *);
715 DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
716 struct ip *, ip, struct ifnet *, ifp,
717 struct ip *, ip, struct ip6_hdr *, NULL);
718 } else if (proto_family == PF_INET6) {
719 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
720 DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
721 struct ip6_hdr *, ip6, struct ifnet *, ifp,
722 struct ip *, NULL, struct ip6_hdr *, ip6);
723 }
724 }
725 #endif /* CONFIG_DTRACE */
726