1 /*
2 * Copyright (c) 2019-2025 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <skywalk/nexus/netif/nx_netif.h>
31 #include <net/if_vlan_var.h>
32 #include <sys/sdt.h>
33
34 #define NETIF_DEMUX_ALLOC_SLOTS 128
35
36 #define OUTBOUND_CHECK_OFF 0
37 #define OUTBOUND_CHECK_ON 1
38 #define OUTBOUND_CHECK_FORCED 2
39
40 /* Turning this off allows packets to be spoofed for testing purposes */
41 static uint32_t outbound_check = OUTBOUND_CHECK_ON;
42
43 /* This controls the per-NA pool size of custom ether and llw NAs */
44 static uint32_t vp_pool_size = 2048;
45
46 /* This enables zerocopy on llw NAs */
47 static uint32_t vp_zerocopy = 0;
48
49 /* TX Ring size */
50 static uint32_t vp_tx_slots = 0;
51
52 /* RX Ring size */
53 static uint32_t vp_rx_slots = 0;
54
55 /*
56 * Disable all packet validation
57 */
58 uint32_t nx_netif_vp_accept_all = 0;
59
60 static uint16_t nx_netif_vpna_gencnt = 0;
61
62 #if (DEVELOPMENT || DEBUG)
63 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, outbound_check,
64 CTLFLAG_RW | CTLFLAG_LOCKED, &outbound_check, 0,
65 "netif outbound packet validation");
66 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_pool_size,
67 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_pool_size, 0,
68 "netif virtual port pool size");
69 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_zerocopy,
70 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_zerocopy, 0,
71 "netif virtual port zero copy");
72 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_tx_slots,
73 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_tx_slots, 0,
74 "netif virtual port tx slots");
75 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_rx_slots,
76 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_rx_slots, 0,
77 "netif virtual port rx slots");
78 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_accept_all,
79 CTLFLAG_RW | CTLFLAG_LOCKED, &nx_netif_vp_accept_all, 0,
80 "netif accept all");
81 #endif /* (DEVELOPMENT || DEBUG) */
82
83 /* XXX Until rdar://118519573 is resolved, do this as a workaround */
84 #pragma clang diagnostic push
85 #pragma clang diagnostic ignored "-Wcast-align"
86
87 static int
88 netif_vp_na_channel_event_notify(struct nexus_adapter *,
89 struct __kern_channel_event *, uint16_t);
90
91 static void
netif_vp_dump_packet(struct __kern_packet * pkt)92 netif_vp_dump_packet(struct __kern_packet *pkt)
93 {
94 uint8_t *baddr;
95
96 MD_BUFLET_ADDR_ABS(pkt, baddr);
97 ASSERT(baddr != NULL);
98 baddr += pkt->pkt_headroom;
99
100 DTRACE_SKYWALK2(dump__packet, struct __kern_packet *,
101 pkt, uint8_t *, baddr);
102 }
103
104 static int
netif_copy_or_attach_pkt(struct __kern_channel_ring * ring,kern_channel_slot_t slot,struct __kern_packet * pkt)105 netif_copy_or_attach_pkt(struct __kern_channel_ring *ring,
106 kern_channel_slot_t slot, struct __kern_packet *pkt)
107 {
108 kern_packet_t ph;
109 struct __kern_packet *dpkt;
110 errno_t err;
111
112 if (pkt->pkt_qum.qum_pp == ring->ckr_pp) {
113 DTRACE_SKYWALK2(attach__pkt, struct __kern_channel_ring *, ring,
114 struct __kern_packet *, pkt);
115 ph = SK_PKT2PH(pkt);
116 err = kern_packet_finalize(ph);
117 VERIFY(err == 0);
118 } else {
119 DTRACE_SKYWALK2(copy__pkt, struct __kern_channel_ring *, ring,
120 struct __kern_packet *, pkt);
121 dpkt = nx_netif_pkt_to_pkt(NIFNA(ring->ckr_na), pkt,
122 ring->ckr_na->na_type == NA_NETIF_VP ? NETIF_CONVERT_RX :
123 NETIF_CONVERT_TX);
124 if (__improbable(dpkt == NULL)) {
125 return ENOMEM;
126 }
127 ph = SK_PKT2PH(dpkt);
128 }
129 err = kern_channel_slot_attach_packet(ring, slot, ph);
130 VERIFY(err == 0);
131 return 0;
132 }
133
134 static errno_t
netif_deliver_pkt(struct nexus_adapter * na,struct __kern_packet * pkt_chain,uint32_t flags)135 netif_deliver_pkt(struct nexus_adapter *na, struct __kern_packet *pkt_chain,
136 uint32_t flags)
137 {
138 #pragma unused(flags)
139 struct __kern_channel_ring *ring = &na->na_rx_rings[0];
140 struct __kern_packet *pkt = pkt_chain, *next;
141 kern_channel_slot_t last_slot = NULL, slot = NULL;
142 struct nexus_netif_adapter *nifna = NIFNA(na);
143 struct nx_netif *nif = nifna->nifna_netif;
144 struct netif_stats *nifs = &nif->nif_stats;
145 sk_protect_t protect;
146 int cnt = 0, dropcnt = 0, err;
147
148 (void) kr_enter(ring, TRUE);
149 protect = sk_sync_protect();
150
151 if (__improbable(KR_DROP(ring))) {
152 nx_netif_free_packet_chain(pkt, &dropcnt);
153 STATS_ADD(nifs,
154 NETIF_STATS_VP_DROP_USER_RING_DISABLED, dropcnt);
155 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
156 DTRACE_SKYWALK2(ring__drop, struct __kern_channel_ring *, ring,
157 int, dropcnt);
158 sk_sync_unprotect(protect);
159 kr_exit(ring);
160 return ENXIO;
161 }
162 while (pkt != NULL) {
163 slot = kern_channel_get_next_slot(ring, last_slot, NULL);
164 if (slot == NULL) {
165 break;
166 }
167 next = pkt->pkt_nextpkt;
168 pkt->pkt_nextpkt = NULL;
169 netif_vp_dump_packet(pkt);
170 err = netif_copy_or_attach_pkt(ring, slot, pkt);
171 if (__probable(err == 0)) {
172 last_slot = slot;
173 }
174 pkt = next;
175 cnt++;
176 }
177 if (NETIF_IS_LOW_LATENCY(nif)) {
178 STATS_ADD(nifs, NETIF_STATS_VP_LL_DELIVERED, cnt);
179 } else {
180 STATS_ADD(nifs, NETIF_STATS_VP_DELIVERED, cnt);
181 }
182 DTRACE_SKYWALK4(delivered, struct nexus_adapter *, na,
183 struct __kern_channel_ring *, ring, struct __kern_packet *, pkt,
184 int, cnt);
185
186 if (pkt != NULL) {
187 nx_netif_free_packet_chain(pkt, &dropcnt);
188 STATS_ADD(nifs,
189 NETIF_STATS_VP_DROP_USER_RING_NO_SPACE, dropcnt);
190 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
191 DTRACE_SKYWALK2(deliver__drop, struct nexus_adapter *, na,
192 int, dropcnt);
193 }
194 if (last_slot != NULL) {
195 kern_channel_advance_slot(ring, last_slot);
196 }
197 sk_sync_unprotect(protect);
198 kr_exit(ring);
199 if (cnt > 0) {
200 (void) kern_channel_notify(ring, 0);
201 }
202 return 0;
203 }
204
205 static errno_t
netif_deliver_cb(void * arg,void * chain,uint32_t flags)206 netif_deliver_cb(void *arg, void *chain, uint32_t flags)
207 {
208 return netif_deliver_pkt(arg, chain, flags);
209 }
210
211 static int
netif_hwna_rx_get_pkts(struct __kern_channel_ring * ring,struct proc * p,uint32_t flags,struct __kern_packet ** chain)212 netif_hwna_rx_get_pkts(struct __kern_channel_ring *ring, struct proc *p,
213 uint32_t flags, struct __kern_packet **chain)
214 {
215 int err, cnt = 0;
216 sk_protect_t protect;
217 slot_idx_t ktail, idx;
218 struct __kern_packet *__single pkt_chain = NULL;
219 struct __kern_packet **tailp = &pkt_chain;
220 struct netif_stats *nifs = &NIFNA(KRNA(ring))->nifna_netif->nif_stats;
221
222 err = kr_enter(ring, ((flags & NA_NOTEF_CAN_SLEEP) != 0 ||
223 (ring->ckr_flags & CKRF_HOST) != 0));
224 if (err != 0) {
225 SK_DF(SK_VERB_VP,
226 "hwna \"%s\" (%p) kr \"%s\" (%p) krflags 0x%x "
227 "(%d)", KRNA(ring)->na_name, SK_KVA(KRNA(ring)),
228 ring->ckr_name, SK_KVA(ring), ring->ckr_flags, err);
229 STATS_INC(nifs, NETIF_STATS_VP_KR_ENTER_FAIL);
230 return err;
231 }
232 if (__improbable(KR_DROP(ring))) {
233 kr_exit(ring);
234 STATS_INC(nifs, NETIF_STATS_VP_DEV_RING_DISABLED);
235 return ENODEV;
236 }
237 protect = sk_sync_protect();
238
239 err = ring->ckr_na_sync(ring, p, 0);
240 if (err != 0 && err != EAGAIN) {
241 STATS_INC(nifs, NETIF_STATS_VP_SYNC_UNKNOWN_ERR);
242 goto out;
243 }
244 ktail = ring->ckr_ktail;
245 if (__improbable(ring->ckr_khead == ktail)) {
246 SK_DF(SK_VERB_VP,
247 "spurious wakeup on hwna %s (%p)", KRNA(ring)->na_name,
248 SK_KVA(KRNA(ring)));
249 STATS_INC(nifs, NETIF_STATS_VP_SPURIOUS_NOTIFY);
250 err = ENOENT;
251 goto out;
252 }
253 /* get all packets from the ring */
254 idx = ring->ckr_rhead;
255 while (idx != ktail) {
256 struct __kern_slot_desc *ksd = KR_KSD(ring, idx);
257 struct __kern_packet *pkt = ksd->sd_pkt;
258
259 ASSERT(pkt->pkt_nextpkt == NULL);
260 KR_SLOT_DETACH_METADATA(ring, ksd);
261 cnt++;
262 *tailp = pkt;
263 tailp = &pkt->pkt_nextpkt;
264 idx = SLOT_NEXT(idx, ring->ckr_lim);
265 }
266 ring->ckr_rhead = ktail;
267 ring->ckr_rtail = ring->ckr_ktail;
268
269 DTRACE_SKYWALK2(rx__notify, struct __kern_channel_ring *, ring,
270 int, cnt);
271 *chain = pkt_chain;
272 out:
273 sk_sync_unprotect(protect);
274 kr_exit(ring);
275 return err;
276 }
277
278 int
netif_llw_rx_notify(struct __kern_channel_ring * ring,struct proc * p,uint32_t flags)279 netif_llw_rx_notify(struct __kern_channel_ring *ring, struct proc *p,
280 uint32_t flags)
281 {
282 int err;
283 struct __kern_packet *__single pkt_chain = NULL;
284
285 err = netif_hwna_rx_get_pkts(ring, p, flags, &pkt_chain);
286 if (err != 0) {
287 return err;
288 }
289 return nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL,
290 NULL, NETIF_FLOW_SOURCE);
291 }
292
293 static void
netif_change_pending(struct nx_netif * nif)294 netif_change_pending(struct nx_netif *nif)
295 {
296 SK_LOCK_ASSERT_HELD();
297 while ((nif->nif_flags & NETIF_FLAG_CHANGE_PENDING) != 0) {
298 DTRACE_SKYWALK1(change__pending__wait, struct nx_netif *, nif);
299 (void) msleep(&nif->nif_flags, &sk_lock, (PZERO - 1),
300 __func__, NULL);
301 DTRACE_SKYWALK1(change__pending__wake, struct nx_netif *, nif);
302 }
303 nif->nif_flags |= NETIF_FLAG_CHANGE_PENDING;
304 }
305
306 static void
netif_change_done(struct nx_netif * nif)307 netif_change_done(struct nx_netif *nif)
308 {
309 SK_LOCK_ASSERT_HELD();
310 ASSERT((nif->nif_flags & NETIF_FLAG_CHANGE_PENDING) != 0);
311 nif->nif_flags &= ~NETIF_FLAG_CHANGE_PENDING;
312 wakeup(&nif->nif_flags);
313 }
314
315 static errno_t
netif_hwna_setup(struct nx_netif * nif)316 netif_hwna_setup(struct nx_netif *nif)
317 {
318 struct kern_channel *ch;
319 struct kern_nexus *nx = nif->nif_nx;
320 struct chreq chr;
321 int err;
322
323 SK_LOCK_ASSERT_HELD();
324 ASSERT(NETIF_IS_LOW_LATENCY(nif));
325 /*
326 * Because sk_lock is released within some functions below, we need
327 * this extra synchronization to ensure that netif_hwna_setup/
328 * netif_hwna_teardown can run atomically.
329 */
330 netif_change_pending(nif);
331 if (nif->nif_hw_ch != NULL) {
332 nif->nif_hw_ch_refcnt++;
333 SK_DF(SK_VERB_VP, "%s: hw channel already open, refcnt %d",
334 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
335 netif_change_done(nif);
336 return 0;
337 }
338 ASSERT(nif->nif_hw_ch_refcnt == 0);
339 bzero(&chr, sizeof(chr));
340 uuid_copy(chr.cr_spec_uuid, nx->nx_uuid);
341 chr.cr_ring_id = 0;
342 chr.cr_port = NEXUS_PORT_NET_IF_DEV;
343 chr.cr_mode |= CHMODE_CONFIG;
344
345 err = 0;
346 ch = ch_open_special(nx, &chr, FALSE, &err);
347 if (ch == NULL) {
348 SK_ERR("%s: failed to open nx %p (err %d)",
349 if_name(nif->nif_ifp), SK_KVA(nx), err);
350 netif_change_done(nif);
351 return err;
352 }
353 netif_hwna_set_mode(ch->ch_na, NETIF_MODE_LLW, NULL);
354 na_start_spec(nx, ch);
355 nif->nif_hw_ch_refcnt = 1;
356 nif->nif_hw_ch = ch;
357 SK_DF(SK_VERB_VP, "%s: hw channel opened %p, %s:%s",
358 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
359 NX_DOM_PROV(nx)->nxdom_prov_name);
360 netif_change_done(nif);
361 return 0;
362 }
363
364 static void
netif_hwna_teardown(struct nx_netif * nif)365 netif_hwna_teardown(struct nx_netif *nif)
366 {
367 struct kern_nexus *nx = nif->nif_nx;
368 struct kern_channel *ch = nif->nif_hw_ch;
369
370 SK_LOCK_ASSERT_HELD();
371 ASSERT(NETIF_IS_LOW_LATENCY(nif));
372 ASSERT(ch != NULL);
373 netif_change_pending(nif);
374 if (--nif->nif_hw_ch_refcnt > 0) {
375 SK_DF(SK_VERB_VP, "%s: hw channel still open, refcnt %d",
376 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
377 netif_change_done(nif);
378 return;
379 }
380 SK_DF(SK_VERB_VP, "%s: hw channel closing %p, %s:%s",
381 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
382 NX_DOM_PROV(nx)->nxdom_prov_name);
383
384 na_stop_spec(nx, ch);
385 netif_hwna_clear_mode(ch->ch_na);
386 ch_close_special(ch);
387 (void) ch_release_locked(ch);
388 nif->nif_hw_ch = NULL;
389 SK_DF(SK_VERB_VP, "%s: hw channel closed, %s:%s",
390 if_name(nif->nif_ifp), NX_DOM(nx)->nxdom_name,
391 NX_DOM_PROV(nx)->nxdom_prov_name);
392 netif_change_done(nif);
393 }
394
395 static int
netif_vp_na_activate_on(struct nexus_adapter * na)396 netif_vp_na_activate_on(struct nexus_adapter *na)
397 {
398 errno_t err;
399 struct netif_flow *__single nf;
400 struct netif_port_info npi;
401 struct nexus_netif_adapter *nifna;
402 struct nx_netif *nif;
403 boolean_t hwna_setup = FALSE;
404
405 nifna = NIFNA(na);
406 nif = nifna->nifna_netif;
407
408 /* lock needed to protect against nxdom_unbind_port */
409 NETIF_WLOCK(nif);
410 err = nx_port_get_info(nif->nif_nx, na->na_nx_port,
411 NX_PORT_INFO_TYPE_NETIF, &npi, sizeof(npi));
412 NETIF_WUNLOCK(nif);
413 if (err != 0) {
414 SK_ERR("port info not found: %d", err);
415 return err;
416 }
417 if (NETIF_IS_LOW_LATENCY(nif)) {
418 err = netif_hwna_setup(nif);
419 if (err != 0) {
420 return err;
421 }
422 hwna_setup = TRUE;
423 }
424 err = nx_netif_flow_add(nif, na->na_nx_port, &npi.npi_fd, na,
425 netif_deliver_cb, &nf);
426 if (err != 0) {
427 if (hwna_setup) {
428 netif_hwna_teardown(nif);
429 }
430 return err;
431 }
432 nifna->nifna_flow = nf;
433 os_atomic_inc(&nx_netif_vpna_gencnt, relaxed);
434 nifna->nifna_gencnt = nx_netif_vpna_gencnt;
435 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
436 return 0;
437 }
438
439 static int
netif_vp_na_activate_off(struct nexus_adapter * na)440 netif_vp_na_activate_off(struct nexus_adapter *na)
441 {
442 errno_t err;
443 struct nexus_netif_adapter *nifna;
444 struct nx_netif *nif;
445
446 if (!NA_IS_ACTIVE(na)) {
447 DTRACE_SKYWALK1(already__off, struct nexus_adapter *, na);
448 return 0;
449 }
450 nifna = NIFNA(na);
451 nif = nifna->nifna_netif;
452 err = nx_netif_flow_remove(nif, nifna->nifna_flow);
453 VERIFY(err == 0);
454
455 nifna->nifna_flow = NULL;
456 if (NETIF_IS_LOW_LATENCY(nif)) {
457 netif_hwna_teardown(nif);
458 }
459 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
460 return 0;
461 }
462
463 static int
netif_vp_na_activate(struct nexus_adapter * na,na_activate_mode_t mode)464 netif_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
465 {
466 errno_t err;
467
468 ASSERT(na->na_type == NA_NETIF_VP);
469 if (mode == NA_ACTIVATE_MODE_ON) {
470 err = netif_vp_na_activate_on(na);
471 } else {
472 err = netif_vp_na_activate_off(na);
473 }
474 SK_DF(SK_VERB_VP, "na \"%s\" (%p) %s err %d", na->na_name,
475 SK_KVA(na), na_activate_mode2str(mode), err);
476 return err;
477 }
478
479 /*
480 * A variation of netif data path for llw, which sends packets to the appropriate
481 * HW queue directly, bypassing AQM.
482 */
483 static int
netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,struct proc * p)484 netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter *dev_nifna,
485 struct __kern_packet *pkt_chain, struct proc *p)
486 {
487 #pragma unused(p)
488 struct __kern_packet *pkt = pkt_chain, *next;
489 struct __kern_packet *dpkt, *__single dpkt_head = NULL, **dpkt_tail = &dpkt_head;
490 struct __kern_channel_ring *ring = &dev_nifna->nifna_up.na_tx_rings[0];
491 #pragma unused(ring)
492 struct nx_netif *nif;
493 struct netif_queue *drvq;
494 struct netif_stats *nifs;
495 struct kern_nexus_provider *nxprov;
496 uint32_t pkt_count = 0, byte_count = 0;
497 errno_t err;
498
499 nif = dev_nifna->nifna_netif;
500 nifs = &nif->nif_stats;
501
502 while (pkt != NULL) {
503 next = pkt->pkt_nextpkt;
504 pkt->pkt_nextpkt = NULL;
505
506 netif_vp_dump_packet(pkt);
507
508 dpkt = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX);
509 if (__improbable(dpkt == NULL)) {
510 pkt = next; // nx_netif_pkt_to_pkt() frees pkt on error as well
511 err = ENOMEM;
512 goto error;
513 }
514
515 *dpkt_tail = dpkt;
516 dpkt_tail = &dpkt->pkt_nextpkt;
517
518 pkt = next;
519 }
520
521 nxprov = NX_PROV(nif->nif_nx);
522 drvq = NETIF_QSET_TX_QUEUE(nif->nif_default_llink->nll_default_qset, 0);
523
524 kern_packet_t ph = SK_PKT2PH(dpkt_head);
525
526 err = nxprov->nxprov_netif_ext.nxnpi_queue_tx_push(nxprov,
527 nif->nif_nx, drvq->nq_ctx, &ph, &pkt_count, &byte_count);
528
529 STATS_ADD(nifs, NETIF_STATS_VP_LL_SENT, pkt_count);
530 DTRACE_SKYWALK3(ll__sent, struct __kern_channel_ring *, ring,
531 uint32_t, pkt_count, uint32_t, byte_count);
532
533 kern_netif_increment_queue_stats(drvq, pkt_count, byte_count);
534
535 /*
536 * Free all unconsumed packets.
537 */
538 if (ph != 0) {
539 int dropcnt;
540
541 nx_netif_free_packet_chain(SK_PTR_ADDR_KPKT(ph), &dropcnt);
542
543 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
544 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
545 int, dropcnt);
546 }
547
548 return err;
549
550 error:
551 /*
552 * Free all packets.
553 */
554 if (pkt != NULL) {
555 int dropcnt;
556
557 nx_netif_free_packet_chain(pkt, &dropcnt);
558
559 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
560 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
561 int, dropcnt);
562 }
563
564 if (dpkt_head != NULL) {
565 int dropcnt;
566
567 nx_netif_free_packet_chain(dpkt_head, &dropcnt);
568
569 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
570 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
571 int, dropcnt);
572 }
573
574 return err;
575 }
576
577 static int
netif_vp_send_pkt_chain_common(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,boolean_t compat)578 netif_vp_send_pkt_chain_common(struct nexus_netif_adapter *dev_nifna,
579 struct __kern_packet *pkt_chain, boolean_t compat)
580 {
581 struct __kern_packet *pkt = pkt_chain, *next, *p;
582 struct nx_netif *nif = dev_nifna->nifna_netif;
583 struct netif_stats *nifs = &nif->nif_stats;
584 ifnet_t ifp = nif->nif_ifp;
585 struct mbuf *m;
586 boolean_t drop;
587 int cnt = 0;
588 errno_t err;
589
590 while (pkt != NULL) {
591 next = pkt->pkt_nextpkt;
592 pkt->pkt_nextpkt = NULL;
593 drop = FALSE;
594
595 if (compat) {
596 m = nx_netif_pkt_to_mbuf(dev_nifna, pkt, NETIF_CONVERT_TX);
597 if (m == NULL) {
598 pkt = next;
599 continue;
600 }
601 err = ifnet_enqueue_mbuf(ifp, m, FALSE, &drop);
602 } else {
603 p = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX);
604 if (p == NULL) {
605 pkt = next;
606 continue;
607 }
608 err = ifnet_enqueue_pkt(ifp, ifp->if_snd, p, FALSE, &drop);
609 }
610 if (err != 0) {
611 SK_ERR("enqueue failed: %d", err);
612 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUE_FAILED);
613 if (drop) {
614 STATS_INC(nifs, NETIF_STATS_DROP);
615 }
616 DTRACE_SKYWALK2(enqueue__failed,
617 struct nexus_netif_adapter *, dev_nifna,
618 boolean_t, drop);
619 } else {
620 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUED);
621 cnt++;
622 }
623 pkt = next;
624 }
625 if (cnt > 0) {
626 ifnet_start(ifp);
627 }
628 return 0;
629 }
630
631 static int
netif_vp_send_pkt_chain(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,struct proc * p)632 netif_vp_send_pkt_chain(struct nexus_netif_adapter *dev_nifna,
633 struct __kern_packet *pkt_chain, struct proc *p)
634 {
635 struct nexus_adapter *na = &dev_nifna->nifna_up;
636
637 if (NETIF_IS_LOW_LATENCY(dev_nifna->nifna_netif)) {
638 return netif_vp_send_pkt_chain_low_latency(dev_nifna,
639 pkt_chain, p);
640 }
641 if (na->na_type == NA_NETIF_DEV) {
642 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, FALSE);
643 }
644 ASSERT(na->na_type == NA_NETIF_COMPAT_DEV);
645 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, TRUE);
646 }
647
648 SK_NO_INLINE_ATTRIBUTE
649 static boolean_t
validate_packet(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt)650 validate_packet(struct nexus_netif_adapter *nifna, struct __kern_packet *pkt)
651 {
652 struct nx_netif *nif = nifna->nifna_netif;
653
654 VERIFY(pkt->pkt_nextpkt == NULL);
655
656 if (nx_netif_vp_accept_all != 0) {
657 return TRUE;
658 }
659 if (outbound_check == 0 ||
660 (NETIF_IS_LOW_LATENCY(nif) &&
661 outbound_check != OUTBOUND_CHECK_FORCED)) {
662 return TRUE;
663 }
664 if (!nx_netif_validate_macaddr(nif, pkt, NETIF_FLOW_OUTBOUND)) {
665 return FALSE;
666 }
667 if (!nx_netif_flow_match(nif, pkt, nifna->nifna_flow,
668 NETIF_FLOW_OUTBOUND)) {
669 return FALSE;
670 }
671 return TRUE;
672 }
673
674 static int
netif_vp_na_txsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)675 netif_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
676 uint32_t flags)
677 {
678 #pragma unused(flags)
679 kern_channel_slot_t last_slot = NULL, slot = NULL;
680 struct __kern_packet *__single head = NULL;
681 struct __kern_packet **tailp = &head, *pkt;
682 struct nexus_netif_adapter *nifna, *dev_nifna;
683 struct nx_netif *nif;
684 struct netif_stats *nifs;
685 kern_packet_t ph;
686 errno_t err;
687 int cnt = 0;
688
689 nifna = NIFNA(KRNA(kring));
690 nif = nifna->nifna_netif;
691 nifs = &nif->nif_stats;
692 for (;;) {
693 slot = kern_channel_get_next_slot(kring, slot, NULL);
694 if (slot == NULL) {
695 break;
696 }
697 ph = kern_channel_slot_get_packet(kring, slot);
698 if (__improbable(ph == 0)) {
699 SK_ERR("packet got dropped by internalize");
700 STATS_INC(nifs, NETIF_STATS_VP_DROP_INTERNALIZE_FAIL);
701 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
702 kring, kern_channel_slot_t, slot);
703 last_slot = slot;
704 continue;
705 }
706 pkt = SK_PTR_ADDR_KPKT(ph);
707 if (__improbable(pkt->pkt_length == 0)) {
708 SK_ERR("dropped zero length packet");
709 STATS_INC(nifs, NETIF_STATS_VP_BAD_PKT_LEN);
710 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
711 kring, kern_channel_slot_t, slot);
712 last_slot = slot;
713 continue;
714 }
715 err = kern_channel_slot_detach_packet(kring, slot, ph);
716 VERIFY(err == 0);
717
718 /* packet needs to be finalized after detach */
719 err = kern_packet_finalize(ph);
720 VERIFY(err == 0);
721 last_slot = slot;
722
723 if (NA_CHANNEL_EVENT_ATTACHED(KRNA(kring))) {
724 __packet_set_tx_nx_port(SK_PKT2PH(pkt),
725 KRNA(kring)->na_nx_port, nifna->nifna_gencnt);
726 }
727
728 if (validate_packet(nifna, pkt)) {
729 nx_netif_snoop(nif, pkt, FALSE);
730 cnt++;
731 *tailp = pkt;
732 tailp = &pkt->pkt_nextpkt;
733 } else {
734 nx_netif_free_packet(pkt);
735 }
736 }
737 if (cnt == 0) {
738 STATS_INC(nifs, NETIF_STATS_VP_SYNC_NO_PKTS);
739 DTRACE_SKYWALK2(no__data, struct nexus_netif_adapter *, nifna,
740 struct __kern_channel_ring *, kring);
741 return 0;
742 }
743 DTRACE_SKYWALK4(injected, struct nexus_netif_adapter *, nifna,
744 struct __kern_channel_ring *, kring, struct __kern_packet *, head,
745 int, cnt);
746 if (last_slot != NULL) {
747 kern_channel_advance_slot(kring, last_slot);
748 }
749
750 dev_nifna = NIFNA(nx_port_get_na(KRNA(kring)->na_nx,
751 NEXUS_PORT_NET_IF_DEV));
752
753 err = netif_vp_send_pkt_chain(dev_nifna, head, p);
754 if (err != 0) {
755 SK_ERR("send failed: %d\n", err);
756 }
757 return 0;
758 }
759
760 static int
netif_vp_na_rxsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)761 netif_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
762 uint32_t flags)
763 {
764 #pragma unused(p, flags)
765 (void) kr_reclaim(kring);
766 return 0;
767 }
768
769 static int
netif_vp_na_krings_create(struct nexus_adapter * na,struct kern_channel * ch)770 netif_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
771 {
772 ASSERT(na->na_type == NA_NETIF_VP);
773 return na_rings_mem_setup(na, FALSE, ch);
774 }
775
776
777 /* na_krings_delete callback for flow switch ports. */
778 static void
netif_vp_na_krings_delete(struct nexus_adapter * na,struct kern_channel * ch,boolean_t defunct)779 netif_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
780 boolean_t defunct)
781 {
782 ASSERT(na->na_type == NA_NETIF_VP);
783 na_rings_mem_teardown(na, ch, defunct);
784 }
785
786 static int
netif_vp_region_params_setup(struct nexus_adapter * na,struct skmem_region_params srp[SKMEM_REGIONS],struct kern_pbufpool ** tx_pp)787 netif_vp_region_params_setup(struct nexus_adapter *na,
788 struct skmem_region_params srp[SKMEM_REGIONS], struct kern_pbufpool **tx_pp)
789 {
790 #pragma unused (tx_pp)
791 uint32_t max_mtu;
792 uint32_t buf_sz, buf_cnt, nslots, afslots, evslots, totalrings;
793 struct nexus_adapter *devna;
794 struct kern_nexus *nx;
795 struct nx_netif *__single nif;
796 int err, i;
797
798 for (i = 0; i < SKMEM_REGIONS; i++) {
799 srp[i] = *skmem_get_default(i);
800 }
801 totalrings = na_get_nrings(na, NR_TX) + na_get_nrings(na, NR_RX) +
802 na_get_nrings(na, NR_A) + na_get_nrings(na, NR_F) +
803 na_get_nrings(na, NR_EV);
804
805 srp[SKMEM_REGION_SCHEMA].srp_r_obj_size =
806 (uint32_t)CHANNEL_SCHEMA_SIZE(totalrings);
807 srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = totalrings;
808 skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]);
809
810 srp[SKMEM_REGION_RING].srp_r_obj_size =
811 sizeof(struct __user_channel_ring);
812 srp[SKMEM_REGION_RING].srp_r_obj_cnt = totalrings;
813 skmem_region_params_config(&srp[SKMEM_REGION_RING]);
814
815 /* USD regions need to be writable to support user packet pool */
816 srp[SKMEM_REGION_TXAUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
817 srp[SKMEM_REGION_RXFUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
818
819 nslots = na_get_nslots(na, NR_TX);
820 afslots = na_get_nslots(na, NR_A);
821 evslots = na_get_nslots(na, NR_EV);
822 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size =
823 MAX(MAX(nslots, afslots), evslots) * SLOT_DESC_SZ;
824 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt =
825 na_get_nrings(na, NR_TX) + na_get_nrings(na, NR_A) +
826 na_get_nrings(na, NR_EV);
827 skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]);
828
829 /* USD and KSD objects share the same size and count */
830 srp[SKMEM_REGION_TXAUSD].srp_r_obj_size =
831 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size;
832 srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt =
833 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt;
834 skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]);
835
836 /*
837 * Since the rx/free slots share the same region and cache,
838 * we will use the same object size for both types of slots.
839 */
840 nslots = na_get_nslots(na, NR_RX);
841 afslots = na_get_nslots(na, NR_F);
842 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size =
843 MAX(nslots, afslots) * SLOT_DESC_SZ;
844 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt =
845 na_get_nrings(na, NR_RX) + na_get_nrings(na, NR_F);
846 skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]);
847
848 /* USD and KSD objects share the same size and count */
849 srp[SKMEM_REGION_RXFUSD].srp_r_obj_size =
850 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size;
851 srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt =
852 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt;
853 skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]);
854
855 /*
856 * No need to create our own buffer pool if we can share the device's
857 * pool. We don't support sharing split pools to user space.
858 */
859 nx = na->na_nx;
860 nif = nx->nx_arg;
861 if (vp_zerocopy != 0 && NETIF_IS_LOW_LATENCY(nif) &&
862 nx->nx_tx_pp != NULL && (nx->nx_rx_pp == NULL ||
863 nx->nx_tx_pp == nx->nx_rx_pp) && !PP_KERNEL_ONLY(nx->nx_tx_pp)) {
864 struct kern_pbufpool *pp = nx->nx_tx_pp;
865
866 if (nif->nif_hw_ch_refcnt != 0) {
867 SK_ERR("only one channel is supported for zero copy");
868 return ENOTSUP;
869 }
870 SK_DF(SK_VERB_VP, "sharing %s's pool", if_name(na->na_ifp));
871
872 /*
873 * These types need to be initialized otherwise some assertions
874 * skmem_arena_create_for_nexus() will fail.
875 */
876 srp[SKMEM_REGION_UMD].srp_md_type = pp->pp_md_type;
877 srp[SKMEM_REGION_UMD].srp_md_subtype = pp->pp_md_subtype;
878 srp[SKMEM_REGION_KMD].srp_md_type = pp->pp_md_type;
879 srp[SKMEM_REGION_KMD].srp_md_subtype = pp->pp_md_subtype;
880 *tx_pp = nx->nx_tx_pp;
881 return 0;
882 }
883
884 devna = nx_port_get_na(nx, NEXUS_PORT_NET_IF_DEV);
885 ASSERT(devna != NULL);
886 if (devna->na_type == NA_NETIF_DEV) {
887 /*
888 * For native devices, use the driver's buffer size
889 */
890 ASSERT(nx->nx_rx_pp != NULL);
891 ASSERT(nx->nx_tx_pp != NULL);
892 buf_sz = PP_BUF_SIZE_DEF(nx->nx_tx_pp);
893 } else {
894 if ((err = nx_netif_get_max_mtu(na->na_ifp, &max_mtu)) != 0) {
895 /*
896 * If the driver doesn't support SIOCGIFDEVMTU, use the
897 * default MTU size.
898 */
899 max_mtu = ifnet_mtu(na->na_ifp);
900 err = 0;
901 }
902 /* max_mtu does not include the L2 header */
903 buf_sz = MAX(max_mtu + sizeof(struct ether_vlan_header), 2048);
904 }
905 buf_cnt = vp_pool_size;
906 pp_regions_params_adjust(srp, NEXUS_META_TYPE_PACKET,
907 NEXUS_META_SUBTYPE_RAW, buf_cnt, 1, buf_sz, 0, buf_cnt, 0,
908 PP_REGION_CONFIG_BUF_IODIR_BIDIR |
909 PP_REGION_CONFIG_MD_MAGAZINE_ENABLE);
910
911 nx_netif_vp_region_params_adjust(na, srp);
912 return 0;
913 }
914
915 static int
netif_vp_na_mem_new(struct kern_nexus * nx,struct nexus_adapter * na)916 netif_vp_na_mem_new(struct kern_nexus *nx, struct nexus_adapter *na)
917 {
918 #pragma unused(nx)
919 struct skmem_region_params srp[SKMEM_REGIONS];
920 struct kern_pbufpool *__single tx_pp = NULL;
921 int err;
922
923 err = netif_vp_region_params_setup(na, srp, &tx_pp);
924 if (err != 0) {
925 return err;
926 }
927 na->na_arena = skmem_arena_create_for_nexus(na, srp,
928 tx_pp != NULL ? &tx_pp : NULL, NULL,
929 FALSE, FALSE, &nx->nx_adv, &err);
930 ASSERT(na->na_arena != NULL || err != 0);
931 return err;
932 }
933
934 static void
netif_vp_na_dtor(struct nexus_adapter * na)935 netif_vp_na_dtor(struct nexus_adapter *na)
936 {
937 struct kern_nexus *nx = na->na_nx;
938 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
939 struct nexus_netif_adapter *nifna = NIFNA(na);
940
941 NETIF_WLOCK(nif);
942 (void) nx_port_unbind(nx, na->na_nx_port);
943 nx_port_free(nx, na->na_nx_port);
944 nif->nif_vp_cnt--;
945 if (na->na_ifp != NULL) {
946 ifnet_decr_iorefcnt(na->na_ifp);
947 na->na_ifp = NULL;
948 }
949 if (nifna->nifna_netif != NULL) {
950 nx_netif_release(nifna->nifna_netif);
951 nifna->nifna_netif = NULL;
952 }
953 NETIF_WUNLOCK(nif);
954 SK_DF(SK_VERB_VP, "na \"%s\" (%p)", na->na_name, SK_KVA(na));
955 }
956
957 int
netif_vp_na_create(struct kern_nexus * nx,struct chreq * chr,struct nexus_adapter ** nap)958 netif_vp_na_create(struct kern_nexus *nx, struct chreq *chr,
959 struct nexus_adapter **nap)
960 {
961 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
962 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
963 struct nexus_adapter *na = NULL;
964 struct nexus_netif_adapter *nifna;
965 uint32_t slots;
966 int err;
967
968 NETIF_WLOCK_ASSERT_HELD(nif);
969 if (nif->nif_ifp == NULL) {
970 SK_ERR("ifnet not yet attached");
971 return ENXIO;
972 }
973 ASSERT((chr->cr_mode & CHMODE_KERNEL) == 0);
974 if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) {
975 SK_ERR("user packet pool required");
976 return EINVAL;
977 }
978 /*
979 * No locking needed while checking for the initialized bit because
980 * if this were not set, no other codepaths would modify the flags.
981 */
982 if ((nif->nif_flow_flags & NETIF_FLOW_FLAG_INITIALIZED) == 0) {
983 SK_ERR("demux vp not supported");
984 return ENOTSUP;
985 }
986 na = (struct nexus_adapter *)na_netif_alloc(Z_WAITOK);
987 nifna = NIFNA(na);
988 nifna->nifna_netif = nif;
989 nx_netif_retain(nif);
990 nifna->nifna_flow = NULL;
991
992 (void) snprintf(na->na_name, sizeof(na->na_name),
993 "netif_vp:%d", chr->cr_port);
994 uuid_generate_random(na->na_uuid);
995
996 na_set_nrings(na, NR_TX, nxp->nxp_tx_rings);
997 na_set_nrings(na, NR_RX, nxp->nxp_rx_rings);
998 /*
999 * If the packet pool is configured to be multi-buflet, then we
1000 * need 2 pairs of alloc/free rings(for packet and buflet).
1001 */
1002 na_set_nrings(na, NR_A, ((nxp->nxp_max_frags > 1) &&
1003 (sk_channel_buflet_alloc != 0)) ? 2 : 1);
1004
1005 slots = vp_tx_slots != 0 ? vp_tx_slots :
1006 NX_DOM(nx)->nxdom_tx_slots.nb_def;
1007 na_set_nslots(na, NR_TX, slots);
1008
1009 slots = vp_rx_slots != 0 ? vp_rx_slots :
1010 NX_DOM(nx)->nxdom_rx_slots.nb_def;
1011 na_set_nslots(na, NR_RX, slots);
1012
1013 na_set_nslots(na, NR_A, NETIF_DEMUX_ALLOC_SLOTS);
1014 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
1015 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
1016 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
1017 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
1018
1019 os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed);
1020
1021 if (chr->cr_mode & CHMODE_EVENT_RING) {
1022 na_set_nrings(na, NR_EV, NX_NETIF_EVENT_RING_NUM);
1023 na_set_nslots(na, NR_EV, NX_NETIF_EVENT_RING_SIZE);
1024 os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed);
1025 na->na_channel_event_notify = netif_vp_na_channel_event_notify;
1026 }
1027
1028 na->na_nx_port = chr->cr_port;
1029 na->na_type = NA_NETIF_VP;
1030 na->na_free = na_netif_free;
1031 na->na_dtor = netif_vp_na_dtor;
1032 na->na_activate = netif_vp_na_activate;
1033 na->na_txsync = netif_vp_na_txsync;
1034 na->na_rxsync = netif_vp_na_rxsync;
1035 na->na_krings_create = netif_vp_na_krings_create;
1036 na->na_krings_delete = netif_vp_na_krings_delete;
1037 na->na_special = NULL;
1038 na->na_ifp = nif->nif_ifp;
1039 ifnet_incr_iorefcnt(na->na_ifp);
1040
1041 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
1042 NEXUS_STATS_TYPE_INVALID;
1043
1044 /* other fields are set in the common routine */
1045 na_attach_common(na, nx, &nx_netif_prov_s);
1046
1047 err = netif_vp_na_mem_new(nx, na);
1048 if (err != 0) {
1049 ASSERT(na->na_arena == NULL);
1050 goto err;
1051 }
1052
1053 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
1054 ASSERT(na->na_flowadv_max == 0 ||
1055 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
1056
1057 nif->nif_vp_cnt++;
1058 *nap = na;
1059 return 0;
1060
1061 err:
1062 if (na != NULL) {
1063 if (na->na_ifp != NULL) {
1064 ifnet_decr_iorefcnt(na->na_ifp);
1065 na->na_ifp = NULL;
1066 }
1067 if (na->na_arena != NULL) {
1068 skmem_arena_release(na->na_arena);
1069 na->na_arena = NULL;
1070 }
1071 if (nifna->nifna_netif != NULL) {
1072 nx_netif_release(nifna->nifna_netif);
1073 nifna->nifna_netif = NULL;
1074 }
1075 NA_FREE(na);
1076 }
1077 SK_ERR("VP NA creation failed, err(%d)", err);
1078 return err;
1079 }
1080
1081 static int
netif_vp_na_channel_event_notify(struct nexus_adapter * vpna,struct __kern_channel_event * ev,uint16_t ev_len)1082 netif_vp_na_channel_event_notify(struct nexus_adapter *vpna,
1083 struct __kern_channel_event *ev, uint16_t ev_len)
1084 {
1085 int err;
1086 char *baddr;
1087 kern_packet_t ph;
1088 kern_buflet_t buf;
1089 sk_protect_t protect;
1090 kern_channel_slot_t slot;
1091 struct __kern_packet *vpna_pkt = NULL;
1092 struct __kern_channel_event_metadata *emd;
1093 struct __kern_channel_ring *ring = &vpna->na_event_rings[0];
1094 struct netif_stats *nifs = &NIFNA(vpna)->nifna_netif->nif_stats;
1095
1096 if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) {
1097 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_STATUS);
1098 }
1099 if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) {
1100 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_EXPIRED);
1101 }
1102 STATS_INC(nifs, NETIF_STATS_EV_RECV);
1103
1104 if (__improbable(!NA_IS_ACTIVE(vpna))) {
1105 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_INACTIVE);
1106 err = ENXIO;
1107 goto error;
1108 }
1109 if (__improbable(NA_IS_DEFUNCT(vpna))) {
1110 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_DEFUNCT);
1111 err = ENXIO;
1112 goto error;
1113 }
1114 if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) {
1115 STATS_INC(nifs, NETIF_STATS_EV_DROP_KEVENT_INACTIVE);
1116 err = ENXIO;
1117 goto error;
1118 }
1119 if (__improbable(KR_DROP(ring))) {
1120 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRDROP_MODE);
1121 err = ENXIO;
1122 goto error;
1123 }
1124 vpna_pkt = nx_netif_alloc_packet(ring->ckr_pp, ev_len, &ph);
1125 if (__improbable(vpna_pkt == NULL)) {
1126 STATS_INC(nifs, NETIF_STATS_EV_DROP_NOMEM_PKT);
1127 err = ENOMEM;
1128 goto error;
1129 }
1130 buf = __packet_get_next_buflet(ph, NULL);
1131 baddr = __buflet_get_data_address(buf);
1132 emd = (struct __kern_channel_event_metadata *)(void *)baddr;
1133 emd->emd_etype = ev->ev_type;
1134 emd->emd_nevents = 1;
1135 bcopy(ev, (baddr + __KERN_CHANNEL_EVENT_OFFSET),
1136 ev->ev_dlen + sizeof(struct __kern_channel_event));
1137 err = __buflet_set_data_length(buf,
1138 (ev_len + __KERN_CHANNEL_EVENT_OFFSET));
1139 VERIFY(err == 0);
1140 err = __packet_finalize(ph);
1141 VERIFY(err == 0);
1142 kr_enter(ring, TRUE);
1143 protect = sk_sync_protect();
1144 slot = kern_channel_get_next_slot(ring, NULL, NULL);
1145 if (slot == NULL) {
1146 sk_sync_unprotect(protect);
1147 kr_exit(ring);
1148 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRSPACE);
1149 err = ENOBUFS;
1150 goto error;
1151 }
1152 err = kern_channel_slot_attach_packet(ring, slot, ph);
1153 VERIFY(err == 0);
1154 vpna_pkt = NULL;
1155 kern_channel_advance_slot(ring, slot);
1156 sk_sync_unprotect(protect);
1157 kr_exit(ring);
1158 kern_channel_event_notify(&vpna->na_tx_rings[0]);
1159 STATS_INC(nifs, NETIF_STATS_EV_SENT);
1160 return 0;
1161
1162 error:
1163 ASSERT(err != 0);
1164 if (vpna_pkt != NULL) {
1165 nx_netif_free_packet(vpna_pkt);
1166 }
1167 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1168 return err;
1169 }
1170
1171 static inline struct nexus_adapter *
nx_netif_find_port_vpna(struct nx_netif * netif,uint32_t nx_port_id)1172 nx_netif_find_port_vpna(struct nx_netif *netif, uint32_t nx_port_id)
1173 {
1174 struct kern_nexus *nx = netif->nif_nx;
1175 struct nexus_adapter *na = NULL;
1176 nexus_port_t port;
1177 uint16_t gencnt;
1178
1179 PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt);
1180 if (port < NEXUS_PORT_NET_IF_CLIENT) {
1181 SK_ERR("non VPNA port");
1182 return NULL;
1183 }
1184 if (__improbable(!nx_port_is_valid(nx, port))) {
1185 SK_ERR("%s[%d] port no longer valid",
1186 if_name(netif->nif_ifp), port);
1187 return NULL;
1188 }
1189 na = nx_port_get_na(nx, port);
1190 if (na != NULL && NIFNA(na)->nifna_gencnt != gencnt) {
1191 return NULL;
1192 }
1193 return na;
1194 }
1195
1196 errno_t
netif_vp_na_channel_event(struct nx_netif * nif,uint32_t nx_port_id,struct __kern_channel_event * event,uint16_t event_len)1197 netif_vp_na_channel_event(struct nx_netif *nif, uint32_t nx_port_id,
1198 struct __kern_channel_event *event, uint16_t event_len)
1199 {
1200 int err = 0;
1201 struct nexus_adapter *netif_vpna;
1202 struct netif_stats *nifs = &nif->nif_stats;
1203
1204 SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu "
1205 "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu",
1206 if_name(nif->nif_ifp), nx_port_id, event, event_len,
1207 event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen);
1208
1209 NETIF_RLOCK(nif);
1210 if (!NETIF_IS_LOW_LATENCY(nif)) {
1211 err = ENOTSUP;
1212 goto error;
1213 }
1214 if (__improbable(nif->nif_vp_cnt == 0)) {
1215 STATS_INC(nifs, NETIF_STATS_EV_DROP_NO_VPNA);
1216 err = ENXIO;
1217 goto error;
1218 }
1219 netif_vpna = nx_netif_find_port_vpna(nif, nx_port_id);
1220 if (__improbable(netif_vpna == NULL)) {
1221 err = ENXIO;
1222 STATS_INC(nifs, NETIF_STATS_EV_DROP_DEMUX_ERR);
1223 goto error;
1224 }
1225 if (__improbable(netif_vpna->na_channel_event_notify == NULL)) {
1226 err = ENOTSUP;
1227 STATS_INC(nifs, NETIF_STATS_EV_DROP_EV_VPNA_NOTSUP);
1228 goto error;
1229 }
1230 err = netif_vpna->na_channel_event_notify(netif_vpna, event, event_len);
1231 NETIF_RUNLOCK(nif);
1232 return err;
1233
1234 error:
1235 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1236 NETIF_RUNLOCK(nif);
1237 return err;
1238 }
1239
1240 /* XXX Remove once rdar://118519573 is resolved */
1241 #pragma clang diagnostic pop
1242