1 /*
2 * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <skywalk/nexus/netif/nx_netif.h>
31 #include <net/if_vlan_var.h>
32 #include <sys/sdt.h>
33
34 #define NETIF_DEMUX_ALLOC_SLOTS 128
35
36 #define OUTBOUND_CHECK_OFF 0
37 #define OUTBOUND_CHECK_ON 1
38 #define OUTBOUND_CHECK_FORCED 2
39
40 /* Turning this off allows packets to be spoofed for testing purposes */
41 static uint32_t outbound_check = OUTBOUND_CHECK_ON;
42
43 /* This controls the per-NA pool size of custom ether and llw NAs */
44 static uint32_t vp_pool_size = 2048;
45
46 /* This enables zerocopy on llw NAs */
47 static uint32_t vp_zerocopy = 0;
48
49 /* TX Ring size */
50 static uint32_t vp_tx_slots = 0;
51
52 /* RX Ring size */
53 static uint32_t vp_rx_slots = 0;
54
55 /*
56 * Disable all packet validation
57 */
58 uint32_t nx_netif_vp_accept_all = 0;
59
60 static uint16_t nx_netif_vpna_gencnt = 0;
61
62 #if (DEVELOPMENT || DEBUG)
63 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, outbound_check,
64 CTLFLAG_RW | CTLFLAG_LOCKED, &outbound_check, 0,
65 "netif outbound packet validation");
66 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_pool_size,
67 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_pool_size, 0,
68 "netif virtual port pool size");
69 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_zerocopy,
70 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_zerocopy, 0,
71 "netif virtual port zero copy");
72 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_tx_slots,
73 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_tx_slots, 0,
74 "netif virtual port tx slots");
75 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_rx_slots,
76 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_rx_slots, 0,
77 "netif virtual port rx slots");
78 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_accept_all,
79 CTLFLAG_RW | CTLFLAG_LOCKED, &nx_netif_vp_accept_all, 0,
80 "netif accept all");
81 #endif /* (DEVELOPMENT || DEBUG) */
82
83 /* XXX Until rdar://118519573 is resolved, do this as a workaround */
84 #pragma clang diagnostic push
85 #pragma clang diagnostic ignored "-Wcast-align"
86
87 static int
88 netif_vp_na_channel_event_notify(struct nexus_adapter *,
89 struct __kern_channel_event *, uint16_t);
90
91 static void
netif_vp_dump_packet(struct __kern_packet * pkt)92 netif_vp_dump_packet(struct __kern_packet *pkt)
93 {
94 uint8_t *baddr;
95
96 MD_BUFLET_ADDR_ABS(pkt, baddr);
97 ASSERT(baddr != NULL);
98 baddr += pkt->pkt_headroom;
99
100 DTRACE_SKYWALK2(dump__packet, struct __kern_packet *,
101 pkt, uint8_t *, baddr);
102 }
103
104 static int
netif_copy_or_attach_pkt(struct __kern_channel_ring * ring,kern_channel_slot_t slot,struct __kern_packet * pkt)105 netif_copy_or_attach_pkt(struct __kern_channel_ring *ring,
106 kern_channel_slot_t slot, struct __kern_packet *pkt)
107 {
108 kern_packet_t ph;
109 struct __kern_packet *dpkt;
110 errno_t err;
111
112 if (pkt->pkt_qum.qum_pp == ring->ckr_pp) {
113 DTRACE_SKYWALK2(attach__pkt, struct __kern_channel_ring *, ring,
114 struct __kern_packet *, pkt);
115 ph = SK_PKT2PH(pkt);
116 err = kern_packet_finalize(ph);
117 VERIFY(err == 0);
118 } else {
119 DTRACE_SKYWALK2(copy__pkt, struct __kern_channel_ring *, ring,
120 struct __kern_packet *, pkt);
121 dpkt = nx_netif_pkt_to_pkt(NIFNA(ring->ckr_na), pkt,
122 ring->ckr_na->na_type == NA_NETIF_VP ? NETIF_CONVERT_RX :
123 NETIF_CONVERT_TX);
124 if (__improbable(dpkt == NULL)) {
125 return ENOMEM;
126 }
127 ph = SK_PKT2PH(dpkt);
128 }
129 err = kern_channel_slot_attach_packet(ring, slot, ph);
130 VERIFY(err == 0);
131 return 0;
132 }
133
134 static errno_t
netif_deliver_pkt(struct nexus_adapter * na,struct __kern_packet * pkt_chain,uint32_t flags)135 netif_deliver_pkt(struct nexus_adapter *na, struct __kern_packet *pkt_chain,
136 uint32_t flags)
137 {
138 #pragma unused(flags)
139 struct __kern_channel_ring *ring = &na->na_rx_rings[0];
140 struct __kern_packet *pkt = pkt_chain, *next;
141 kern_channel_slot_t last_slot = NULL, slot = NULL;
142 struct nexus_netif_adapter *nifna = NIFNA(na);
143 struct nx_netif *nif = nifna->nifna_netif;
144 struct netif_stats *nifs = &nif->nif_stats;
145 sk_protect_t protect;
146 int cnt = 0, dropcnt = 0, err;
147
148 (void) kr_enter(ring, TRUE);
149 protect = sk_sync_protect();
150
151 if (__improbable(KR_DROP(ring))) {
152 nx_netif_free_packet_chain(pkt, &dropcnt);
153 STATS_ADD(nifs,
154 NETIF_STATS_VP_DROP_USER_RING_DISABLED, dropcnt);
155 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
156 DTRACE_SKYWALK2(ring__drop, struct __kern_channel_ring *, ring,
157 int, dropcnt);
158 sk_sync_unprotect(protect);
159 kr_exit(ring);
160 return ENXIO;
161 }
162 while (pkt != NULL) {
163 slot = kern_channel_get_next_slot(ring, last_slot, NULL);
164 if (slot == NULL) {
165 break;
166 }
167 next = pkt->pkt_nextpkt;
168 pkt->pkt_nextpkt = NULL;
169 netif_vp_dump_packet(pkt);
170 err = netif_copy_or_attach_pkt(ring, slot, pkt);
171 if (__probable(err == 0)) {
172 last_slot = slot;
173 }
174 pkt = next;
175 cnt++;
176 }
177 if (NETIF_IS_LOW_LATENCY(nif)) {
178 STATS_ADD(nifs, NETIF_STATS_VP_LL_DELIVERED, cnt);
179 } else {
180 STATS_ADD(nifs, NETIF_STATS_VP_DELIVERED, cnt);
181 }
182 DTRACE_SKYWALK4(delivered, struct nexus_adapter *, na,
183 struct __kern_channel_ring *, ring, struct __kern_packet *, pkt,
184 int, cnt);
185
186 if (pkt != NULL) {
187 nx_netif_free_packet_chain(pkt, &dropcnt);
188 STATS_ADD(nifs,
189 NETIF_STATS_VP_DROP_USER_RING_NO_SPACE, dropcnt);
190 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
191 DTRACE_SKYWALK2(deliver__drop, struct nexus_adapter *, na,
192 int, dropcnt);
193 }
194 if (last_slot != NULL) {
195 kern_channel_advance_slot(ring, last_slot);
196 }
197 sk_sync_unprotect(protect);
198 kr_exit(ring);
199 if (cnt > 0) {
200 (void) kern_channel_notify(ring, 0);
201 }
202 return 0;
203 }
204
205 static errno_t
netif_deliver_cb(void * arg,void * chain,uint32_t flags)206 netif_deliver_cb(void *arg, void *chain, uint32_t flags)
207 {
208 return netif_deliver_pkt(arg, chain, flags);
209 }
210
211 static int
netif_hwna_rx_get_pkts(struct __kern_channel_ring * ring,struct proc * p,uint32_t flags,struct __kern_packet ** chain)212 netif_hwna_rx_get_pkts(struct __kern_channel_ring *ring, struct proc *p,
213 uint32_t flags, struct __kern_packet **chain)
214 {
215 int err, cnt = 0;
216 sk_protect_t protect;
217 slot_idx_t ktail, idx;
218 struct __kern_packet *__single pkt_chain = NULL;
219 struct __kern_packet **tailp = &pkt_chain;
220 struct netif_stats *nifs = &NIFNA(KRNA(ring))->nifna_netif->nif_stats;
221
222 err = kr_enter(ring, ((flags & NA_NOTEF_CAN_SLEEP) != 0 ||
223 (ring->ckr_flags & CKRF_HOST) != 0));
224 if (err != 0) {
225 SK_DF(SK_VERB_VP,
226 "hwna \"%s\" (0x%llx) kr \"%s\" (0x%llx) krflags 0x%b "
227 "(%d)", KRNA(ring)->na_name, SK_KVA(KRNA(ring)),
228 ring->ckr_name, SK_KVA(ring), ring->ckr_flags,
229 CKRF_BITS, err);
230 STATS_INC(nifs, NETIF_STATS_VP_KR_ENTER_FAIL);
231 return err;
232 }
233 if (__improbable(KR_DROP(ring))) {
234 kr_exit(ring);
235 STATS_INC(nifs, NETIF_STATS_VP_DEV_RING_DISABLED);
236 return ENODEV;
237 }
238 protect = sk_sync_protect();
239
240 err = ring->ckr_na_sync(ring, p, 0);
241 if (err != 0 && err != EAGAIN) {
242 STATS_INC(nifs, NETIF_STATS_VP_SYNC_UNKNOWN_ERR);
243 goto out;
244 }
245 ktail = ring->ckr_ktail;
246 if (__improbable(ring->ckr_khead == ktail)) {
247 SK_DF(SK_VERB_VP,
248 "spurious wakeup on hwna %s (0x%llx)", KRNA(ring)->na_name,
249 SK_KVA(KRNA(ring)));
250 STATS_INC(nifs, NETIF_STATS_VP_SPURIOUS_NOTIFY);
251 err = ENOENT;
252 goto out;
253 }
254 /* get all packets from the ring */
255 idx = ring->ckr_rhead;
256 while (idx != ktail) {
257 struct __kern_slot_desc *ksd = KR_KSD(ring, idx);
258 struct __kern_packet *pkt = ksd->sd_pkt;
259
260 ASSERT(pkt->pkt_nextpkt == NULL);
261 KR_SLOT_DETACH_METADATA(ring, ksd);
262 cnt++;
263 *tailp = pkt;
264 tailp = &pkt->pkt_nextpkt;
265 idx = SLOT_NEXT(idx, ring->ckr_lim);
266 }
267 ring->ckr_rhead = ktail;
268 ring->ckr_rtail = ring->ckr_ktail;
269
270 DTRACE_SKYWALK2(rx__notify, struct __kern_channel_ring *, ring,
271 int, cnt);
272 *chain = pkt_chain;
273 out:
274 sk_sync_unprotect(protect);
275 kr_exit(ring);
276 return err;
277 }
278
279 int
netif_llw_rx_notify_fast(struct __kern_channel_ring * ring,struct proc * p,uint32_t flags)280 netif_llw_rx_notify_fast(struct __kern_channel_ring *ring, struct proc *p,
281 uint32_t flags)
282 {
283 #pragma unused (p, flags)
284 struct nexus_adapter *hwna;
285 uint32_t count;
286 int i, err;
287
288 hwna = KRNA(ring);
289 count = na_get_nslots(hwna, NR_RX);
290 err = nx_rx_sync_packets(ring, ring->ckr_scratch, &count);
291 if (__improbable(err != 0)) {
292 SK_ERR("nx_rx_sync_packets failed: %d", err);
293 DTRACE_SKYWALK2(rx__sync__packets__failed,
294 struct __kern_channel_ring *, ring, int, err);
295 return err;
296 }
297 DTRACE_SKYWALK1(chain__count, uint32_t, count);
298 for (i = 0; i < count; i++) {
299 struct __kern_packet *pkt_chain;
300
301 pkt_chain = SK_PTR_ADDR_KPKT(ring->ckr_scratch[i]);
302 ASSERT(pkt_chain != NULL);
303 (void) nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL,
304 NULL, NETIF_FLOW_SOURCE);
305 }
306 return 0;
307 }
308
309 int
netif_llw_rx_notify_default(struct __kern_channel_ring * ring,struct proc * p,uint32_t flags)310 netif_llw_rx_notify_default(struct __kern_channel_ring *ring, struct proc *p,
311 uint32_t flags)
312 {
313 int err;
314 struct __kern_packet *__single pkt_chain = NULL;
315
316 err = netif_hwna_rx_get_pkts(ring, p, flags, &pkt_chain);
317 if (err != 0) {
318 return err;
319 }
320 return nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL,
321 NULL, NETIF_FLOW_SOURCE);
322 }
323
324 static errno_t
netif_hwna_setup(struct nx_netif * nif)325 netif_hwna_setup(struct nx_netif *nif)
326 {
327 struct kern_channel *ch;
328 struct kern_nexus *nx = nif->nif_nx;
329 struct chreq chr;
330 int err;
331
332 SK_LOCK_ASSERT_HELD();
333 ASSERT(NETIF_IS_LOW_LATENCY(nif));
334 if (nif->nif_hw_ch != NULL) {
335 nif->nif_hw_ch_refcnt++;
336 SK_DF(SK_VERB_VP, "%s: hw channel already open, refcnt %d",
337 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
338 return 0;
339 }
340 ASSERT(nif->nif_hw_ch_refcnt == 0);
341 bzero(&chr, sizeof(chr));
342 uuid_copy(chr.cr_spec_uuid, nx->nx_uuid);
343 chr.cr_ring_id = 0;
344 chr.cr_port = NEXUS_PORT_NET_IF_DEV;
345 chr.cr_mode |= CHMODE_CONFIG;
346
347 err = 0;
348 ch = ch_open_special(nx, &chr, FALSE, &err);
349 if (ch == NULL) {
350 SK_ERR("%s: failed to open nx 0x%llx (err %d)",
351 if_name(nif->nif_ifp), SK_KVA(nx), err);
352 return err;
353 }
354 netif_hwna_set_mode(ch->ch_na, NETIF_MODE_LLW, NULL);
355 na_start_spec(nx, ch);
356 nif->nif_hw_ch_refcnt = 1;
357 nif->nif_hw_ch = ch;
358 SK_DF(SK_VERB_VP, "%s: hw channel opened 0x%llx, %s:%s",
359 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
360 NX_DOM_PROV(nx)->nxdom_prov_name);
361 return 0;
362 }
363
364 static void
netif_hwna_teardown(struct nx_netif * nif)365 netif_hwna_teardown(struct nx_netif *nif)
366 {
367 struct kern_nexus *nx = nif->nif_nx;
368 struct kern_channel *ch = nif->nif_hw_ch;
369
370 SK_LOCK_ASSERT_HELD();
371 ASSERT(NETIF_IS_LOW_LATENCY(nif));
372 ASSERT(ch != NULL);
373 if (--nif->nif_hw_ch_refcnt > 0) {
374 SK_DF(SK_VERB_VP, "%s: hw channel still open, refcnt %d",
375 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
376 return;
377 }
378 SK_DF(SK_VERB_VP, "%s: hw channel closing 0x%llx, %s:%s",
379 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
380 NX_DOM_PROV(nx)->nxdom_prov_name);
381
382 na_stop_spec(nx, ch);
383 netif_hwna_clear_mode(ch->ch_na);
384 ch_close_special(ch);
385 (void) ch_release_locked(ch);
386 nif->nif_hw_ch = NULL;
387 SK_DF(SK_VERB_VP, "%s: hw channel closed, %s:%s",
388 if_name(nif->nif_ifp), NX_DOM(nx)->nxdom_name,
389 NX_DOM_PROV(nx)->nxdom_prov_name);
390 }
391
392 static int
netif_vp_na_activate_on(struct nexus_adapter * na)393 netif_vp_na_activate_on(struct nexus_adapter *na)
394 {
395 errno_t err;
396 struct netif_flow *__single nf;
397 struct netif_port_info npi;
398 struct nexus_netif_adapter *nifna;
399 struct nx_netif *nif;
400 boolean_t hwna_setup = FALSE;
401
402 nifna = NIFNA(na);
403 nif = nifna->nifna_netif;
404
405 /* lock needed to protect against nxdom_unbind_port */
406 NETIF_WLOCK(nif);
407 err = nx_port_get_info(nif->nif_nx, na->na_nx_port,
408 NX_PORT_INFO_TYPE_NETIF, &npi, sizeof(npi));
409 NETIF_WUNLOCK(nif);
410 if (err != 0) {
411 SK_ERR("port info not found: %d", err);
412 return err;
413 }
414 if (NETIF_IS_LOW_LATENCY(nif)) {
415 err = netif_hwna_setup(nif);
416 if (err != 0) {
417 return err;
418 }
419 hwna_setup = TRUE;
420 }
421 err = nx_netif_flow_add(nif, na->na_nx_port, &npi.npi_fd, na,
422 netif_deliver_cb, &nf);
423 if (err != 0) {
424 if (hwna_setup) {
425 netif_hwna_teardown(nif);
426 }
427 return err;
428 }
429 nifna->nifna_flow = nf;
430 os_atomic_inc(&nx_netif_vpna_gencnt, relaxed);
431 nifna->nifna_gencnt = nx_netif_vpna_gencnt;
432 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
433 return 0;
434 }
435
436 static int
netif_vp_na_activate_off(struct nexus_adapter * na)437 netif_vp_na_activate_off(struct nexus_adapter *na)
438 {
439 errno_t err;
440 struct nexus_netif_adapter *nifna;
441 struct nx_netif *nif;
442
443 if (!NA_IS_ACTIVE(na)) {
444 DTRACE_SKYWALK1(already__off, struct nexus_adapter *, na);
445 return 0;
446 }
447 nifna = NIFNA(na);
448 nif = nifna->nifna_netif;
449 err = nx_netif_flow_remove(nif, nifna->nifna_flow);
450 VERIFY(err == 0);
451
452 nifna->nifna_flow = NULL;
453 if (NETIF_IS_LOW_LATENCY(nif)) {
454 netif_hwna_teardown(nif);
455 }
456 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
457 return 0;
458 }
459
460 static int
netif_vp_na_activate(struct nexus_adapter * na,na_activate_mode_t mode)461 netif_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
462 {
463 errno_t err;
464
465 ASSERT(na->na_type == NA_NETIF_VP);
466 if (mode == NA_ACTIVATE_MODE_ON) {
467 err = netif_vp_na_activate_on(na);
468 } else {
469 err = netif_vp_na_activate_off(na);
470 }
471 SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx) %s err %d", na->na_name,
472 SK_KVA(na), na_activate_mode2str(mode), err);
473 return err;
474 }
475
476 /*
477 * A variation of netif data path for llw, which sends packets to the appropriate
478 * HW queue directly, bypassing AQM.
479 */
480 static int
netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,struct proc * p)481 netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter *dev_nifna,
482 struct __kern_packet *pkt_chain, struct proc *p)
483 {
484 #pragma unused(p)
485 struct __kern_packet *pkt = pkt_chain, *next;
486 struct __kern_packet *dpkt, *__single dpkt_head = NULL, **dpkt_tail = &dpkt_head;
487 struct __kern_channel_ring *ring = &dev_nifna->nifna_up.na_tx_rings[0];
488 #pragma unused(ring)
489 struct nx_netif *nif;
490 struct netif_queue *drvq;
491 struct netif_stats *nifs;
492 struct kern_nexus_provider *nxprov;
493 uint32_t pkt_count = 0, byte_count = 0;
494 errno_t err;
495
496 nif = dev_nifna->nifna_netif;
497 nifs = &nif->nif_stats;
498
499 while (pkt != NULL) {
500 next = pkt->pkt_nextpkt;
501 pkt->pkt_nextpkt = NULL;
502
503 netif_vp_dump_packet(pkt);
504
505 dpkt = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX);
506 if (__improbable(dpkt == NULL)) {
507 pkt = next; // nx_netif_pkt_to_pkt() frees pkt on error as well
508 err = ENOMEM;
509 goto error;
510 }
511
512 *dpkt_tail = dpkt;
513 dpkt_tail = &dpkt->pkt_nextpkt;
514
515 pkt = next;
516 }
517
518 nxprov = NX_PROV(nif->nif_nx);
519 drvq = NETIF_QSET_TX_QUEUE(nif->nif_default_llink->nll_default_qset, 0);
520
521 kern_packet_t ph = SK_PKT2PH(dpkt_head);
522
523 err = nxprov->nxprov_netif_ext.nxnpi_queue_tx_push(nxprov,
524 nif->nif_nx, drvq->nq_ctx, &ph, &pkt_count, &byte_count);
525
526 STATS_ADD(nifs, NETIF_STATS_VP_LL_SENT, pkt_count);
527 DTRACE_SKYWALK3(ll__sent, struct __kern_channel_ring *, ring,
528 uint32_t, pkt_count, uint32_t, byte_count);
529
530 kern_netif_increment_queue_stats(drvq, pkt_count, byte_count);
531
532 /*
533 * Free all unconsumed packets.
534 */
535 if (ph != 0) {
536 int dropcnt;
537
538 nx_netif_free_packet_chain(SK_PTR_ADDR_KPKT(ph), &dropcnt);
539
540 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
541 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
542 int, dropcnt);
543 }
544
545 return err;
546
547 error:
548 /*
549 * Free all packets.
550 */
551 if (pkt != NULL) {
552 int dropcnt;
553
554 nx_netif_free_packet_chain(pkt, &dropcnt);
555
556 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
557 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
558 int, dropcnt);
559 }
560
561 if (dpkt_head != NULL) {
562 int dropcnt;
563
564 nx_netif_free_packet_chain(dpkt_head, &dropcnt);
565
566 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
567 DTRACE_SKYWALK2(ll__dropped, struct __kern_channel_ring *, ring,
568 int, dropcnt);
569 }
570
571 return err;
572 }
573
574 static int
netif_vp_send_pkt_chain_common(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,boolean_t compat)575 netif_vp_send_pkt_chain_common(struct nexus_netif_adapter *dev_nifna,
576 struct __kern_packet *pkt_chain, boolean_t compat)
577 {
578 struct __kern_packet *pkt = pkt_chain, *next, *p;
579 struct nx_netif *nif = dev_nifna->nifna_netif;
580 struct netif_stats *nifs = &nif->nif_stats;
581 ifnet_t ifp = nif->nif_ifp;
582 struct mbuf *m;
583 boolean_t drop;
584 int cnt = 0;
585 errno_t err;
586
587 while (pkt != NULL) {
588 next = pkt->pkt_nextpkt;
589 pkt->pkt_nextpkt = NULL;
590 drop = FALSE;
591
592 if (compat) {
593 m = nx_netif_pkt_to_mbuf(dev_nifna, pkt, NETIF_CONVERT_TX);
594 if (m == NULL) {
595 pkt = next;
596 continue;
597 }
598 err = ifnet_enqueue_mbuf(ifp, m, FALSE, &drop);
599 } else {
600 p = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX);
601 if (p == NULL) {
602 pkt = next;
603 continue;
604 }
605 err = ifnet_enqueue_pkt(ifp, p, FALSE, &drop);
606 }
607 if (err != 0) {
608 SK_ERR("enqueue failed: %d", err);
609 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUE_FAILED);
610 if (drop) {
611 STATS_INC(nifs, NETIF_STATS_DROP);
612 }
613 DTRACE_SKYWALK2(enqueue__failed,
614 struct nexus_netif_adapter *, dev_nifna,
615 boolean_t, drop);
616 } else {
617 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUED);
618 cnt++;
619 }
620 pkt = next;
621 }
622 if (cnt > 0) {
623 ifnet_start(ifp);
624 }
625 return 0;
626 }
627
628 static int
netif_vp_send_pkt_chain(struct nexus_netif_adapter * dev_nifna,struct __kern_packet * pkt_chain,struct proc * p)629 netif_vp_send_pkt_chain(struct nexus_netif_adapter *dev_nifna,
630 struct __kern_packet *pkt_chain, struct proc *p)
631 {
632 struct nexus_adapter *na = &dev_nifna->nifna_up;
633
634 if (NETIF_IS_LOW_LATENCY(dev_nifna->nifna_netif)) {
635 return netif_vp_send_pkt_chain_low_latency(dev_nifna,
636 pkt_chain, p);
637 }
638 if (na->na_type == NA_NETIF_DEV) {
639 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, FALSE);
640 }
641 ASSERT(na->na_type == NA_NETIF_COMPAT_DEV);
642 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, TRUE);
643 }
644
645 SK_NO_INLINE_ATTRIBUTE
646 static boolean_t
validate_packet(struct nexus_netif_adapter * nifna,struct __kern_packet * pkt)647 validate_packet(struct nexus_netif_adapter *nifna, struct __kern_packet *pkt)
648 {
649 struct nx_netif *nif = nifna->nifna_netif;
650
651 VERIFY(pkt->pkt_nextpkt == NULL);
652
653 if (nx_netif_vp_accept_all != 0) {
654 return TRUE;
655 }
656 if (outbound_check == 0 ||
657 (NETIF_IS_LOW_LATENCY(nif) &&
658 outbound_check != OUTBOUND_CHECK_FORCED)) {
659 return TRUE;
660 }
661 if (!nx_netif_validate_macaddr(nif, pkt, NETIF_FLOW_OUTBOUND)) {
662 return FALSE;
663 }
664 if (!nx_netif_flow_match(nif, pkt, nifna->nifna_flow,
665 NETIF_FLOW_OUTBOUND)) {
666 return FALSE;
667 }
668 return TRUE;
669 }
670
671 static int
netif_vp_na_txsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)672 netif_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
673 uint32_t flags)
674 {
675 #pragma unused(flags)
676 kern_channel_slot_t last_slot = NULL, slot = NULL;
677 struct __kern_packet *__single head = NULL;
678 struct __kern_packet **tailp = &head, *pkt;
679 struct nexus_netif_adapter *nifna, *dev_nifna;
680 struct nx_netif *nif;
681 struct netif_stats *nifs;
682 kern_packet_t ph;
683 errno_t err;
684 int cnt = 0;
685
686 nifna = NIFNA(KRNA(kring));
687 nif = nifna->nifna_netif;
688 nifs = &nif->nif_stats;
689 for (;;) {
690 slot = kern_channel_get_next_slot(kring, slot, NULL);
691 if (slot == NULL) {
692 break;
693 }
694 ph = kern_channel_slot_get_packet(kring, slot);
695 if (__improbable(ph == 0)) {
696 SK_ERR("packet got dropped by internalize");
697 STATS_INC(nifs, NETIF_STATS_VP_DROP_INTERNALIZE_FAIL);
698 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
699 kring, kern_channel_slot_t, slot);
700 last_slot = slot;
701 continue;
702 }
703 pkt = SK_PTR_ADDR_KPKT(ph);
704 if (__improbable(pkt->pkt_length == 0)) {
705 SK_ERR("dropped zero length packet");
706 STATS_INC(nifs, NETIF_STATS_VP_BAD_PKT_LEN);
707 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
708 kring, kern_channel_slot_t, slot);
709 last_slot = slot;
710 continue;
711 }
712 err = kern_channel_slot_detach_packet(kring, slot, ph);
713 VERIFY(err == 0);
714
715 /* packet needs to be finalized after detach */
716 err = kern_packet_finalize(ph);
717 VERIFY(err == 0);
718 last_slot = slot;
719
720 if (NA_CHANNEL_EVENT_ATTACHED(KRNA(kring))) {
721 __packet_set_tx_nx_port(SK_PKT2PH(pkt),
722 KRNA(kring)->na_nx_port, nifna->nifna_gencnt);
723 }
724
725 if (validate_packet(nifna, pkt)) {
726 nx_netif_snoop(nif, pkt, FALSE);
727 cnt++;
728 *tailp = pkt;
729 tailp = &pkt->pkt_nextpkt;
730 } else {
731 nx_netif_free_packet(pkt);
732 }
733 }
734 if (cnt == 0) {
735 STATS_INC(nifs, NETIF_STATS_VP_SYNC_NO_PKTS);
736 DTRACE_SKYWALK2(no__data, struct nexus_netif_adapter *, nifna,
737 struct __kern_channel_ring *, kring);
738 return 0;
739 }
740 DTRACE_SKYWALK4(injected, struct nexus_netif_adapter *, nifna,
741 struct __kern_channel_ring *, kring, struct __kern_packet *, head,
742 int, cnt);
743 if (last_slot != NULL) {
744 kern_channel_advance_slot(kring, last_slot);
745 }
746
747 dev_nifna = NIFNA(nx_port_get_na(KRNA(kring)->na_nx,
748 NEXUS_PORT_NET_IF_DEV));
749
750 err = netif_vp_send_pkt_chain(dev_nifna, head, p);
751 if (err != 0) {
752 SK_ERR("send failed: %d\n", err);
753 }
754 return 0;
755 }
756
757 static int
netif_vp_na_rxsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)758 netif_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
759 uint32_t flags)
760 {
761 #pragma unused(p, flags)
762 (void) kr_reclaim(kring);
763 return 0;
764 }
765
766 static int
netif_vp_na_krings_create(struct nexus_adapter * na,struct kern_channel * ch)767 netif_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
768 {
769 ASSERT(na->na_type == NA_NETIF_VP);
770 return na_rings_mem_setup(na, FALSE, ch);
771 }
772
773
774 /* na_krings_delete callback for flow switch ports. */
775 static void
netif_vp_na_krings_delete(struct nexus_adapter * na,struct kern_channel * ch,boolean_t defunct)776 netif_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
777 boolean_t defunct)
778 {
779 ASSERT(na->na_type == NA_NETIF_VP);
780 na_rings_mem_teardown(na, ch, defunct);
781 }
782
783 static int
netif_vp_region_params_setup(struct nexus_adapter * na,struct skmem_region_params srp[SKMEM_REGIONS],struct kern_pbufpool ** tx_pp)784 netif_vp_region_params_setup(struct nexus_adapter *na,
785 struct skmem_region_params srp[SKMEM_REGIONS], struct kern_pbufpool **tx_pp)
786 {
787 #pragma unused (tx_pp)
788 uint32_t max_mtu;
789 uint32_t buf_sz, buf_cnt, nslots, afslots, evslots, totalrings;
790 struct nexus_adapter *devna;
791 struct kern_nexus *nx;
792 struct nx_netif *__single nif;
793 int err, i;
794
795 for (i = 0; i < SKMEM_REGIONS; i++) {
796 srp[i] = *skmem_get_default(i);
797 }
798 totalrings = na_get_nrings(na, NR_TX) + na_get_nrings(na, NR_RX) +
799 na_get_nrings(na, NR_A) + na_get_nrings(na, NR_F) +
800 na_get_nrings(na, NR_EV);
801
802 srp[SKMEM_REGION_SCHEMA].srp_r_obj_size =
803 (uint32_t)CHANNEL_SCHEMA_SIZE(totalrings);
804 srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = totalrings;
805 skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]);
806
807 srp[SKMEM_REGION_RING].srp_r_obj_size =
808 sizeof(struct __user_channel_ring);
809 srp[SKMEM_REGION_RING].srp_r_obj_cnt = totalrings;
810 skmem_region_params_config(&srp[SKMEM_REGION_RING]);
811
812 /* USD regions need to be writable to support user packet pool */
813 srp[SKMEM_REGION_TXAUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
814 srp[SKMEM_REGION_RXFUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
815
816 nslots = na_get_nslots(na, NR_TX);
817 afslots = na_get_nslots(na, NR_A);
818 evslots = na_get_nslots(na, NR_EV);
819 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size =
820 MAX(MAX(nslots, afslots), evslots) * SLOT_DESC_SZ;
821 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt =
822 na_get_nrings(na, NR_TX) + na_get_nrings(na, NR_A) +
823 na_get_nrings(na, NR_EV);
824 skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]);
825
826 /* USD and KSD objects share the same size and count */
827 srp[SKMEM_REGION_TXAUSD].srp_r_obj_size =
828 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size;
829 srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt =
830 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt;
831 skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]);
832
833 /*
834 * Since the rx/free slots share the same region and cache,
835 * we will use the same object size for both types of slots.
836 */
837 nslots = na_get_nslots(na, NR_RX);
838 afslots = na_get_nslots(na, NR_F);
839 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size =
840 MAX(nslots, afslots) * SLOT_DESC_SZ;
841 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt =
842 na_get_nrings(na, NR_RX) + na_get_nrings(na, NR_F);
843 skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]);
844
845 /* USD and KSD objects share the same size and count */
846 srp[SKMEM_REGION_RXFUSD].srp_r_obj_size =
847 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size;
848 srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt =
849 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt;
850 skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]);
851
852 /*
853 * No need to create our own buffer pool if we can share the device's
854 * pool. We don't support sharing split pools to user space.
855 */
856 nx = na->na_nx;
857 nif = nx->nx_arg;
858 if (vp_zerocopy != 0 && NETIF_IS_LOW_LATENCY(nif) &&
859 nx->nx_tx_pp != NULL && (nx->nx_rx_pp == NULL ||
860 nx->nx_tx_pp == nx->nx_rx_pp) && !PP_KERNEL_ONLY(nx->nx_tx_pp)) {
861 struct kern_pbufpool *pp = nx->nx_tx_pp;
862
863 if (nif->nif_hw_ch_refcnt != 0) {
864 SK_ERR("only one channel is supported for zero copy");
865 return ENOTSUP;
866 }
867 SK_DF(SK_VERB_VP, "sharing %s's pool", if_name(na->na_ifp));
868
869 /*
870 * These types need to be initialized otherwise some assertions
871 * skmem_arena_create_for_nexus() will fail.
872 */
873 srp[SKMEM_REGION_UMD].srp_md_type = pp->pp_md_type;
874 srp[SKMEM_REGION_UMD].srp_md_subtype = pp->pp_md_subtype;
875 srp[SKMEM_REGION_KMD].srp_md_type = pp->pp_md_type;
876 srp[SKMEM_REGION_KMD].srp_md_subtype = pp->pp_md_subtype;
877 *tx_pp = nx->nx_tx_pp;
878 return 0;
879 }
880
881 devna = nx_port_get_na(nx, NEXUS_PORT_NET_IF_DEV);
882 ASSERT(devna != NULL);
883 if (devna->na_type == NA_NETIF_DEV) {
884 /*
885 * For native devices, use the driver's buffer size
886 */
887 ASSERT(nx->nx_rx_pp != NULL);
888 ASSERT(nx->nx_tx_pp != NULL);
889 buf_sz = PP_BUF_SIZE_DEF(nx->nx_tx_pp);
890 } else {
891 if ((err = nx_netif_get_max_mtu(na->na_ifp, &max_mtu)) != 0) {
892 /*
893 * If the driver doesn't support SIOCGIFDEVMTU, use the
894 * default MTU size.
895 */
896 max_mtu = ifnet_mtu(na->na_ifp);
897 err = 0;
898 }
899 /* max_mtu does not include the L2 header */
900 buf_sz = MAX(max_mtu + sizeof(struct ether_vlan_header), 2048);
901 }
902 buf_cnt = vp_pool_size;
903 pp_regions_params_adjust(srp, NEXUS_META_TYPE_PACKET,
904 NEXUS_META_SUBTYPE_RAW, buf_cnt, 1, buf_sz, 0, buf_cnt, 0,
905 PP_REGION_CONFIG_BUF_IODIR_BIDIR |
906 PP_REGION_CONFIG_MD_MAGAZINE_ENABLE);
907
908 nx_netif_vp_region_params_adjust(na, srp);
909 return 0;
910 }
911
912 static int
netif_vp_na_mem_new(struct kern_nexus * nx,struct nexus_adapter * na)913 netif_vp_na_mem_new(struct kern_nexus *nx, struct nexus_adapter *na)
914 {
915 #pragma unused(nx)
916 struct skmem_region_params srp[SKMEM_REGIONS];
917 struct kern_pbufpool *__single tx_pp = NULL;
918 int err;
919
920 err = netif_vp_region_params_setup(na, srp, &tx_pp);
921 if (err != 0) {
922 return err;
923 }
924 na->na_arena = skmem_arena_create_for_nexus(na, srp,
925 tx_pp != NULL ? &tx_pp : NULL, NULL,
926 FALSE, FALSE, &nx->nx_adv, &err);
927 ASSERT(na->na_arena != NULL || err != 0);
928 return err;
929 }
930
931 static void
netif_vp_na_dtor(struct nexus_adapter * na)932 netif_vp_na_dtor(struct nexus_adapter *na)
933 {
934 struct kern_nexus *nx = na->na_nx;
935 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
936 struct nexus_netif_adapter *nifna = NIFNA(na);
937
938 NETIF_WLOCK(nif);
939 (void) nx_port_unbind(nx, na->na_nx_port);
940 nx_port_free(nx, na->na_nx_port);
941 nif->nif_vp_cnt--;
942 if (na->na_ifp != NULL) {
943 ifnet_decr_iorefcnt(na->na_ifp);
944 na->na_ifp = NULL;
945 }
946 if (nifna->nifna_netif != NULL) {
947 nx_netif_release(nifna->nifna_netif);
948 nifna->nifna_netif = NULL;
949 }
950 NETIF_WUNLOCK(nif);
951 SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx)", na->na_name, SK_KVA(na));
952 }
953
954 int
netif_vp_na_create(struct kern_nexus * nx,struct chreq * chr,struct nexus_adapter ** nap)955 netif_vp_na_create(struct kern_nexus *nx, struct chreq *chr,
956 struct nexus_adapter **nap)
957 {
958 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
959 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
960 struct nexus_adapter *na = NULL;
961 struct nexus_netif_adapter *nifna;
962 uint32_t slots;
963 int err;
964
965 NETIF_WLOCK_ASSERT_HELD(nif);
966 if (nif->nif_ifp == NULL) {
967 SK_ERR("ifnet not yet attached");
968 return ENXIO;
969 }
970 ASSERT((chr->cr_mode & CHMODE_KERNEL) == 0);
971 if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) {
972 SK_ERR("user packet pool required");
973 return EINVAL;
974 }
975 /*
976 * No locking needed while checking for the initialized bit because
977 * if this were not set, no other codepaths would modify the flags.
978 */
979 if ((nif->nif_flow_flags & NETIF_FLOW_FLAG_INITIALIZED) == 0) {
980 SK_ERR("demux vp not supported");
981 return ENOTSUP;
982 }
983 na = (struct nexus_adapter *)na_netif_alloc(Z_WAITOK);
984 nifna = NIFNA(na);
985 nifna->nifna_netif = nif;
986 nx_netif_retain(nif);
987 nifna->nifna_flow = NULL;
988
989 (void) snprintf(na->na_name, sizeof(na->na_name),
990 "netif_vp:%d", chr->cr_port);
991 uuid_generate_random(na->na_uuid);
992
993 na_set_nrings(na, NR_TX, nxp->nxp_tx_rings);
994 na_set_nrings(na, NR_RX, nxp->nxp_rx_rings);
995 /*
996 * If the packet pool is configured to be multi-buflet, then we
997 * need 2 pairs of alloc/free rings(for packet and buflet).
998 */
999 na_set_nrings(na, NR_A, ((nxp->nxp_max_frags > 1) &&
1000 (sk_channel_buflet_alloc != 0)) ? 2 : 1);
1001
1002 slots = vp_tx_slots != 0 ? vp_tx_slots :
1003 NX_DOM(nx)->nxdom_tx_slots.nb_def;
1004 na_set_nslots(na, NR_TX, slots);
1005
1006 slots = vp_rx_slots != 0 ? vp_rx_slots :
1007 NX_DOM(nx)->nxdom_rx_slots.nb_def;
1008 na_set_nslots(na, NR_RX, slots);
1009
1010 na_set_nslots(na, NR_A, NETIF_DEMUX_ALLOC_SLOTS);
1011 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
1012 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
1013 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
1014 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
1015
1016 os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed);
1017
1018 if (chr->cr_mode & CHMODE_EVENT_RING) {
1019 na_set_nrings(na, NR_EV, NX_NETIF_EVENT_RING_NUM);
1020 na_set_nslots(na, NR_EV, NX_NETIF_EVENT_RING_SIZE);
1021 os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed);
1022 na->na_channel_event_notify = netif_vp_na_channel_event_notify;
1023 }
1024
1025 na->na_nx_port = chr->cr_port;
1026 na->na_type = NA_NETIF_VP;
1027 na->na_free = na_netif_free;
1028 na->na_dtor = netif_vp_na_dtor;
1029 na->na_activate = netif_vp_na_activate;
1030 na->na_txsync = netif_vp_na_txsync;
1031 na->na_rxsync = netif_vp_na_rxsync;
1032 na->na_krings_create = netif_vp_na_krings_create;
1033 na->na_krings_delete = netif_vp_na_krings_delete;
1034 na->na_special = NULL;
1035 na->na_ifp = nif->nif_ifp;
1036 ifnet_incr_iorefcnt(na->na_ifp);
1037
1038 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
1039 NEXUS_STATS_TYPE_INVALID;
1040
1041 /* other fields are set in the common routine */
1042 na_attach_common(na, nx, &nx_netif_prov_s);
1043
1044 err = netif_vp_na_mem_new(nx, na);
1045 if (err != 0) {
1046 ASSERT(na->na_arena == NULL);
1047 goto err;
1048 }
1049
1050 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
1051 ASSERT(na->na_flowadv_max == 0 ||
1052 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
1053
1054 nif->nif_vp_cnt++;
1055 *nap = na;
1056 return 0;
1057
1058 err:
1059 if (na != NULL) {
1060 if (na->na_ifp != NULL) {
1061 ifnet_decr_iorefcnt(na->na_ifp);
1062 na->na_ifp = NULL;
1063 }
1064 if (na->na_arena != NULL) {
1065 skmem_arena_release(na->na_arena);
1066 na->na_arena = NULL;
1067 }
1068 if (nifna->nifna_netif != NULL) {
1069 nx_netif_release(nifna->nifna_netif);
1070 nifna->nifna_netif = NULL;
1071 }
1072 NA_FREE(na);
1073 }
1074 SK_ERR("VP NA creation failed, err(%d)", err);
1075 return err;
1076 }
1077
1078 static int
netif_vp_na_channel_event_notify(struct nexus_adapter * vpna,struct __kern_channel_event * ev,uint16_t ev_len)1079 netif_vp_na_channel_event_notify(struct nexus_adapter *vpna,
1080 struct __kern_channel_event *ev, uint16_t ev_len)
1081 {
1082 int err;
1083 char *baddr;
1084 kern_packet_t ph;
1085 kern_buflet_t buf;
1086 sk_protect_t protect;
1087 kern_channel_slot_t slot;
1088 struct __kern_packet *vpna_pkt = NULL;
1089 struct __kern_channel_event_metadata *emd;
1090 struct __kern_channel_ring *ring = &vpna->na_event_rings[0];
1091 struct netif_stats *nifs = &NIFNA(vpna)->nifna_netif->nif_stats;
1092
1093 if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) {
1094 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_STATUS);
1095 }
1096 if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) {
1097 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_EXPIRED);
1098 }
1099 STATS_INC(nifs, NETIF_STATS_EV_RECV);
1100
1101 if (__improbable(!NA_IS_ACTIVE(vpna))) {
1102 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_INACTIVE);
1103 err = ENXIO;
1104 goto error;
1105 }
1106 if (__improbable(NA_IS_DEFUNCT(vpna))) {
1107 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_DEFUNCT);
1108 err = ENXIO;
1109 goto error;
1110 }
1111 if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) {
1112 STATS_INC(nifs, NETIF_STATS_EV_DROP_KEVENT_INACTIVE);
1113 err = ENXIO;
1114 goto error;
1115 }
1116 if (__improbable(KR_DROP(ring))) {
1117 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRDROP_MODE);
1118 err = ENXIO;
1119 goto error;
1120 }
1121 vpna_pkt = nx_netif_alloc_packet(ring->ckr_pp, ev_len, &ph);
1122 if (__improbable(vpna_pkt == NULL)) {
1123 STATS_INC(nifs, NETIF_STATS_EV_DROP_NOMEM_PKT);
1124 err = ENOMEM;
1125 goto error;
1126 }
1127 buf = __packet_get_next_buflet(ph, NULL);
1128 baddr = __buflet_get_data_address(buf);
1129 emd = (struct __kern_channel_event_metadata *)(void *)baddr;
1130 emd->emd_etype = ev->ev_type;
1131 emd->emd_nevents = 1;
1132 bcopy(ev, (baddr + __KERN_CHANNEL_EVENT_OFFSET),
1133 ev->ev_dlen + sizeof(struct __kern_channel_event));
1134 err = __buflet_set_data_length(buf,
1135 (ev_len + __KERN_CHANNEL_EVENT_OFFSET));
1136 VERIFY(err == 0);
1137 err = __packet_finalize(ph);
1138 VERIFY(err == 0);
1139 kr_enter(ring, TRUE);
1140 protect = sk_sync_protect();
1141 slot = kern_channel_get_next_slot(ring, NULL, NULL);
1142 if (slot == NULL) {
1143 sk_sync_unprotect(protect);
1144 kr_exit(ring);
1145 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRSPACE);
1146 err = ENOBUFS;
1147 goto error;
1148 }
1149 err = kern_channel_slot_attach_packet(ring, slot, ph);
1150 VERIFY(err == 0);
1151 vpna_pkt = NULL;
1152 kern_channel_advance_slot(ring, slot);
1153 sk_sync_unprotect(protect);
1154 kr_exit(ring);
1155 kern_channel_event_notify(&vpna->na_tx_rings[0]);
1156 STATS_INC(nifs, NETIF_STATS_EV_SENT);
1157 return 0;
1158
1159 error:
1160 ASSERT(err != 0);
1161 if (vpna_pkt != NULL) {
1162 nx_netif_free_packet(vpna_pkt);
1163 }
1164 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1165 return err;
1166 }
1167
1168 static inline struct nexus_adapter *
nx_netif_find_port_vpna(struct nx_netif * netif,uint32_t nx_port_id)1169 nx_netif_find_port_vpna(struct nx_netif *netif, uint32_t nx_port_id)
1170 {
1171 struct kern_nexus *nx = netif->nif_nx;
1172 struct nexus_adapter *na = NULL;
1173 nexus_port_t port;
1174 uint16_t gencnt;
1175
1176 PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt);
1177 if (port < NEXUS_PORT_NET_IF_CLIENT) {
1178 SK_ERR("non VPNA port");
1179 return NULL;
1180 }
1181 if (__improbable(!nx_port_is_valid(nx, port))) {
1182 SK_ERR("%s[%d] port no longer valid",
1183 if_name(netif->nif_ifp), port);
1184 return NULL;
1185 }
1186 na = nx_port_get_na(nx, port);
1187 if (na != NULL && NIFNA(na)->nifna_gencnt != gencnt) {
1188 return NULL;
1189 }
1190 return na;
1191 }
1192
1193 errno_t
netif_vp_na_channel_event(struct nx_netif * nif,uint32_t nx_port_id,struct __kern_channel_event * event,uint16_t event_len)1194 netif_vp_na_channel_event(struct nx_netif *nif, uint32_t nx_port_id,
1195 struct __kern_channel_event *event, uint16_t event_len)
1196 {
1197 int err = 0;
1198 struct nexus_adapter *netif_vpna;
1199 struct netif_stats *nifs = &nif->nif_stats;
1200
1201 SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu "
1202 "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu",
1203 if_name(nif->nif_ifp), nx_port_id, event, event_len,
1204 event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen);
1205
1206 NETIF_RLOCK(nif);
1207 if (!NETIF_IS_LOW_LATENCY(nif)) {
1208 err = ENOTSUP;
1209 goto error;
1210 }
1211 if (__improbable(nif->nif_vp_cnt == 0)) {
1212 STATS_INC(nifs, NETIF_STATS_EV_DROP_NO_VPNA);
1213 err = ENXIO;
1214 goto error;
1215 }
1216 netif_vpna = nx_netif_find_port_vpna(nif, nx_port_id);
1217 if (__improbable(netif_vpna == NULL)) {
1218 err = ENXIO;
1219 STATS_INC(nifs, NETIF_STATS_EV_DROP_DEMUX_ERR);
1220 goto error;
1221 }
1222 if (__improbable(netif_vpna->na_channel_event_notify == NULL)) {
1223 err = ENOTSUP;
1224 STATS_INC(nifs, NETIF_STATS_EV_DROP_EV_VPNA_NOTSUP);
1225 goto error;
1226 }
1227 err = netif_vpna->na_channel_event_notify(netif_vpna, event, event_len);
1228 NETIF_RUNLOCK(nif);
1229 return err;
1230
1231 error:
1232 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1233 NETIF_RUNLOCK(nif);
1234 return err;
1235 }
1236
1237 /* XXX Remove once rdar://118519573 is resolved */
1238 #pragma clang diagnostic pop
1239