1 /*
2 * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <skywalk/os_skywalk_private.h>
29 #include <skywalk/nexus/netif/nx_netif.h>
30 #include <net/if_vlan_var.h>
31 #include <sys/sdt.h>
32
33 #define NETIF_FILTER_RX_RINGS 2 /* ring[0] for ingress, ring[1] for egress */
34 #define NETIF_FILTER_TX_RINGS 2 /* ring[0] for ingress, ring[1] for egress */
35 #define NETIF_FILTER_ALLOC_SLOTS 128 /* alloc ring size */
36 #define NETIF_FILTER_RING_INBOUND 0
37 #define NETIF_FILTER_RING_OUTBOUND 1
38
39 /* 0 means the buffer size is derived from the device MTU */
40 static uint32_t filter_buf_sz = 0;
41 static uint32_t filter_pool_size = 8192;
42 static uint32_t filter_tx_slots = 0;
43 static uint32_t filter_rx_slots = 0;
44
45 #if (DEVELOPMENT || DEBUG)
46 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, filter_buf_sz,
47 CTLFLAG_RW | CTLFLAG_LOCKED, &filter_buf_sz, 0,
48 "filter buffer size");
49 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, filter_pool_size,
50 CTLFLAG_RW | CTLFLAG_LOCKED, &filter_pool_size, 0,
51 "filter pool size");
52 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, filter_tx_slots,
53 CTLFLAG_RW | CTLFLAG_LOCKED, &filter_tx_slots, 0,
54 "filter tx slots");
55 SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, filter_rx_slots,
56 CTLFLAG_RW | CTLFLAG_LOCKED, &filter_rx_slots, 0,
57 "filter rx slots");
58 #endif /* (DEVELOPMENT || DEBUG) */
59
60 static void
netif_filter_dump_packet(struct __kern_packet * pkt)61 netif_filter_dump_packet(struct __kern_packet *pkt)
62 {
63 uint8_t *baddr;
64
65 MD_BUFLET_ADDR_ABS(pkt, baddr);
66 ASSERT(baddr != NULL);
67 baddr += pkt->pkt_headroom;
68
69 DTRACE_SKYWALK2(dump__packet, struct __kern_packet *,
70 pkt, uint8_t *, baddr);
71 }
72
73 SK_NO_INLINE_ATTRIBUTE
74 static errno_t
netif_filter_deliver(struct nexus_adapter * na,struct __kern_channel_ring * ring,struct __kern_packet * pkt_chain,uint32_t flags)75 netif_filter_deliver(struct nexus_adapter *na, struct __kern_channel_ring *ring,
76 struct __kern_packet *pkt_chain, uint32_t flags)
77 {
78 #pragma unused(flags)
79 struct __kern_packet *pkt = pkt_chain, *next;
80 kern_channel_slot_t last_slot = NULL, slot = NULL;
81 struct nexus_netif_adapter *nifna = NIFNA(na);
82 struct netif_stats *nifs = &nifna->nifna_netif->nif_stats;
83 sk_protect_t protect;
84 kern_packet_t ph;
85 int cnt = 0, dropcnt = 0;
86 errno_t err;
87
88 kr_enter(ring, TRUE);
89 protect = sk_sync_protect();
90
91 if (__improbable(KR_DROP(ring))) {
92 nx_netif_free_packet_chain(pkt, &dropcnt);
93 STATS_ADD(nifs, NETIF_STATS_FILTER_DROP_DISABLED_RING, dropcnt);
94 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
95 DTRACE_SKYWALK2(ring__drop, struct __kern_channel_ring *, ring,
96 int, dropcnt);
97 sk_sync_unprotect(protect);
98 kr_exit(ring);
99 return ENXIO;
100 }
101 while (pkt != NULL) {
102 slot = kern_channel_get_next_slot(ring, slot, NULL);
103 if (slot == NULL) {
104 break;
105 }
106
107 next = pkt->pkt_nextpkt;
108 pkt->pkt_nextpkt = NULL;
109 netif_filter_dump_packet(pkt);
110
111 ph = SK_PKT2PH(pkt);
112 err = kern_channel_slot_attach_packet(ring, slot, ph);
113 VERIFY(err == 0);
114
115 last_slot = slot;
116 pkt = next;
117 cnt++;
118 }
119 if (ring->ckr_ring_id == NETIF_FILTER_RING_INBOUND) {
120 STATS_ADD(nifs, NETIF_STATS_FILTER_RX_DELIVER, cnt);
121 } else {
122 STATS_ADD(nifs, NETIF_STATS_FILTER_TX_DELIVER, cnt);
123 }
124 DTRACE_SKYWALK4(delivered, struct nexus_adapter *, na,
125 struct __kern_channel_ring *, ring, struct __kern_packet *, pkt,
126 int, cnt);
127
128 if (pkt != NULL) {
129 nx_netif_free_packet_chain(pkt, &dropcnt);
130 STATS_ADD(nifs, NETIF_STATS_FILTER_DROP_NO_SPACE, dropcnt);
131 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
132 DTRACE_SKYWALK2(deliver__drop, struct nexus_adapter *, na,
133 int, dropcnt);
134 }
135 if (last_slot != NULL) {
136 kern_channel_advance_slot(ring, last_slot);
137 }
138 sk_sync_unprotect(protect);
139 kr_exit(ring);
140 if (cnt > 0) {
141 (void) kern_channel_notify(ring, 0);
142 }
143 return 0;
144 }
145
146 static errno_t
netif_filter_rx_cb(void * arg,struct __kern_packet * pkt_chain,uint32_t flags)147 netif_filter_rx_cb(void *arg, struct __kern_packet *pkt_chain, uint32_t flags)
148 {
149 struct nexus_adapter *__single na = arg;
150 struct __kern_channel_ring *ring =
151 &na->na_rx_rings[NETIF_FILTER_RING_INBOUND];
152 return netif_filter_deliver(na, ring, pkt_chain, flags);
153 }
154
155 static errno_t
netif_filter_tx_cb(void * arg,struct __kern_packet * pkt_chain,uint32_t flags)156 netif_filter_tx_cb(void *arg, struct __kern_packet *pkt_chain, uint32_t flags)
157 {
158 struct nexus_adapter *__single na = arg;
159 struct __kern_channel_ring *ring =
160 &na->na_rx_rings[NETIF_FILTER_RING_OUTBOUND];
161 return netif_filter_deliver(na, ring, pkt_chain, flags);
162 }
163
164 static errno_t
netif_filter_cb(void * arg,struct __kern_packet * pkt_chain,uint32_t flags)165 netif_filter_cb(void *arg, struct __kern_packet *pkt_chain, uint32_t flags)
166 {
167 errno_t err;
168
169 if ((flags & NETIF_FILTER_RX) != 0) {
170 err = netif_filter_rx_cb(arg, pkt_chain, flags);
171 } else {
172 err = netif_filter_tx_cb(arg, pkt_chain, flags);
173 }
174 return err;
175 }
176
177 static int
netif_filter_na_activate(struct nexus_adapter * na,na_activate_mode_t mode)178 netif_filter_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
179 {
180 errno_t err;
181 struct netif_filter *__single nf;
182 struct nexus_netif_adapter *nifna;
183
184 ASSERT(na->na_type == NA_NETIF_FILTER);
185 nifna = NIFNA(na);
186 if (mode == NA_ACTIVATE_MODE_ON) {
187 err = nx_netif_filter_add(nifna->nifna_netif, na->na_nx_port,
188 na, netif_filter_cb, &nf);
189 if (err != 0) {
190 return err;
191 }
192 nifna->nifna_filter = nf;
193 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
194 } else {
195 err = nx_netif_filter_remove(nifna->nifna_netif,
196 nifna->nifna_filter);
197 VERIFY(err == 0);
198 nifna->nifna_filter = NULL;
199 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
200 }
201
202 SK_DF(SK_VERB_FILTER, "na \"%s\" (0x%llx) %s", na->na_name,
203 SK_KVA(na), na_activate_mode2str(mode));
204 return 0;
205 }
206
207 static int
netif_filter_na_krings_create(struct nexus_adapter * na,struct kern_channel * ch)208 netif_filter_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
209 {
210 ASSERT(na->na_type == NA_NETIF_FILTER);
211 return na_rings_mem_setup(na, FALSE, ch);
212 }
213
214 static void
netif_filter_na_krings_delete(struct nexus_adapter * na,struct kern_channel * ch,boolean_t defunct)215 netif_filter_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
216 boolean_t defunct)
217 {
218 ASSERT(na->na_type == NA_NETIF_FILTER);
219 na_rings_mem_teardown(na, ch, defunct);
220 }
221
222 static int
netif_filter_region_params_setup(struct nexus_adapter * na,struct skmem_region_params srp[SKMEM_REGIONS])223 netif_filter_region_params_setup(struct nexus_adapter *na,
224 struct skmem_region_params srp[SKMEM_REGIONS])
225 {
226 uint32_t max_mtu;
227 uint32_t buf_sz, buf_cnt, nslots, afslots, totalrings;
228 int err, i;
229
230 if ((err = nx_netif_get_max_mtu(na->na_ifp, &max_mtu)) != 0) {
231 /*
232 * Use a default mtu if the driver doesn't support this
233 * ioctl.
234 */
235 max_mtu = ETHERMTU;
236 }
237 for (i = 0; i < SKMEM_REGIONS; i++) {
238 srp[i] = *skmem_get_default(i);
239 }
240 totalrings = na_get_nrings(na, NR_TX) + na_get_nrings(na, NR_RX) +
241 na_get_nrings(na, NR_A) + na_get_nrings(na, NR_F);
242
243 srp[SKMEM_REGION_SCHEMA].srp_r_obj_size =
244 (uint32_t)CHANNEL_SCHEMA_SIZE(totalrings);
245 srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = totalrings;
246 skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]);
247
248 srp[SKMEM_REGION_RING].srp_r_obj_size =
249 sizeof(struct __user_channel_ring);
250 srp[SKMEM_REGION_RING].srp_r_obj_cnt = totalrings;
251 skmem_region_params_config(&srp[SKMEM_REGION_RING]);
252
253 /* USD regions need to be writable to support user packet pool */
254 srp[SKMEM_REGION_TXAUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
255 srp[SKMEM_REGION_RXFUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
256
257 nslots = na_get_nslots(na, NR_TX);
258 afslots = na_get_nslots(na, NR_A);
259 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size =
260 MAX(nslots, afslots) * SLOT_DESC_SZ;
261 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt =
262 na_get_nrings(na, NR_TX) +
263 na_get_nrings(na, NR_A);
264 skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]);
265
266 /* USD and KSD objects share the same size and count */
267 srp[SKMEM_REGION_TXAUSD].srp_r_obj_size =
268 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size;
269 srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt =
270 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt;
271 skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]);
272
273 /*
274 * Since the rx/free slots share the same region and cache,
275 * we will use the same object size for both types of slots.
276 */
277 nslots = na_get_nslots(na, NR_RX);
278 afslots = na_get_nslots(na, NR_F);
279 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size =
280 MAX(nslots, afslots) * SLOT_DESC_SZ;
281 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt =
282 na_get_nrings(na, NR_RX) +
283 na_get_nrings(na, NR_F);
284 skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]);
285
286 /* USD and KSD objects share the same size and count */
287 srp[SKMEM_REGION_RXFUSD].srp_r_obj_size =
288 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size;
289 srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt =
290 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt;
291 skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]);
292
293 /* max_mtu does not include the L2 header */
294 buf_sz = (filter_buf_sz != 0) ? filter_buf_sz :
295 MAX(max_mtu + sizeof(struct ether_vlan_header), 2048);
296 buf_cnt = filter_pool_size;
297 pp_regions_params_adjust(srp, NEXUS_META_TYPE_PACKET,
298 NEXUS_META_SUBTYPE_RAW, buf_cnt, 1, buf_sz, 0, buf_cnt, 0,
299 (PP_REGION_CONFIG_BUF_IODIR_BIDIR |
300 PP_REGION_CONFIG_MD_MAGAZINE_ENABLE |
301 PP_REGION_CONFIG_BUF_UREADONLY));
302
303 nx_netif_vp_region_params_adjust(na, srp);
304 return 0;
305 }
306
307 static int
netif_filter_na_mem_new(struct nexus_adapter * na)308 netif_filter_na_mem_new(struct nexus_adapter *na)
309 {
310 struct kern_nexus *nx = na->na_nx;
311 struct nx_netif *__single nif = nx->nx_arg;
312 struct skmem_region_params srp[SKMEM_REGIONS];
313 int err;
314
315 NETIF_WLOCK_ASSERT_HELD(nif);
316 ASSERT(nif->nif_ifp != NULL);
317
318 err = netif_filter_region_params_setup(na, srp);
319 if (err != 0) {
320 return err;
321 }
322 /*
323 * Create buffer pool on first use.
324 * No locks held because no one will be using this until
325 * nif_filter_cnt > 0.
326 */
327 if (nif->nif_filter_pp == NULL) {
328 struct kern_pbufpool *pp = NULL;
329 uint32_t pp_flags = 0;
330 char pp_name[64];
331 const char *__null_terminated filter_pp_name;
332
333 filter_pp_name = tsnprintf(pp_name, sizeof(pp_name),
334 "%s_netif_filter_pp", if_name(nif->nif_ifp));
335 if (srp[SKMEM_REGION_KMD].srp_max_frags > 1) {
336 pp_flags |= PPCREATEF_ONDEMAND_BUF;
337 }
338 pp = pp_create(filter_pp_name, srp, NULL, NULL, NULL, NULL, NULL,
339 pp_flags);
340 if (pp == NULL) {
341 SK_ERR("failed to create filter pp");
342 return ENOMEM;
343 }
344 nif->nif_filter_pp = pp;
345 }
346 na->na_arena = skmem_arena_create_for_nexus(na, srp,
347 &nif->nif_filter_pp, NULL, FALSE, FALSE, NULL, &err);
348 ASSERT(na->na_arena != NULL || err != 0);
349 ASSERT(nx->nx_tx_pp == NULL || (nx->nx_tx_pp->pp_md_type ==
350 NX_DOM(nx)->nxdom_md_type && nx->nx_tx_pp->pp_md_subtype ==
351 NX_DOM(nx)->nxdom_md_subtype));
352
353 return 0;
354 }
355
356 static int
netif_filter_na_rxsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)357 netif_filter_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
358 uint32_t flags)
359 {
360 #pragma unused(p, flags)
361 (void) kr_reclaim(kring);
362 return 0;
363 }
364
365 struct filter_pktq {
366 struct __kern_packet *fp_head;
367 struct __kern_packet **fp_tailp;
368 };
369
370 static int
netif_filter_na_txsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)371 netif_filter_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
372 uint32_t flags)
373 {
374 #pragma unused(p, flags)
375 kern_channel_slot_t last_slot = NULL, slot = NULL;
376 struct filter_pktq pktq[KPKT_TC_MAX], *q;
377 struct __kern_packet *pkt;
378 struct nexus_netif_adapter *dev_nifna;
379 struct nexus_netif_adapter *nifna;
380 struct netif_stats *nifs;
381 uint32_t ringid, iflags = 0;
382 kern_packet_traffic_class_t tc;
383 kern_packet_t ph;
384 errno_t err;
385 int cnt = 0, i;
386
387 dev_nifna = NIFNA(nx_port_get_na(KRNA(kring)->na_nx,
388 NEXUS_PORT_NET_IF_DEV));
389 nifna = NIFNA(KRNA(kring));
390 nifs = &nifna->nifna_netif->nif_stats;
391
392 ringid = kring->ckr_ring_id;
393 ASSERT(ringid == NETIF_FILTER_RING_INBOUND ||
394 ringid == NETIF_FILTER_RING_OUTBOUND);
395 iflags = (ringid == NETIF_FILTER_RING_INBOUND) ? NETIF_FILTER_RX :
396 NETIF_FILTER_TX;
397 iflags |= NETIF_FILTER_INJECT;
398
399 for (i = 0; i < KPKT_TC_MAX; i++) {
400 pktq[i].fp_head = NULL;
401 pktq[i].fp_tailp = &pktq[i].fp_head;
402 }
403 for (;;) {
404 slot = kern_channel_get_next_slot(kring, slot, NULL);
405 if (slot == NULL) {
406 break;
407 }
408 ph = kern_channel_slot_get_packet(kring, slot);
409 if (__improbable(ph == 0)) {
410 SK_ERR("packet got dropped by internalize");
411 STATS_INC(nifs, NETIF_STATS_FILTER_DROP_INTERNALIZE);
412 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
413 kring, kern_channel_slot_t, slot);
414 last_slot = slot;
415 continue;
416 }
417 pkt = SK_PTR_ADDR_KPKT(ph);
418 if (__improbable(pkt->pkt_length == 0)) {
419 SK_ERR("dropped zero length packet");
420 STATS_INC(nifs, NETIF_STATS_FILTER_BAD_PKT_LEN);
421 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
422 kring, kern_channel_slot_t, slot);
423 last_slot = slot;
424 continue;
425 }
426 err = kern_channel_slot_detach_packet(kring, slot, ph);
427 VERIFY(err == 0);
428
429 /* packet needs to be finalized after detach */
430 err = kern_packet_finalize(ph);
431 VERIFY(err == 0);
432 last_slot = slot;
433
434 cnt++;
435 netif_filter_dump_packet(pkt);
436 VERIFY(pkt->pkt_nextpkt == NULL);
437
438 /*
439 * This returns a valid value even if the packet doesn't have a
440 * traffic class.
441 */
442 tc = kern_packet_get_traffic_class(ph);
443 VERIFY(tc < KPKT_TC_MAX);
444 q = &pktq[tc];
445
446 *q->fp_tailp = pkt;
447 q->fp_tailp = &pkt->pkt_nextpkt;
448 }
449 if (cnt == 0) {
450 STATS_ADD(nifs, NETIF_STATS_FILTER_SYNC_NO_PKTS, cnt);
451 DTRACE_SKYWALK2(no__data, struct nexus_netif_adapter *, nifna,
452 struct __kern_channel_ring *, kring);
453 return 0;
454 }
455 if (kring->ckr_ring_id == NETIF_FILTER_RING_INBOUND) {
456 STATS_ADD(nifs, NETIF_STATS_FILTER_RX_INJECT, cnt);
457 } else {
458 STATS_ADD(nifs, NETIF_STATS_FILTER_TX_INJECT, cnt);
459 }
460 DTRACE_SKYWALK4(injected, struct nexus_netif_adapter *, nifna,
461 struct __kern_channel_ring *, kring, int, cnt, uint32_t, iflags);
462
463 if (last_slot != NULL) {
464 kern_channel_advance_slot(kring, last_slot);
465 }
466 for (i = 0; i < KPKT_TC_MAX; i++) {
467 q = &pktq[i];
468 if (q->fp_head == NULL) {
469 continue;
470 }
471 err = nx_netif_filter_inject(dev_nifna, nifna->nifna_filter,
472 q->fp_head, iflags);
473 if (err != 0) {
474 DTRACE_SKYWALK3(inject__failed,
475 struct nexus_netif_adapter *, nifna,
476 struct __kern_channel_ring *, kring, int, err);
477 }
478 }
479 return 0;
480 }
481
482 static void
netif_filter_na_dtor(struct nexus_adapter * na)483 netif_filter_na_dtor(struct nexus_adapter *na)
484 {
485 struct kern_nexus *nx = na->na_nx;
486 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
487 struct nexus_netif_adapter *nifna = NIFNA(na);
488
489 NETIF_WLOCK(nif);
490 /*
491 * XXX port free does not belong here as this is not symmetrical
492 * with na_create below.
493 */
494 (void) nx_port_unbind(nx, na->na_nx_port);
495 nx_port_free(nx, na->na_nx_port);
496 nif->nif_filter_vp_cnt--;
497
498 /*
499 * TODO
500 * Move back the buffer pool free to here when we have the proper
501 * fix.
502 */
503 if (na->na_ifp != NULL) {
504 ifnet_decr_iorefcnt(na->na_ifp);
505 na->na_ifp = NULL;
506 }
507 if (nifna->nifna_netif != NULL) {
508 nx_netif_release(nifna->nifna_netif);
509 nifna->nifna_netif = NULL;
510 }
511 NETIF_WUNLOCK(nif);
512 SK_DF(SK_VERB_FILTER, "na \"%s\" (0x%llx)", na->na_name, SK_KVA(na));
513 }
514
515 int
netif_filter_na_create(struct kern_nexus * nx,struct chreq * chr,struct nexus_adapter ** nap)516 netif_filter_na_create(struct kern_nexus *nx, struct chreq *chr,
517 struct nexus_adapter **nap)
518 {
519 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
520 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
521 struct nexus_adapter *na = NULL;
522 struct nexus_netif_adapter *nifna;
523 uint32_t slots;
524 int err;
525
526 NETIF_WLOCK_ASSERT_HELD(nif);
527 if (nif->nif_ifp == NULL) {
528 SK_ERR("ifnet not yet attached");
529 return ENXIO;
530 }
531 ASSERT((chr->cr_mode & CHMODE_KERNEL) == 0);
532 ASSERT((chr->cr_mode & CHMODE_FILTER) != 0);
533 if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) {
534 SK_ERR("user packet pool required");
535 return EINVAL;
536 }
537 if ((chr->cr_mode & CHMODE_EVENT_RING) != 0) {
538 SK_ERR("event ring is not supported for netif filter channel");
539 return ENOTSUP;
540 }
541 /*
542 * No locking needed while checking for the initialized bit because
543 * if this were not set, no other codepaths would modify the flags.
544 */
545 if ((nif->nif_filter_flags & NETIF_FILTER_FLAG_INITIALIZED) == 0) {
546 SK_ERR("filter vp not supported");
547 return ENOTSUP;
548 }
549
550 /*
551 * XXX
552 * We should not be allocating from na_netif_alloc() because
553 * most fields are irrelevant. The filter adapter needs
554 * its own zone.
555 */
556 na = (struct nexus_adapter *)na_netif_alloc(Z_WAITOK);
557 nifna = NIFNA(na);
558 nifna->nifna_netif = nif;
559 nx_netif_retain(nif);
560 nifna->nifna_filter = NULL;
561
562 (void) snprintf(na->na_name, sizeof(na->na_name), "%s_filter:%d",
563 if_name(nif->nif_ifp), chr->cr_port);
564 uuid_generate_random(na->na_uuid);
565
566 na_set_nrings(na, NR_RX, NETIF_FILTER_RX_RINGS);
567 na_set_nrings(na, NR_TX, NETIF_FILTER_TX_RINGS);
568 /*
569 * If the packet pool is configured to be multi-buflet, then we
570 * need 2 pairs of alloc/free rings(for packet and buflet).
571 */
572 na_set_nrings(na, NR_A, ((nxp->nxp_max_frags > 1) &&
573 (sk_channel_buflet_alloc != 0)) ? 2 : 1);
574 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
575 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
576
577 slots = MAX(NX_DOM(nx)->nxdom_tx_slots.nb_def,
578 NX_DOM(nx)->nxdom_rx_slots.nb_def);
579
580 na_set_nslots(na, NR_TX, filter_tx_slots != 0 ?
581 filter_tx_slots : slots);
582 na_set_nslots(na, NR_RX, filter_rx_slots != 0 ?
583 filter_rx_slots : slots);
584
585 na_set_nslots(na, NR_A, NETIF_FILTER_ALLOC_SLOTS);
586 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
587 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
588
589 os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed);
590
591 na->na_nx_port = chr->cr_port;
592 na->na_type = NA_NETIF_FILTER;
593 na->na_free = na_netif_free;
594 na->na_dtor = netif_filter_na_dtor;
595 na->na_activate = netif_filter_na_activate;
596 na->na_txsync = netif_filter_na_txsync;
597 na->na_rxsync = netif_filter_na_rxsync;
598 na->na_krings_create = netif_filter_na_krings_create;
599 na->na_krings_delete = netif_filter_na_krings_delete;
600 na->na_special = NULL;
601 na->na_ifp = nif->nif_ifp;
602 ifnet_incr_iorefcnt(na->na_ifp);
603
604 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
605 NEXUS_STATS_TYPE_INVALID;
606
607 /* other fields are set in the common routine */
608 na_attach_common(na, nx, &nx_netif_prov_s);
609
610 err = netif_filter_na_mem_new(na);
611 if (err != 0) {
612 ASSERT(na->na_arena == NULL);
613 goto err;
614 }
615
616 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
617 ASSERT(na->na_flowadv_max == 0 ||
618 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
619
620 /*
621 * This is different from nif->nif_filter_cnt. This tracks the number
622 * of filter NAs created. nif->nif_filter_cnt tracks how many are
623 * attached to the datapath.
624 */
625 nif->nif_filter_vp_cnt++;
626 *nap = na;
627
628 SK_DF(SK_VERB_FILTER, "%s created", na->na_name);
629 return 0;
630
631 err:
632 if (na != NULL) {
633 if (na->na_ifp != NULL) {
634 ifnet_decr_iorefcnt(na->na_ifp);
635 na->na_ifp = NULL;
636 }
637 if (na->na_arena != NULL) {
638 skmem_arena_release(na->na_arena);
639 na->na_arena = NULL;
640 }
641 if (nifna->nifna_netif != NULL) {
642 nx_netif_release(nifna->nifna_netif);
643 nifna->nifna_netif = NULL;
644 }
645 NA_FREE(na);
646 }
647 SK_ERR("filter NA creation failed, err(%d)", err);
648 return err;
649 }
650