1 /*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54 /*
55 * BSD LICENSE
56 *
57 * Copyright(c) 2015 NEC Europe Ltd. All rights reserved.
58 * All rights reserved.
59 *
60 * Redistribution and use in source and binary forms, with or without
61 * modification, are permitted provided that the following conditions
62 * are met:
63 *
64 * * Redistributions of source code must retain the above copyright
65 * notice, this list of conditions and the following disclaimer.
66 * * Redistributions in binary form must reproduce the above copyright
67 * notice, this list of conditions and the following disclaimer in
68 * the documentation and/or other materials provided with the
69 * distribution.
70 * * Neither the name of NEC Europe Ltd. nor the names of
71 * its contributors may be used to endorse or promote products derived
72 * from this software without specific prior written permission.
73 *
74 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
75 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
76 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
77 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
78 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
79 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
80 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
81 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
82 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
83 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
84 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
85 */
86
87 #include <skywalk/os_skywalk_private.h>
88 #include <skywalk/nexus/flowswitch/nx_flowswitch.h>
89 #include <skywalk/nexus/flowswitch/fsw_var.h>
90 #include <skywalk/nexus/netif/nx_netif.h>
91 #include <skywalk/nexus/netif/nx_netif_compat.h>
92 #include <kern/sched_prim.h>
93 #include <sys/kdebug.h>
94 #include <sys/sdt.h>
95 #include <net/bpf.h>
96 #include <net/if_ports_used.h>
97 #include <net/pktap.h>
98 #include <net/pktsched/pktsched_netem.h>
99 #include <netinet/tcp.h>
100 #include <netinet/udp.h>
101 #include <netinet/ip.h>
102 #include <netinet/ip6.h>
103 #include <netinet/in_var.h>
104
105 extern kern_return_t thread_terminate(thread_t);
106
107 #define FSW_ZONE_MAX 256
108 #define FSW_ZONE_NAME "skywalk.nx.fsw"
109
110 static uint64_t fsw_reap_last __sk_aligned(8);
111 static uint64_t fsw_want_purge __sk_aligned(8);
112
113 #define NX_FSW_FE_TABLESZ 256 /* some power of 2 */
114 static uint32_t fsw_fe_table_size = NX_FSW_FE_TABLESZ;
115
116 #define NX_FSW_FOB_HASHSZ 31 /* some mersenne prime */
117 static uint32_t fsw_flow_owner_buckets = NX_FSW_FOB_HASHSZ;
118
119 #define NX_FSW_FRB_HASHSZ 128 /* some power of 2 */
120 static uint32_t fsw_flow_route_buckets = NX_FSW_FRB_HASHSZ;
121
122 #define NX_FSW_FRIB_HASHSZ 13 /* some mersenne prime */
123 static uint32_t fsw_flow_route_id_buckets = NX_FSW_FRIB_HASHSZ;
124
125 #define NX_FSW_FLOW_REAP_INTERVAL 1 /* seconds */
126 static uint32_t fsw_flow_reap_interval = NX_FSW_FLOW_REAP_INTERVAL;
127
128 #define NX_FSW_FLOW_PURGE_THRES 0 /* purge every N reaps (0 = disable) */
129 static uint32_t fsw_flow_purge_thresh = NX_FSW_FLOW_PURGE_THRES;
130
131 #define FSW_REAP_IVAL (MAX(1, fsw_flow_reap_interval))
132 #define FSW_REAP_SK_THRES (FSW_REAP_IVAL << 5)
133 #define FSW_REAP_IF_THRES (FSW_REAP_IVAL << 5)
134 #define FSW_DRAIN_CH_THRES (FSW_REAP_IVAL << 5)
135 #define FSW_IFSTATS_THRES 1
136
137 #define NX_FSW_CHANNEL_REAP_THRES 1000 /* threshold (bytes/sec) for reaping*/
138 uint64_t fsw_channel_reap_thresh = NX_FSW_CHANNEL_REAP_THRES;
139
140 #define RX_BUFLET_BATCH_COUNT 64 /* max batch size for buflet allocation */
141
142 uint32_t fsw_rx_batch = NX_FSW_RXBATCH; /* # of packets per batch (RX) */
143 uint32_t fsw_tx_batch = NX_FSW_TXBATCH; /* # of packets per batch (TX) */
144 uint32_t fsw_gso_batch = 8;
145 #if (DEVELOPMENT || DEBUG)
146 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, rx_batch,
147 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_rx_batch, 0,
148 "flowswitch Rx batch size");
149 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, tx_batch,
150 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_tx_batch, 0,
151 "flowswitch Tx batch size");
152 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, gso_batch,
153 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_gso_batch, 0,
154 "flowswitch GSO batch size");
155 SYSCTL_QUAD(_kern_skywalk_flowswitch, OID_AUTO, reap_throughput,
156 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_channel_reap_thresh,
157 "flowswitch channel reap threshold throughput (bytes/sec)");
158 #endif /* !DEVELOPMENT && !DEBUG */
159
160 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, rx_agg_tcp,
161 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_fsw_rx_agg_tcp, 0,
162 "flowswitch RX aggregation for tcp flows (enable/disable)");
163 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, rx_agg_tcp_host,
164 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_fsw_rx_agg_tcp_host, 0,
165 "flowswitch RX aggregation for tcp kernel path (0/1/2 (off/on/auto))");
166 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, gso_mtu,
167 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_fsw_gso_mtu, 0,
168 "flowswitch GSO for tcp flows (mtu > 0: enable, mtu == 0: disable)");
169
170 /*
171 * IP reassembly
172 * The "kern.skywalk.flowswitch.ip_reass" sysctl can be used to force
173 * enable/disable the reassembly routine regardless of whether the
174 * transport netagent is enabled or not.
175 *
176 * 'fsw_ip_reass' is a tri-state:
177 * 0 means force IP reassembly off
178 * 1 means force IP reassembly on
179 * 2 means don't force the value, use what's appropriate for this flowswitch
180 */
181 #define FSW_IP_REASS_FORCE_OFF 0
182 #define FSW_IP_REASS_FORCE_ON 1
183 #define FSW_IP_REASS_NO_FORCE 2
184
185 uint32_t fsw_ip_reass = FSW_IP_REASS_NO_FORCE;
186
187 static int
188 fsw_ip_reass_sysctl SYSCTL_HANDLER_ARGS
189 {
190 #pragma unused(oidp, arg1, arg2)
191 unsigned int new_value;
192 int changed;
193 int error;
194
195 error = sysctl_io_number(req, fsw_ip_reass, sizeof(fsw_ip_reass),
196 &new_value, &changed);
197 if (error == 0 && changed != 0) {
198 if (new_value > FSW_IP_REASS_NO_FORCE) {
199 return EINVAL;
200 }
201 fsw_ip_reass = new_value;
202 }
203 return error;
204 }
205
206 SYSCTL_PROC(_kern_skywalk_flowswitch, OID_AUTO, ip_reass,
207 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
208 0, 0, fsw_ip_reass_sysctl, "IU",
209 "adjust flowswitch IP reassembly");
210
211 #if (DEVELOPMENT || DEBUG)
212 static uint64_t _fsw_inject_error = 0;
213 #define _FSW_INJECT_ERROR(_en, _ev, _ec, _f, ...) \
214 _SK_INJECT_ERROR(_fsw_inject_error, _en, _ev, _ec, \
215 &FSW_STATS_VAL(_FSW_STATS_ERROR_INJECTIONS), _f, __VA_ARGS__)
216
217 #define _FSW_INJECT_ERROR_SET(_en, _f, ...) do { \
218 if (__improbable(((_fsw_inject_error) & (1ULL << (_en))) != 0)) { \
219 SK_DF(SK_VERB_ERROR_INJECT, "injecting error %d", (_en));\
220 if ((_f) != NULL) \
221 (_f)(__VA_ARGS__); \
222 } \
223 } while (0)
224
225 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, flow_owner_buckets,
226 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_flow_owner_buckets, 0, "");
227 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, fe_table_size,
228 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_fe_table_size, 0, "");
229 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, flow_route_buckets,
230 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_flow_route_buckets, 0, "");
231 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO,
232 flow_route_id_buckets, CTLFLAG_RW | CTLFLAG_LOCKED,
233 &fsw_flow_route_id_buckets, 0, "");
234 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, flow_reap_interval,
235 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_flow_reap_interval, 0, "");
236 SYSCTL_UINT(_kern_skywalk_flowswitch, OID_AUTO, flow_purge_thresh,
237 CTLFLAG_RW | CTLFLAG_LOCKED, &fsw_flow_purge_thresh, 0, "");
238 SYSCTL_QUAD(_kern_skywalk_flowswitch, OID_AUTO, fsw_inject_error,
239 CTLFLAG_RW | CTLFLAG_LOCKED, &_fsw_inject_error, "");
240 #else
241 #define _FSW_INJECT_ERROR(_en, _ev, _ec, _f, ...) do { } while (0)
242 #define _FSW_INJECT_ERROR_SET(_en, _f, ...) do { } while (0)
243 #endif /* !DEVELOPMENT && !DEBUG */
244
245 static void fsw_linger_remove_internal(struct flow_entry_linger_head *,
246 struct flow_entry *);
247 static void fsw_reap_thread_func(void *, wait_result_t);
248 static void fsw_reap_thread_cont(void *, wait_result_t);
249 static void fsw_purge_cache(struct nx_flowswitch *, boolean_t);
250 static void fsw_drain_channels(struct nx_flowswitch *, uint64_t, boolean_t);
251 static uint32_t fsw_process_deferred(struct nx_flowswitch *);
252 static uint32_t fsw_process_linger(struct nx_flowswitch *, uint32_t *);
253
254 static int copy_packet_from_dev(struct nx_flowswitch *, struct __kern_packet *,
255 struct __kern_packet *);
256
257 static void fsw_ifp_inc_traffic_class_in_pkt(struct ifnet *, kern_packet_t);
258 static void fsw_ifp_inc_traffic_class_out_pkt(struct ifnet *, uint32_t,
259 uint32_t, uint32_t);
260
261 static int __fsw_dp_inited = 0;
262
263 int
fsw_dp_init(void)264 fsw_dp_init(void)
265 {
266 _CASSERT(FSW_VP_DEV == 0);
267 _CASSERT(FSW_VP_HOST == 1);
268 _CASSERT((FSW_VP_HOST + FSW_VP_DEV) < FSW_VP_USER_MIN);
269 _CASSERT((FSW_VP_HOST + FSW_VP_DEV) < NEXUS_PORT_FLOW_SWITCH_CLIENT);
270
271 ASSERT(!__fsw_dp_inited);
272
273 flow_mgr_init();
274 flow_init();
275
276 __fsw_dp_inited = 1;
277
278 return 0;
279 }
280
281 void
fsw_dp_uninit(void)282 fsw_dp_uninit(void)
283 {
284 if (__fsw_dp_inited) {
285 flow_fini();
286 flow_mgr_fini();
287
288 __fsw_dp_inited = 0;
289 }
290 }
291
292 static void
dp_free_pktq(struct nx_flowswitch * fsw __sk_unused,struct pktq * pktq)293 dp_free_pktq(struct nx_flowswitch *fsw __sk_unused, struct pktq *pktq)
294 {
295 pp_free_pktq(pktq);
296 }
297
298 #define dp_drop_pktq(fsw, pktq) do { \
299 uint32_t _len = KPKTQ_LEN(pktq); \
300 if (KPKTQ_EMPTY(pktq)) { \
301 ASSERT(_len == 0); \
302 return; \
303 } \
304 SK_DF(SK_VERB_FSW_DP | SK_VERB_DROP, "drop %d packets", _len); \
305 FSW_STATS_ADD(FSW_STATS_DROP, _len); \
306 DTRACE_SKYWALK1(fsw__dp__drop, int, _len); \
307 dp_free_pktq(fsw, pktq); \
308 } while (0)
309
310 SK_NO_INLINE_ATTRIBUTE
311 void
fsw_snoop(struct nx_flowswitch * fsw,struct flow_entry * fe,bool input)312 fsw_snoop(struct nx_flowswitch *fsw, struct flow_entry *fe, bool input)
313 {
314 pid_t pid;
315 char proc_name_buf[FLOW_PROCESS_NAME_LENGTH];
316 char *proc_name = NULL;
317 pid_t epid;
318 char eproc_name_buf[FLOW_PROCESS_NAME_LENGTH];
319 char *eproc_name = NULL;
320 sa_family_t af;
321 bool tap_early = false;
322 struct __kern_packet *pkt;
323
324 ASSERT(fe != NULL);
325 ASSERT(fsw->fsw_ifp != NULL);
326
327 if (fe->fe_nx_port == FSW_VP_HOST) {
328 /* allow packets to be tapped before aggregation happens */
329 tap_early = (input && fe->fe_key.fk_proto == IPPROTO_TCP);
330 if (!tap_early) {
331 /* all other traffic will be tapped in the dlil input path */
332 return;
333 }
334 }
335 if (fe->fe_key.fk_ipver == IPVERSION) {
336 af = AF_INET;
337 } else if (fe->fe_key.fk_ipver == IPV6_VERSION) {
338 af = AF_INET6;
339 } else {
340 return;
341 }
342
343 pid = fe->fe_pid;
344 if (fe->fe_proc_name[0] != '\0') {
345 (void) strlcpy(proc_name_buf, fe->fe_proc_name,
346 sizeof(proc_name_buf));
347 proc_name = proc_name_buf;
348 }
349 epid = fe->fe_epid;
350 if (fe->fe_eproc_name[0] != '\0') {
351 (void) strlcpy(eproc_name_buf, fe->fe_eproc_name,
352 sizeof(eproc_name_buf));
353 eproc_name = eproc_name_buf;
354 }
355 if (input) {
356 KPKTQ_FOREACH(pkt, &fe->fe_rx_pktq) {
357 pktap_input_packet(fsw->fsw_ifp, af,
358 fsw->fsw_ifp_dlt, pid, proc_name, epid,
359 eproc_name, SK_PKT2PH(pkt), NULL, 0,
360 IPPROTO_TCP, fe->fe_flowid,
361 tap_early ? PTH_FLAG_SOCKET: PTH_FLAG_NEXUS_CHAN);
362 }
363 } else {
364 KPKTQ_FOREACH(pkt, &fe->fe_tx_pktq) {
365 pktap_output_packet(fsw->fsw_ifp, af,
366 fsw->fsw_ifp_dlt, pid, proc_name, epid,
367 eproc_name, SK_PKT2PH(pkt), NULL, 0,
368 0, 0, PTH_FLAG_NEXUS_CHAN);
369 }
370 }
371 }
372
373 #if (DEVELOPMENT || DEBUG)
374 static void
_fsw_error35_handler(int step,struct flow_route * fr,struct __kern_packet * pkt,int * ret)375 _fsw_error35_handler(int step, struct flow_route *fr, struct __kern_packet *pkt,
376 int *ret)
377 {
378 static boolean_t _err35_flag_modified = FALSE;
379
380 switch (step) {
381 case 1:
382 if ((fr->fr_flags & (FLOWRTF_RESOLVED | FLOWRTF_HAS_LLINFO)) ==
383 (FLOWRTF_RESOLVED | FLOWRTF_HAS_LLINFO)) {
384 fr->fr_flags &= ~FLOWRTF_RESOLVED;
385 _err35_flag_modified = TRUE;
386 }
387 break;
388
389 case 2:
390 if (!_err35_flag_modified) {
391 return;
392 }
393 if (pkt->pkt_pflags & PKT_F_MBUF_DATA) {
394 m_freem(pkt->pkt_mbuf);
395 pkt->pkt_pflags &= ~PKT_F_MBUF_DATA;
396 pkt->pkt_mbuf = NULL;
397 }
398 *ret = EJUSTRETURN;
399 fr->fr_flags |= FLOWRTF_RESOLVED;
400 _err35_flag_modified = FALSE;
401 break;
402
403 default:
404 VERIFY(0);
405 /* not reached */
406 }
407 }
408
409 static void
_fsw_error36_handler(int step,struct flow_route * fr,int * ret)410 _fsw_error36_handler(int step, struct flow_route *fr, int *ret)
411 {
412 static boolean_t _err36_flag_modified = FALSE;
413
414 switch (step) {
415 case 1:
416 if ((fr->fr_flags & (FLOWRTF_RESOLVED | FLOWRTF_HAS_LLINFO)) ==
417 (FLOWRTF_RESOLVED | FLOWRTF_HAS_LLINFO)) {
418 fr->fr_flags &= ~FLOWRTF_RESOLVED;
419 _err36_flag_modified = TRUE;
420 }
421 break;
422
423 case 2:
424 if (!_err36_flag_modified) {
425 return;
426 }
427 *ret = ENETUNREACH;
428 fr->fr_flags |= FLOWRTF_RESOLVED;
429 _err36_flag_modified = FALSE;
430 break;
431
432 default:
433 VERIFY(0);
434 /* not reached */
435 }
436 }
437 #else /* !DEVELOPMENT && !DEBUG */
438 #define _fsw_error35_handler(...)
439 #define _fsw_error36_handler(...)
440 #endif /* DEVELOPMENT || DEBUG */
441
442 /*
443 * Check if the source packet content can fit into the destination
444 * ring's packet. Returns TRUE if the source packet can fit.
445 * Note: Failures could be caused by misconfigured packet pool sizes,
446 * missing packet size check again MTU or if the source packet is from
447 * a compat netif and the attached mbuf is larger than MTU due to LRO.
448 */
449 static inline boolean_t
validate_pkt_len(struct __kern_packet * spkt,kern_packet_t dph,uint32_t skip_l2hlen,uint32_t l2hlen,uint16_t headroom,uint32_t * copy_len)450 validate_pkt_len(struct __kern_packet *spkt, kern_packet_t dph,
451 uint32_t skip_l2hlen, uint32_t l2hlen, uint16_t headroom,
452 uint32_t *copy_len)
453 {
454 uint32_t tlen = 0;
455 uint32_t splen = spkt->pkt_length - skip_l2hlen;
456
457 if (l2hlen != 0) {
458 VERIFY(skip_l2hlen == 0);
459 tlen += l2hlen;
460 } else if ((spkt->pkt_link_flags & PKT_LINKF_ETHFCS) != 0) {
461 splen -= ETHER_CRC_LEN;
462 }
463
464 tlen += splen;
465 *copy_len = splen;
466
467 return tlen <= ((__packet_get_buflet_count(dph) *
468 PP_BUF_SIZE_DEF(SK_PTR_ADDR_KPKT(dph)->pkt_qum.qum_pp)) -
469 headroom);
470 }
471
472 #if SK_LOG
473 /* Hoisted out of line to reduce kernel stack footprint */
474 SK_LOG_ATTRIBUTE
475 static void
copy_packet_from_dev_log(struct __kern_packet * spkt,struct __kern_packet * dpkt,struct proc * p)476 copy_packet_from_dev_log(struct __kern_packet *spkt,
477 struct __kern_packet *dpkt, struct proc *p)
478 {
479 uint64_t logflags = ((SK_VERB_FSW | SK_VERB_RX) |
480 ((spkt->pkt_pflags & PKT_F_MBUF_DATA) ?
481 SK_VERB_COPY_MBUF : SK_VERB_COPY));
482 char *daddr;
483 MD_BUFLET_ADDR_ABS(dpkt, daddr);
484 SK_DF(logflags, "%s(%d) splen %u dplen %u hr %u l2 %u",
485 sk_proc_name_address(p), sk_proc_pid(p), spkt->pkt_length,
486 dpkt->pkt_length, (uint32_t)dpkt->pkt_headroom,
487 (uint32_t)dpkt->pkt_l2_len);
488 SK_DF(logflags | SK_VERB_DUMP, "%s",
489 sk_dump("buf", daddr, dpkt->pkt_length, 128, NULL, 0));
490 }
491 #else
492 #define copy_packet_from_dev_log(...)
493 #endif /* SK_LOG */
494
495
496 static inline int
copy_packet_from_dev(struct nx_flowswitch * fsw,struct __kern_packet * spkt,struct __kern_packet * dpkt)497 copy_packet_from_dev(struct nx_flowswitch *fsw, struct __kern_packet *spkt,
498 struct __kern_packet *dpkt)
499 {
500 /*
501 * source and destination nexus don't share the packet pool
502 * sync operation here is to
503 * - alloc packet for the rx(dst) ring
504 * - copy data/metadata from src packet to dst packet
505 * - attach alloc'd packet to rx(dst) ring
506 */
507 kern_packet_t dph = SK_PTR_ENCODE(dpkt,
508 METADATA_TYPE(dpkt), METADATA_SUBTYPE(dpkt));
509 kern_packet_t sph = SK_PTR_ENCODE(spkt, METADATA_TYPE(spkt),
510 METADATA_SUBTYPE(spkt));
511 boolean_t do_cksum_rx;
512 uint16_t skip_l2h_len = spkt->pkt_l2_len;
513 uint16_t iphlen;
514 uint32_t dlen;
515 int err;
516
517 if (__improbable(!validate_pkt_len(spkt, dph, skip_l2h_len, 0, 0,
518 &dlen))) {
519 SK_ERR("bufcnt %d, bufsz %d", __packet_get_buflet_count(dph),
520 PP_BUF_SIZE_DEF(dpkt->pkt_qum.qum_pp));
521 FSW_STATS_INC(FSW_STATS_RX_COPY_BAD_LEN);
522 return EINVAL;
523 }
524
525 /* Copy packet metadata */
526 _QUM_COPY(&(spkt)->pkt_qum, &(dpkt)->pkt_qum);
527 _PKT_COPY(spkt, dpkt);
528 ASSERT(!(dpkt->pkt_qum.qum_qflags & QUM_F_KERNEL_ONLY) ||
529 PP_KERNEL_ONLY(dpkt->pkt_qum.qum_pp));
530 ASSERT(dpkt->pkt_mbuf == NULL);
531
532 dpkt->pkt_headroom = 0;
533 dpkt->pkt_l2_len = 0;
534
535 /* don't include IP header from partial sum */
536 if (__probable((spkt->pkt_qum_qflags & QUM_F_FLOW_CLASSIFIED) != 0)) {
537 iphlen = spkt->pkt_flow_ip_hlen;
538 do_cksum_rx = sk_cksum_rx;
539 } else {
540 iphlen = 0;
541 do_cksum_rx = FALSE;
542 }
543
544 /* Copy packet payload */
545 if ((spkt->pkt_pflags & PKT_F_MBUF_DATA) &&
546 (spkt->pkt_pflags & PKT_F_TRUNCATED)) {
547 FSW_STATS_INC(FSW_STATS_RX_COPY_MBUF2PKT);
548 /*
549 * Source packet has truncated contents (just enough for
550 * the classifer) of an mbuf from the compat driver; copy
551 * the entire entire mbuf contents to destination packet.
552 */
553 m_adj(spkt->pkt_mbuf, skip_l2h_len);
554 ASSERT((uint32_t)m_pktlen(spkt->pkt_mbuf) >= dlen);
555 fsw->fsw_pkt_copy_from_mbuf(NR_RX, dph, 0,
556 spkt->pkt_mbuf, 0, dlen, do_cksum_rx, iphlen);
557 } else {
558 FSW_STATS_INC(FSW_STATS_RX_COPY_PKT2PKT);
559 /*
560 * Source packet has full contents, either from an mbuf
561 * that came up from the compat driver, or because it
562 * originated on the native driver; copy to destination.
563 */
564 fsw->fsw_pkt_copy_from_pkt(NR_RX, dph, 0, sph,
565 (spkt->pkt_headroom + spkt->pkt_l2_len), dlen, do_cksum_rx,
566 iphlen, 0, FALSE);
567 }
568
569 #if DEBUG || DEVELOPMENT
570 if (__improbable(pkt_trailers > 0)) {
571 dlen += pkt_add_trailers(dph, dlen, iphlen);
572 }
573 #endif /* DEBUG || DEVELOPMENT */
574
575 /* Finalize and attach packet to Rx ring */
576 METADATA_ADJUST_LEN(dpkt, 0, 0);
577 err = __packet_finalize(dph);
578 VERIFY(err == 0);
579
580 copy_packet_from_dev_log(spkt, dpkt, kernproc);
581
582 if (spkt->pkt_pflags & PKT_F_MBUF_DATA) {
583 ifp_inc_traffic_class_in(fsw->fsw_ifp, spkt->pkt_mbuf);
584 mbuf_free(spkt->pkt_mbuf);
585 KPKT_CLEAR_MBUF_DATA(spkt);
586 } else {
587 fsw_ifp_inc_traffic_class_in_pkt(fsw->fsw_ifp, dph);
588 }
589
590 if (__probable(do_cksum_rx != 0)) {
591 FSW_STATS_INC(FSW_STATS_RX_COPY_SUM);
592 }
593
594 return 0;
595 }
596
597 SK_NO_INLINE_ATTRIBUTE
598 static struct __kern_packet *
rx_process_ip_frag(struct nx_flowswitch * fsw,struct __kern_packet * pkt)599 rx_process_ip_frag(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
600 {
601 char *pkt_buf;
602 void *l3_hdr;
603 uint16_t nfrags, tlen;
604 int err = 0;
605
606 switch (fsw_ip_reass) {
607 case FSW_IP_REASS_FORCE_OFF:
608 return pkt;
609 case FSW_IP_REASS_FORCE_ON:
610 break;
611 default:
612 if (!FSW_NETAGENT_ENABLED(fsw) ||
613 flow_mgr_get_num_flows(fsw->fsw_flow_mgr) == 0) {
614 return pkt;
615 }
616 break;
617 }
618
619 MD_BUFLET_ADDR_ABS(pkt, pkt_buf);
620 l3_hdr = pkt_buf + pkt->pkt_headroom + pkt->pkt_l2_len;
621
622 ASSERT(fsw->fsw_ipfm != NULL);
623 ASSERT((pkt->pkt_qum_qflags & QUM_F_FLOW_CLASSIFIED) != 0);
624
625 if (pkt->pkt_flow_ip_ver == IPVERSION) {
626 err = fsw_ip_frag_reass_v4(fsw->fsw_ipfm, &pkt,
627 (struct ip *)l3_hdr, &nfrags, &tlen);
628 } else {
629 ASSERT(pkt->pkt_flow_ip_ver == IPV6_VERSION);
630 /* we only handle frag header immediately after v6 header */
631 err = fsw_ip_frag_reass_v6(fsw->fsw_ipfm, &pkt,
632 (struct ip6_hdr *)l3_hdr,
633 (struct ip6_frag *)((uintptr_t)l3_hdr + sizeof(struct ip6_hdr)),
634 &nfrags, &tlen);
635 }
636 if (__improbable(err != 0)) {
637 /* if we get a bad fragment, free it */
638 pp_free_packet_single(pkt);
639 pkt = NULL;
640 } else {
641 ASSERT(!((pkt != NULL) ^ (nfrags > 0)));
642 }
643
644 return pkt;
645 }
646
647 SK_NO_INLINE_ATTRIBUTE
648 static void
rx_prepare_packet_mbuf(struct nx_flowswitch * fsw,struct __kern_packet * pkt)649 rx_prepare_packet_mbuf(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
650 {
651 ASSERT(pkt->pkt_pflags & PKT_F_MBUF_DATA);
652 uint32_t mlen = (uint32_t)m_pktlen(pkt->pkt_mbuf);
653 kern_packet_t ph = SK_PTR_ENCODE(pkt,
654 METADATA_TYPE(pkt), METADATA_SUBTYPE(pkt));
655 /*
656 * This is the case when the packet is coming in from
657 * compat-netif. This packet only has valid metadata
658 * and an attached mbuf. We need to copy enough data
659 * from the mbuf to the packet buffer for the
660 * classifier. Compat netif packet pool is configured
661 * with buffer size of NETIF_COMPAT_MAX_MBUF_DATA_COPY
662 * which is just enough to hold the protocol headers
663 * for the flowswitch classifier.
664 */
665
666 pkt->pkt_headroom = 0;
667 METADATA_ADJUST_LEN(pkt, 0, 0);
668 /*
669 * Copy the initial 128 bytes of the packet for
670 * classification.
671 * Ethernet(14) + IPv6 header(40) +
672 * + IPv6 fragment header(8) +
673 * TCP header with options(60).
674 */
675 fsw->fsw_pkt_copy_from_mbuf(NR_RX, ph,
676 pkt->pkt_headroom, pkt->pkt_mbuf, 0,
677 MIN(mlen, NETIF_COMPAT_MAX_MBUF_DATA_COPY),
678 FALSE, 0);
679
680 int err = __packet_finalize_with_mbuf(pkt);
681 VERIFY(err == 0);
682 }
683
684 static struct __kern_packet *
rx_prepare_packet(struct nx_flowswitch * fsw,struct __kern_packet * pkt)685 rx_prepare_packet(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
686 {
687 pkt->pkt_qum_qflags &= ~QUM_F_FLOW_CLASSIFIED;
688
689 if (__improbable(pkt->pkt_pflags & PKT_F_MBUF_DATA)) {
690 rx_prepare_packet_mbuf(fsw, pkt);
691 }
692
693 return pkt;
694 }
695
696 static struct flow_entry *
lookup_flow_with_pkt(struct nx_flowswitch * fsw,struct __kern_packet * pkt,bool input,struct flow_entry * prev_fe)697 lookup_flow_with_pkt(struct nx_flowswitch *fsw, struct __kern_packet *pkt,
698 bool input, struct flow_entry *prev_fe)
699 {
700 struct flow_key key __sk_aligned(16);
701 struct flow_entry *fe = NULL;
702
703 ASSERT(pkt->pkt_qum_qflags & QUM_F_FLOW_CLASSIFIED);
704 flow_pkt2key(pkt, input, &key);
705
706 if (__probable(prev_fe != NULL &&
707 prev_fe->fe_key.fk_mask == FKMASK_5TUPLE)) {
708 uint16_t saved_mask = key.fk_mask;
709 key.fk_mask = FKMASK_5TUPLE;
710 if (flow_key_cmp_mask(&prev_fe->fe_key, &key, &fk_mask_5tuple) == 0) {
711 flow_entry_retain(prev_fe);
712 fe = prev_fe;
713 } else {
714 key.fk_mask = saved_mask;
715 }
716 }
717
718 top:
719 if (__improbable(fe == NULL)) {
720 fe = flow_mgr_find_fe_by_key(fsw->fsw_flow_mgr, &key);
721 }
722
723 if (__improbable(fe != NULL &&
724 (fe->fe_flags & (FLOWENTF_PARENT | FLOWENTF_CHILD)) != 0)) {
725 /* Rx */
726 if (input) {
727 if (fe->fe_flags & FLOWENTF_PARENT) {
728 struct flow_entry *child_fe = rx_lookup_child_flow(fsw, fe, pkt);
729 if (child_fe != NULL) {
730 flow_entry_release(&fe);
731 fe = child_fe;
732 }
733 } else {
734 if (!rx_flow_demux_match(fsw, fe, pkt)) {
735 flow_entry_release(&fe);
736 fe = NULL;
737 goto top;
738 }
739 }
740 } else {
741 /* Tx */
742 if (__improbable(!_UUID_MATCH(pkt->pkt_flow_id, fe->fe_uuid))) {
743 if (__probable(fe->fe_flags & FLOWENTF_PARENT)) {
744 struct flow_entry *parent_fe = fe;
745 fe = tx_lookup_child_flow(parent_fe, pkt->pkt_flow_id);
746 flow_entry_release(&parent_fe);
747 } else {
748 flow_entry_release(&fe);
749 fe = NULL;
750 goto top;
751 }
752 }
753 }
754 }
755
756 SK_LOG_VAR(char fkbuf[FLOWKEY_DBGBUF_SIZE]);
757 SK_DF(SK_VERB_FSW_DP | SK_VERB_LOOKUP,
758 "%s %s %s \"%s\" fe 0x%llx",
759 input ? "Rx" : "Tx", if_name(fsw->fsw_ifp),
760 sk_proc_name_address(current_proc()),
761 fk_as_string(&key, fkbuf, sizeof(fkbuf)),
762 SK_KVA(fe));
763
764 return fe;
765 }
766
767 SK_NO_INLINE_ATTRIBUTE
768 static bool
pkt_is_for_listener(struct flow_entry * fe,struct __kern_packet * pkt)769 pkt_is_for_listener(struct flow_entry *fe, struct __kern_packet *pkt)
770 {
771 struct nx_flowswitch *fsw = fe->fe_fsw;
772 struct ifnet *ifp = fsw->fsw_ifp;
773 struct in_ifaddr *ia = NULL;
774 struct in_ifaddr *best_ia = NULL;
775 struct in6_ifaddr *ia6 = NULL;
776 struct in6_ifaddr *best_ia6 = NULL;
777 struct ifnet *match_ifp = NULL;
778 struct __flow *flow = pkt->pkt_flow;
779 bool result = false;
780
781 ASSERT(pkt->pkt_qum_qflags & QUM_F_FLOW_CLASSIFIED);
782
783 if (flow->flow_ip_ver == IPVERSION) {
784 if (IN_ZERONET(ntohl(flow->flow_ipv4_dst.s_addr)) ||
785 IN_LOOPBACK(ntohl(flow->flow_ipv4_dst.s_addr)) ||
786 IN_LINKLOCAL(ntohl(flow->flow_ipv4_dst.s_addr)) ||
787 IN_DS_LITE(ntohl(flow->flow_ipv4_dst.s_addr)) ||
788 IN_6TO4_RELAY_ANYCAST(ntohl(flow->flow_ipv4_dst.s_addr)) ||
789 IN_MULTICAST(ntohl(flow->flow_ipv4_dst.s_addr)) ||
790 INADDR_BROADCAST == flow->flow_ipv4_dst.s_addr) {
791 result = true;
792 goto done;
793 }
794
795 /*
796 * Check for a match in the hash bucket.
797 */
798 lck_rw_lock_shared(&in_ifaddr_rwlock);
799 TAILQ_FOREACH(ia, INADDR_HASH(flow->flow_ipv4_dst.s_addr), ia_hash) {
800 if (IA_SIN(ia)->sin_addr.s_addr == flow->flow_ipv4_dst.s_addr) {
801 best_ia = ia;
802 match_ifp = ia->ia_ifp;
803
804 if (match_ifp == ifp) {
805 break;
806 }
807 /*
808 * Continue the loop in case there's a exact match with another
809 * interface
810 */
811 }
812 }
813
814 if (best_ia != NULL) {
815 if (match_ifp != ifp && ipforwarding == 0 &&
816 (match_ifp->if_family == IFNET_FAMILY_IPSEC ||
817 match_ifp->if_family == IFNET_FAMILY_UTUN)) {
818 /*
819 * Drop when interface address check is strict and forwarding
820 * is disabled
821 */
822 } else {
823 lck_rw_done(&in_ifaddr_rwlock);
824 result = true;
825 goto done;
826 }
827 }
828 lck_rw_done(&in_ifaddr_rwlock);
829
830 if (ifp->if_flags & IFF_BROADCAST) {
831 /*
832 * Check for broadcast addresses.
833 *
834 * Only accept broadcast packets that arrive via the matching
835 * interface. Reception of forwarded directed broadcasts would be
836 * handled via ip_forward() and ether_frameout() with the loopback
837 * into the stack for SIMPLEX interfaces handled by ether_frameout().
838 */
839 struct ifaddr *ifa;
840
841 ifnet_lock_shared(ifp);
842 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
843 if (ifa->ifa_addr->sa_family != AF_INET) {
844 continue;
845 }
846 ia = ifatoia(ifa);
847 if (satosin(&ia->ia_broadaddr)->sin_addr.s_addr == flow->flow_ipv4_dst.s_addr ||
848 ia->ia_netbroadcast.s_addr == flow->flow_ipv4_dst.s_addr) {
849 ifnet_lock_done(ifp);
850 result = true;
851 goto done;
852 }
853 }
854 ifnet_lock_done(ifp);
855 }
856 } else {
857 if (IN6_IS_ADDR_LOOPBACK(&flow->flow_ipv6_dst) ||
858 IN6_IS_ADDR_LINKLOCAL(&flow->flow_ipv6_dst) ||
859 IN6_IS_ADDR_MULTICAST(&flow->flow_ipv6_dst)) {
860 result = true;
861 goto done;
862 }
863
864 /*
865 * Check for exact addresses in the hash bucket.
866 */
867 lck_rw_lock_shared(&in6_ifaddr_rwlock);
868 TAILQ_FOREACH(ia6, IN6ADDR_HASH(&flow->flow_ipv6_dst), ia6_hash) {
869 if (in6_are_addr_equal_scoped(&ia6->ia_addr.sin6_addr, &flow->flow_ipv6_dst, ia6->ia_ifp->if_index, ifp->if_index)) {
870 if ((ia6->ia6_flags & (IN6_IFF_NOTREADY | IN6_IFF_CLAT46))) {
871 continue;
872 }
873 best_ia6 = ia6;
874 if (ia6->ia_ifp == ifp) {
875 break;
876 }
877 /*
878 * Continue the loop in case there's a exact match with another
879 * interface
880 */
881 }
882 }
883 if (best_ia6 != NULL) {
884 if (best_ia6->ia_ifp != ifp && ip6_forwarding == 0 &&
885 (best_ia6->ia_ifp->if_family == IFNET_FAMILY_IPSEC ||
886 best_ia6->ia_ifp->if_family == IFNET_FAMILY_UTUN)) {
887 /*
888 * Drop when interface address check is strict and forwarding
889 * is disabled
890 */
891 } else {
892 lck_rw_done(&in6_ifaddr_rwlock);
893 result = true;
894 goto done;
895 }
896 }
897 lck_rw_done(&in6_ifaddr_rwlock);
898 }
899
900 /*
901 * In forwarding mode, if the destination address
902 * of the packet does not match any interface
903 * address, it maybe destined to the client device
904 */
905 SK_DF(SK_VERB_FSW_DP | SK_VERB_RX | SK_VERB_FLOW,
906 "Rx flow does not match interface address");
907 done:
908 return result;
909 }
910
911 static struct flow_entry *
rx_lookup_flow(struct nx_flowswitch * fsw,struct __kern_packet * pkt,struct flow_entry * prev_fe)912 rx_lookup_flow(struct nx_flowswitch *fsw, struct __kern_packet *pkt,
913 struct flow_entry *prev_fe)
914 {
915 struct flow_entry *fe;
916
917 fe = lookup_flow_with_pkt(fsw, pkt, true, prev_fe);
918 _FSW_INJECT_ERROR(2, fe, NULL, flow_entry_release, &fe);
919 if (fe == NULL) {
920 FSW_STATS_INC(FSW_STATS_RX_FLOW_NOT_FOUND);
921 return NULL;
922 }
923
924 if (__improbable(fe->fe_key.fk_mask == FKMASK_2TUPLE &&
925 fe->fe_flags & FLOWENTF_LISTENER) &&
926 !pkt_is_for_listener(fe, pkt)) {
927 FSW_STATS_INC(FSW_STATS_RX_PKT_NOT_LISTENER);
928 flow_entry_release(&fe);
929 return NULL;
930 }
931
932 if (__improbable(fe->fe_flags & FLOWENTF_TORN_DOWN)) {
933 FSW_STATS_INC(FSW_STATS_RX_FLOW_TORNDOWN);
934 SK_DF(SK_VERB_FSW_DP | SK_VERB_RX | SK_VERB_FLOW,
935 "Rx flow torn down");
936 flow_entry_release(&fe);
937 fe = NULL;
938 }
939
940 return fe;
941 }
942
943 static inline void
rx_flow_batch_packet(struct flow_entry_list * fes,struct flow_entry * fe,struct __kern_packet * pkt)944 rx_flow_batch_packet(struct flow_entry_list *fes, struct flow_entry *fe,
945 struct __kern_packet *pkt)
946 {
947 if (__improbable(pkt->pkt_flow_ip_is_frag)) {
948 fe->fe_rx_frag_count++;
949 }
950
951 /* KPKTQ_ENQUEUE_LIST is needed until frags become chained buflet */
952 if (KPKTQ_EMPTY(&fe->fe_rx_pktq)) {
953 ASSERT(KPKTQ_LEN(&fe->fe_rx_pktq) == 0);
954 TAILQ_INSERT_TAIL(fes, fe, fe_rx_link);
955 KPKTQ_ENQUEUE_LIST(&fe->fe_rx_pktq, pkt);
956 } else {
957 ASSERT(!TAILQ_EMPTY(fes));
958 KPKTQ_ENQUEUE_LIST(&fe->fe_rx_pktq, pkt);
959 flow_entry_release(&fe);
960 }
961 }
962
963 static void
tx_flow_batch_packet(struct flow_entry_list * fes,struct flow_entry * fe,struct __kern_packet * pkt)964 tx_flow_batch_packet(struct flow_entry_list *fes, struct flow_entry *fe,
965 struct __kern_packet *pkt)
966 {
967 /* record frag continuation */
968 if (__improbable(pkt->pkt_flow_ip_is_first_frag)) {
969 ASSERT(pkt->pkt_flow_ip_is_frag);
970 fe->fe_tx_is_cont_frag = true;
971 fe->fe_tx_frag_id = pkt->pkt_flow_ip_frag_id;
972 } else if (__probable(!pkt->pkt_flow_ip_is_frag)) {
973 fe->fe_tx_is_cont_frag = false;
974 fe->fe_tx_frag_id = 0;
975 }
976
977 if (KPKTQ_EMPTY(&fe->fe_tx_pktq)) {
978 ASSERT(KPKTQ_LEN(&fe->fe_tx_pktq) == 0);
979 TAILQ_INSERT_TAIL(fes, fe, fe_tx_link);
980 KPKTQ_ENQUEUE(&fe->fe_tx_pktq, pkt);
981 } else {
982 ASSERT(!TAILQ_EMPTY(fes));
983 KPKTQ_ENQUEUE(&fe->fe_tx_pktq, pkt);
984 flow_entry_release(&fe);
985 }
986 }
987
988 static inline void
fsw_rx_ring_dequeue_pktq(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,uint32_t n_pkts_max,struct pktq * pktq,uint32_t * n_bytes)989 fsw_rx_ring_dequeue_pktq(struct nx_flowswitch *fsw, struct __kern_channel_ring *r,
990 uint32_t n_pkts_max, struct pktq *pktq, uint32_t *n_bytes)
991 {
992 uint32_t n_pkts = 0;
993 slot_idx_t idx, idx_end;
994 idx = r->ckr_khead;
995 idx_end = r->ckr_rhead;
996
997 ASSERT(KPKTQ_EMPTY(pktq));
998 *n_bytes = 0;
999 for (; n_pkts < n_pkts_max && idx != idx_end;
1000 idx = SLOT_NEXT(idx, r->ckr_lim)) {
1001 struct __kern_slot_desc *ksd = KR_KSD(r, idx);
1002 struct __kern_packet *pkt = ksd->sd_pkt;
1003
1004 ASSERT(pkt->pkt_nextpkt == NULL);
1005 KR_SLOT_DETACH_METADATA(r, ksd);
1006
1007 _FSW_INJECT_ERROR(20, pkt->pkt_qum_qflags,
1008 pkt->pkt_qum_qflags | QUM_F_DROPPED, null_func);
1009 if (__improbable(((pkt->pkt_qum_qflags & QUM_F_DROPPED) != 0))
1010 || (pkt->pkt_length == 0)) {
1011 FSW_STATS_INC(FSW_STATS_DROP);
1012 pp_free_packet_single(pkt);
1013 continue;
1014 }
1015 n_pkts++;
1016 *n_bytes += pkt->pkt_length;
1017
1018 KPKTQ_ENQUEUE(pktq, pkt);
1019 }
1020 r->ckr_khead = idx;
1021 r->ckr_ktail = SLOT_PREV(idx, r->ckr_lim);
1022 }
1023
1024 /*
1025 * This is only for estimating how many packets each GSO packet will need.
1026 * The number does not need to be exact because any leftover packets allocated
1027 * will be freed.
1028 */
1029 static uint32_t
estimate_gso_pkts(struct __kern_packet * pkt)1030 estimate_gso_pkts(struct __kern_packet *pkt)
1031 {
1032 packet_tso_flags_t tso_flags;
1033 uint16_t mss;
1034 uint32_t n_pkts = 0, total_hlen = 0, total_len = 0;
1035
1036 tso_flags = pkt->pkt_csum_flags & PACKET_CSUM_TSO_FLAGS;
1037 mss = pkt->pkt_proto_seg_sz;
1038
1039 if (tso_flags == PACKET_TSO_IPV4) {
1040 total_hlen = sizeof(struct ip) + sizeof(struct tcphdr);
1041 } else if (tso_flags == PACKET_TSO_IPV6) {
1042 total_hlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
1043 }
1044 if (total_hlen != 0 && mss != 0) {
1045 total_len = pkt->pkt_length;
1046 n_pkts = (uint32_t)
1047 (SK_ROUNDUP((total_len - total_hlen), mss) / mss);
1048 }
1049 DTRACE_SKYWALK5(estimate__gso, packet_tso_flags_t, tso_flags,
1050 uint32_t, total_hlen, uint32_t, total_len, uint16_t, mss,
1051 uint32_t, n_pkts);
1052 return n_pkts;
1053 }
1054
1055 /*
1056 * This function retrieves a chain of packets of the same type only
1057 * (GSO or non-GSO).
1058 */
1059 static inline void
fsw_tx_ring_dequeue_pktq(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,uint32_t n_pkts_max,struct pktq * pktq,uint32_t * n_bytes,uint32_t * gso_pkts_estimate)1060 fsw_tx_ring_dequeue_pktq(struct nx_flowswitch *fsw,
1061 struct __kern_channel_ring *r, uint32_t n_pkts_max,
1062 struct pktq *pktq, uint32_t *n_bytes, uint32_t *gso_pkts_estimate)
1063 {
1064 uint32_t n_pkts = 0;
1065 slot_idx_t idx, idx_end;
1066 idx = r->ckr_khead;
1067 idx_end = r->ckr_rhead;
1068 struct nexus_vp_adapter *vpna = VPNA(KRNA(r));
1069 boolean_t gso_enabled, gso_required;
1070 uint32_t gso_pkts;
1071
1072 gso_enabled = (fsw->fsw_tso_mode == FSW_TSO_MODE_SW);
1073 ASSERT(KPKTQ_EMPTY(pktq));
1074 *n_bytes = 0;
1075 for (; n_pkts < n_pkts_max &&
1076 (!gso_enabled || fsw_gso_batch == 0 ||
1077 *gso_pkts_estimate < fsw_gso_batch) &&
1078 idx != idx_end; idx = SLOT_NEXT(idx, r->ckr_lim)) {
1079 struct __kern_slot_desc *ksd = KR_KSD(r, idx);
1080 struct __kern_packet *pkt = ksd->sd_pkt;
1081
1082 ASSERT(pkt->pkt_nextpkt == NULL);
1083
1084 _FSW_INJECT_ERROR(20, pkt->pkt_qum_qflags,
1085 pkt->pkt_qum_qflags | QUM_F_DROPPED, null_func);
1086 if (__improbable(((pkt->pkt_qum_qflags & QUM_F_DROPPED) != 0))
1087 || (pkt->pkt_length == 0)) {
1088 KR_SLOT_DETACH_METADATA(r, ksd);
1089 FSW_STATS_INC(FSW_STATS_DROP);
1090 pp_free_packet_single(pkt);
1091 continue;
1092 }
1093 if (gso_enabled) {
1094 gso_pkts = estimate_gso_pkts(pkt);
1095
1096 /*
1097 * We use the first packet to determine what
1098 * type the subsequent ones need to be (GSO or
1099 * non-GSO).
1100 */
1101 if (n_pkts == 0) {
1102 gso_required = (gso_pkts != 0);
1103 } else {
1104 if (gso_required != (gso_pkts != 0)) {
1105 break;
1106 }
1107 }
1108 *gso_pkts_estimate += gso_pkts;
1109 }
1110 KR_SLOT_DETACH_METADATA(r, ksd);
1111 if (NA_CHANNEL_EVENT_ATTACHED(&vpna->vpna_up)) {
1112 __packet_set_tx_nx_port(SK_PKT2PH(pkt),
1113 vpna->vpna_nx_port, vpna->vpna_gencnt);
1114 }
1115 n_pkts++;
1116 *n_bytes += pkt->pkt_length;
1117 KPKTQ_ENQUEUE(pktq, pkt);
1118 }
1119 r->ckr_khead = idx;
1120 r->ckr_ktail = SLOT_PREV(idx, r->ckr_lim);
1121 DTRACE_SKYWALK5(tx__ring__dequeue, struct nx_flowswitch *, fsw,
1122 ifnet_t, fsw->fsw_ifp, uint32_t, n_pkts, uint32_t, *n_bytes,
1123 uint32_t, *gso_pkts_estimate);
1124 }
1125
1126 static void
fsw_ring_enqueue_pktq(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,struct pktq * pktq)1127 fsw_ring_enqueue_pktq(struct nx_flowswitch *fsw, struct __kern_channel_ring *r,
1128 struct pktq *pktq)
1129 {
1130 #pragma unused(fsw)
1131 struct __kern_packet *pkt;
1132 struct __kern_quantum *kqum;
1133 uint32_t kr_space_avail = 0;
1134 uint32_t n, n_pkts = 0, n_bytes = 0;
1135 slot_idx_t idx = 0, idx_start = 0, idx_end = 0;
1136
1137 kr_enter(r, TRUE);
1138
1139 idx_start = r->ckr_ktail;
1140 kr_space_avail = kr_available_slots_rxring(r);
1141 _FSW_INJECT_ERROR(40, kr_space_avail, 0, null_func);
1142 n = MIN(kr_space_avail, KPKTQ_LEN(pktq));
1143 _FSW_INJECT_ERROR(41, n, 0, null_func);
1144 idx_end = SLOT_INCREMENT(idx_start, n, r->ckr_lim);
1145
1146 idx = idx_start;
1147 while (idx != idx_end) {
1148 KPKTQ_DEQUEUE(pktq, pkt);
1149 kqum = SK_PTR_ADDR_KQUM(pkt);
1150 kqum->qum_qflags |= QUM_F_FINALIZED;
1151 n_pkts++;
1152 n_bytes += pkt->pkt_length;
1153 KR_SLOT_ATTACH_METADATA(r, KR_KSD(r, idx), kqum);
1154 if (__improbable(pkt->pkt_trace_id != 0)) {
1155 KDBG(SK_KTRACE_PKT_RX_FSW | DBG_FUNC_END, pkt->pkt_trace_id);
1156 KDBG(SK_KTRACE_PKT_RX_CHN | DBG_FUNC_START, pkt->pkt_trace_id);
1157 }
1158 idx = SLOT_NEXT(idx, r->ckr_lim);
1159 }
1160
1161 kr_update_stats(r, n_pkts, n_bytes);
1162
1163 /*
1164 * ensure slot attachments are visible before updating the
1165 * tail pointer
1166 */
1167 os_atomic_thread_fence(seq_cst);
1168
1169 r->ckr_ktail = idx_end;
1170
1171 kr_exit(r);
1172
1173 r->ckr_na_notify(r, kernproc, NA_NOTEF_PUSH);
1174
1175 SK_DF(SK_VERB_FSW_DP | SK_VERB_RING, "%s enqueued %d pkts",
1176 r->ckr_name, n_pkts);
1177 }
1178
1179 static void
pkts_to_pktq(struct __kern_packet * pkts[],uint32_t n_pkts,struct pktq * pktq)1180 pkts_to_pktq(struct __kern_packet *pkts[], uint32_t n_pkts, struct pktq *pktq)
1181 {
1182 ASSERT(KPKTQ_EMPTY(pktq));
1183
1184 for (uint32_t i = 0; i < n_pkts; i++) {
1185 struct __kern_packet *pkt = pkts[i];
1186 ASSERT(pkt->pkt_nextpkt == NULL);
1187 KPKTQ_ENQUEUE(pktq, pkt);
1188 }
1189 }
1190
1191 /*
1192 * This function is modeled after nx_netif_host_grab_pkts() in nx_netif_host.c.
1193 */
1194 SK_NO_INLINE_ATTRIBUTE
1195 static void
convert_native_pktq_to_mbufs(struct nx_flowswitch * fsw,struct pktq * pktq,struct mbuf ** m_headp,struct mbuf ** m_tailp,uint32_t * cnt,uint32_t * bytes)1196 convert_native_pktq_to_mbufs(struct nx_flowswitch *fsw, struct pktq *pktq,
1197 struct mbuf **m_headp, struct mbuf **m_tailp, uint32_t *cnt, uint32_t *bytes)
1198 {
1199 uint32_t tot_cnt;
1200 unsigned int num_segs = 1;
1201 struct mbuf *mhead, *head = NULL, *tail = NULL, **tailp = &head;
1202 uint32_t mhead_cnt, mhead_bufsize;
1203 uint32_t mhead_waste = 0;
1204 uint32_t mcnt = 0, mbytes = 0;
1205 uint32_t largest, max_pkt_len;
1206 struct __kern_packet *pkt;
1207 struct kern_pbufpool *pp;
1208
1209 tot_cnt = KPKTQ_LEN(pktq);
1210 ASSERT(tot_cnt > 0);
1211 mhead_cnt = tot_cnt;
1212
1213 /*
1214 * Opportunistically batch-allocate the mbufs based on the largest
1215 * packet size we've seen in the recent past. Note that we reset
1216 * fe_rx_largest_size below if we notice that we're under-utilizing the
1217 * allocated buffers (thus disabling this batch allocation).
1218 */
1219 largest = *(volatile uint32_t*)&fsw->fsw_rx_largest_size; /* read once */
1220 if (__probable(largest != 0)) {
1221 if (largest <= MCLBYTES) {
1222 mhead = m_allocpacket_internal(&mhead_cnt, MCLBYTES,
1223 &num_segs, M_NOWAIT, 1, 0);
1224 mhead_bufsize = MCLBYTES;
1225 } else if (largest <= MBIGCLBYTES) {
1226 mhead = m_allocpacket_internal(&mhead_cnt, MBIGCLBYTES,
1227 &num_segs, M_NOWAIT, 1, 0);
1228 mhead_bufsize = MBIGCLBYTES;
1229 } else if (largest <= M16KCLBYTES) {
1230 mhead = m_allocpacket_internal(&mhead_cnt, M16KCLBYTES,
1231 &num_segs, M_NOWAIT, 1, 0);
1232 mhead_bufsize = M16KCLBYTES;
1233 } else if (largest <= M16KCLBYTES * 2) {
1234 num_segs = 2;
1235 mhead = m_allocpacket_internal(&mhead_cnt, M16KCLBYTES * 2,
1236 &num_segs, M_NOWAIT, 1, 0);
1237 mhead_bufsize = M16KCLBYTES * 2;
1238 } else {
1239 mhead = NULL;
1240 mhead_bufsize = mhead_cnt = 0;
1241 }
1242 } else {
1243 mhead = NULL;
1244 mhead_bufsize = mhead_cnt = 0;
1245 }
1246 DTRACE_SKYWALK4(bufstats, uint32_t, largest, uint32_t, mhead_bufsize,
1247 uint32_t, mhead_cnt, uint32_t, tot_cnt);
1248
1249 pp = __DECONST(struct kern_pbufpool *, KPKTQ_FIRST(pktq)->pkt_qum.qum_pp);
1250 max_pkt_len = PP_BUF_SIZE_DEF(pp) * pp->pp_max_frags;
1251
1252 KPKTQ_FOREACH(pkt, pktq) {
1253 uint32_t tot_len, len;
1254 uint16_t pad, llhlen, iphlen;
1255 boolean_t do_cksum_rx;
1256 struct mbuf *m;
1257 int error;
1258
1259 llhlen = pkt->pkt_l2_len;
1260 len = pkt->pkt_length;
1261 if (__improbable(len > max_pkt_len || llhlen > len)) {
1262 DTRACE_SKYWALK2(bad__len, struct nx_flowswitch *, fsw,
1263 struct __kern_packet *, pkt);
1264 FSW_STATS_INC(FSW_STATS_DROP);
1265 FSW_STATS_INC(FSW_STATS_RX_COPY_BAD_LEN);
1266 continue;
1267 }
1268 /* begin payload on 32-bit boundary; figure out the padding */
1269 pad = (uint16_t)P2ROUNDUP(llhlen, sizeof(uint32_t)) - llhlen;
1270 tot_len = pad + len;
1271
1272 /* remember largest packet size */
1273 if (__improbable(largest < tot_len)) {
1274 largest = MAX(tot_len, MCLBYTES);
1275 }
1276
1277 /*
1278 * If the above batch allocation returned partial
1279 * success, we try a blocking allocation here again.
1280 */
1281 m = mhead;
1282 if (__improbable(m == NULL || tot_len > mhead_bufsize)) {
1283 ASSERT(mhead != NULL || mhead_cnt == 0);
1284 num_segs = 1;
1285 if (tot_len > M16KCLBYTES) {
1286 num_segs = 0;
1287 }
1288 if ((error = mbuf_allocpacket(MBUF_DONTWAIT, tot_len,
1289 &num_segs, &m)) != 0) {
1290 DTRACE_SKYWALK2(bad__len,
1291 struct nx_flowswitch *, fsw,
1292 struct __kern_packet *, pkt);
1293 FSW_STATS_INC(FSW_STATS_DROP_NOMEM_MBUF);
1294 FSW_STATS_INC(FSW_STATS_DROP);
1295 continue;
1296 }
1297 } else {
1298 mhead = m->m_nextpkt;
1299 m->m_nextpkt = NULL;
1300 ASSERT(mhead_cnt != 0);
1301 --mhead_cnt;
1302
1303 /* check if we're underutilizing large buffers */
1304 if (__improbable(mhead_bufsize > MCLBYTES &&
1305 tot_len < (mhead_bufsize >> 1))) {
1306 ++mhead_waste;
1307 }
1308 /*
1309 * Clean up unused mbuf.
1310 * Ony need to do this when we pre-alloc 2x16K mbufs
1311 */
1312 if (__improbable(mhead_bufsize >= tot_len + M16KCLBYTES)) {
1313 ASSERT(mhead_bufsize == 2 * M16KCLBYTES);
1314 struct mbuf *m_extra = m->m_next;
1315 ASSERT(m_extra != NULL);
1316 ASSERT(m_extra->m_len == 0);
1317 ASSERT(M_SIZE(m_extra) == M16KCLBYTES);
1318 m->m_next = NULL;
1319 m_freem(m_extra);
1320 FSW_STATS_INC(FSW_STATS_RX_WASTED_16KMBUF);
1321 }
1322 }
1323 m->m_data += pad;
1324 m->m_pkthdr.pkt_hdr = mtod(m, uint8_t *);
1325
1326 /* don't include IP header from partial sum */
1327 if (__probable((pkt->pkt_qum_qflags &
1328 QUM_F_FLOW_CLASSIFIED) != 0)) {
1329 iphlen = pkt->pkt_flow_ip_hlen;
1330 do_cksum_rx = sk_cksum_rx;
1331 } else {
1332 iphlen = 0;
1333 do_cksum_rx = FALSE;
1334 }
1335
1336 fsw->fsw_pkt_copy_to_mbuf(NR_RX, SK_PKT2PH(pkt),
1337 pkt->pkt_headroom, m, 0, len, do_cksum_rx,
1338 llhlen + iphlen);
1339
1340 FSW_STATS_INC(FSW_STATS_RX_COPY_PKT2MBUF);
1341 if (do_cksum_rx) {
1342 FSW_STATS_INC(FSW_STATS_RX_COPY_SUM);
1343 }
1344 #if DEBUG || DEVELOPMENT
1345 if (__improbable(pkt_trailers > 0)) {
1346 (void) pkt_add_trailers_mbuf(m, llhlen + iphlen);
1347 }
1348 #endif /* DEBUG || DEVELOPMENT */
1349 m_adj(m, llhlen);
1350
1351 m->m_pkthdr.rcvif = fsw->fsw_ifp;
1352 if (__improbable((pkt->pkt_link_flags &
1353 PKT_LINKF_ETHFCS) != 0)) {
1354 m->m_flags |= M_HASFCS;
1355 }
1356 if (__improbable(pkt->pkt_pflags & PKT_F_WAKE_PKT)) {
1357 m->m_pkthdr.pkt_flags |= PKTF_WAKE_PKT;
1358 }
1359 ASSERT(m->m_nextpkt == NULL);
1360 tail = m;
1361 *tailp = m;
1362 tailp = &m->m_nextpkt;
1363 mcnt++;
1364 mbytes += m_pktlen(m);
1365 }
1366 /* free any leftovers */
1367 if (__improbable(mhead != NULL)) {
1368 DTRACE_SKYWALK1(mhead__leftover, uint32_t, mhead_cnt);
1369 ASSERT(mhead_cnt != 0);
1370 (void) m_freem_list(mhead);
1371 mhead = NULL;
1372 mhead_cnt = 0;
1373 }
1374
1375 /* reset if most packets (>50%) are smaller than our batch buffers */
1376 if (__improbable(mhead_waste > ((uint32_t)tot_cnt >> 1))) {
1377 DTRACE_SKYWALK4(mhead__waste, struct nx_flowswitch *, fsw,
1378 struct flow_entry *, NULL, uint32_t, mhead_waste,
1379 uint32_t, tot_cnt);
1380 largest = 0;
1381 }
1382
1383 if (largest != fsw->fsw_rx_largest_size) {
1384 os_atomic_store(&fsw->fsw_rx_largest_size, largest, release);
1385 }
1386
1387 pp_free_pktq(pktq);
1388 *m_headp = head;
1389 *m_tailp = tail;
1390 *cnt = mcnt;
1391 *bytes = mbytes;
1392 }
1393
1394 /*
1395 * This function only extracts the mbuf from the packet. The caller frees
1396 * the packet.
1397 */
1398 static inline struct mbuf *
convert_compat_pkt_to_mbuf(struct nx_flowswitch * fsw,struct __kern_packet * pkt)1399 convert_compat_pkt_to_mbuf(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
1400 {
1401 struct mbuf *m;
1402 struct pkthdr *mhdr;
1403 uint16_t llhlen;
1404
1405 m = pkt->pkt_mbuf;
1406 ASSERT(m != NULL);
1407
1408 llhlen = pkt->pkt_l2_len;
1409 if (llhlen > pkt->pkt_length) {
1410 m_freem(m);
1411 KPKT_CLEAR_MBUF_DATA(pkt);
1412 DTRACE_SKYWALK2(bad__len, struct nx_flowswitch *, fsw,
1413 struct __kern_packet *, pkt);
1414 FSW_STATS_INC(FSW_STATS_DROP);
1415 FSW_STATS_INC(FSW_STATS_RX_COPY_BAD_LEN);
1416 return NULL;
1417 }
1418 mhdr = &m->m_pkthdr;
1419 if ((mhdr->csum_flags & CSUM_DATA_VALID) == 0 &&
1420 PACKET_HAS_PARTIAL_CHECKSUM(pkt)) {
1421 mhdr->csum_flags &= ~CSUM_RX_FLAGS;
1422 mhdr->csum_flags |= (CSUM_DATA_VALID | CSUM_PARTIAL);
1423 mhdr->csum_rx_start = pkt->pkt_csum_rx_start_off;
1424 mhdr->csum_rx_val = pkt->pkt_csum_rx_value;
1425 }
1426 #if DEBUG || DEVELOPMENT
1427 uint32_t extra = 0;
1428 if (__improbable(pkt_trailers > 0)) {
1429 extra = pkt_add_trailers_mbuf(m, llhlen);
1430 }
1431 #endif /* DEBUG || DEVELOPMENT */
1432 m_adj(m, llhlen);
1433 ASSERT((uint32_t)m_pktlen(m) == ((pkt->pkt_length - llhlen) + extra));
1434 KPKT_CLEAR_MBUF_DATA(pkt);
1435 return m;
1436 }
1437
1438 SK_NO_INLINE_ATTRIBUTE
1439 static void
convert_compat_pktq_to_mbufs(struct nx_flowswitch * fsw,struct pktq * pktq,struct mbuf ** m_head,struct mbuf ** m_tail,uint32_t * cnt,uint32_t * bytes)1440 convert_compat_pktq_to_mbufs(struct nx_flowswitch *fsw, struct pktq *pktq,
1441 struct mbuf **m_head, struct mbuf **m_tail, uint32_t *cnt, uint32_t *bytes)
1442 {
1443 struct __kern_packet *pkt;
1444 struct mbuf *m, *head = NULL, *tail = NULL, **tailp = &head;
1445 uint32_t c = 0, b = 0;
1446
1447 KPKTQ_FOREACH(pkt, pktq) {
1448 m = convert_compat_pkt_to_mbuf(fsw, pkt);
1449 if (__improbable(m == NULL)) {
1450 continue;
1451 }
1452 tail = m;
1453 *tailp = m;
1454 tailp = &m->m_nextpkt;
1455 c++;
1456 b += m_pktlen(m);
1457 }
1458 pp_free_pktq(pktq);
1459 *m_head = head;
1460 *m_tail = tail;
1461 *cnt = c;
1462 *bytes = b;
1463 }
1464
1465 void
fsw_host_sendup(ifnet_t ifp,struct mbuf * m_head,struct mbuf * m_tail,uint32_t cnt,uint32_t bytes)1466 fsw_host_sendup(ifnet_t ifp, struct mbuf *m_head, struct mbuf *m_tail,
1467 uint32_t cnt, uint32_t bytes)
1468 {
1469 struct ifnet_stat_increment_param s;
1470
1471 bzero(&s, sizeof(s));
1472 s.packets_in = cnt;
1473 s.bytes_in = bytes;
1474 dlil_input_handler(ifp, m_head, m_tail, &s, FALSE, NULL);
1475 }
1476
1477 void
fsw_host_rx(struct nx_flowswitch * fsw,struct pktq * pktq)1478 fsw_host_rx(struct nx_flowswitch *fsw, struct pktq *pktq)
1479 {
1480 struct mbuf *m_head = NULL, *m_tail = NULL;
1481 uint32_t cnt = 0, bytes = 0;
1482 ifnet_fsw_rx_cb_t cb;
1483 void *cb_arg;
1484 boolean_t compat;
1485
1486 ASSERT(!KPKTQ_EMPTY(pktq));
1487 if (ifnet_get_flowswitch_rx_callback(fsw->fsw_ifp, &cb, &cb_arg) == 0) {
1488 ASSERT(cb != NULL);
1489 ASSERT(cb_arg != NULL);
1490 /* callback consumes packets */
1491 (*cb)(cb_arg, pktq);
1492 ifnet_release_flowswitch_rx_callback(fsw->fsw_ifp);
1493 return;
1494 }
1495
1496 /* All packets in the pktq must have the same type */
1497 compat = ((KPKTQ_FIRST(pktq)->pkt_pflags & PKT_F_MBUF_DATA) != 0);
1498 if (compat) {
1499 convert_compat_pktq_to_mbufs(fsw, pktq, &m_head, &m_tail, &cnt,
1500 &bytes);
1501 } else {
1502 convert_native_pktq_to_mbufs(fsw, pktq, &m_head, &m_tail, &cnt,
1503 &bytes);
1504 }
1505 if (__improbable(m_head == NULL)) {
1506 DTRACE_SKYWALK1(empty__head, struct nx_flowswitch *, fsw);
1507 return;
1508 }
1509 fsw_host_sendup(fsw->fsw_ifp, m_head, m_tail, cnt, bytes);
1510 }
1511
1512 void
fsw_ring_enqueue_tail_drop(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,struct pktq * pktq)1513 fsw_ring_enqueue_tail_drop(struct nx_flowswitch *fsw,
1514 struct __kern_channel_ring *r, struct pktq *pktq)
1515 {
1516 fsw_ring_enqueue_pktq(fsw, r, pktq);
1517 FSW_STATS_ADD(FSW_STATS_RX_DST_RING_FULL, KPKTQ_LEN(pktq));
1518 dp_drop_pktq(fsw, pktq);
1519 }
1520
1521 static struct nexus_adapter *
flow_get_na(struct nx_flowswitch * fsw,struct flow_entry * fe)1522 flow_get_na(struct nx_flowswitch *fsw, struct flow_entry *fe)
1523 {
1524 struct kern_nexus *nx = fsw->fsw_nx;
1525 struct nexus_adapter *na = NULL;
1526 nexus_port_t port = fe->fe_nx_port;
1527
1528 if (port == FSW_VP_DEV || port == FSW_VP_HOST) {
1529 SK_ERR("dev or host ports have no NA");
1530 return NULL;
1531 }
1532
1533 if (__improbable(!nx_port_is_valid(nx, port))) {
1534 SK_DF(SK_VERB_FSW_DP, "%s[%d] port no longer valid",
1535 if_name(fsw->fsw_ifp), port);
1536 return NULL;
1537 }
1538
1539 na = nx_port_get_na(nx, port);
1540 if (__improbable(na == NULL)) {
1541 FSW_STATS_INC(FSW_STATS_DST_NXPORT_INVALID);
1542 SK_DF(SK_VERB_FSW_DP, "%s[%d] NA no longer valid",
1543 if_name(fsw->fsw_ifp), port);
1544 return NULL;
1545 }
1546
1547 if (__improbable(!NA_IS_ACTIVE(na))) {
1548 FSW_STATS_INC(FSW_STATS_DST_NXPORT_INACTIVE);
1549 SK_DF(SK_VERB_FSW_DP, "%s[%d] NA no longer active",
1550 if_name(fsw->fsw_ifp), port);
1551 return NULL;
1552 }
1553
1554 if (__improbable(nx_port_is_defunct(nx, port))) {
1555 FSW_STATS_INC(FSW_STATS_DST_NXPORT_DEFUNCT);
1556 SK_DF(SK_VERB_FSW_DP, "%s[%d] NA defuncted",
1557 if_name(fsw->fsw_ifp), port);
1558 return NULL;
1559 }
1560
1561 return na;
1562 }
1563
1564 static inline struct __kern_channel_ring *
flow_get_ring(struct nx_flowswitch * fsw,struct flow_entry * fe,enum txrx txrx)1565 flow_get_ring(struct nx_flowswitch *fsw, struct flow_entry *fe, enum txrx txrx)
1566 {
1567 struct nexus_vp_adapter *na = NULL;
1568 struct __kern_channel_ring *r = NULL;
1569
1570 na = VPNA(flow_get_na(fsw, fe));
1571 if (__improbable(na == NULL)) {
1572 return NULL;
1573 }
1574
1575 switch (txrx) {
1576 case NR_RX:
1577 r = &na->vpna_up.na_rx_rings[0];
1578 break;
1579 case NR_TX:
1580 r = &na->vpna_up.na_tx_rings[0];
1581 break;
1582 default:
1583 __builtin_unreachable();
1584 VERIFY(0);
1585 }
1586
1587 if (__improbable(KR_DROP(r))) {
1588 FSW_STATS_INC(FSW_STATS_DST_RING_DROPMODE);
1589 SK_DF(SK_VERB_FSW_DP | SK_VERB_RING, "r %0xllx %s drop mode",
1590 r->ckr_name, SK_KVA(r));
1591 return NULL;
1592 }
1593
1594 ASSERT(KRNA(r)->na_md_type == NEXUS_META_TYPE_PACKET);
1595
1596 #if (DEVELOPMENT || DEBUG)
1597 if (r != NULL) {
1598 _FSW_INJECT_ERROR(4, r, NULL, null_func);
1599 }
1600 #endif /* DEVELOPMENT || DEBUG */
1601
1602 return r;
1603 }
1604
1605 struct __kern_channel_ring *
fsw_flow_get_rx_ring(struct nx_flowswitch * fsw,struct flow_entry * fe)1606 fsw_flow_get_rx_ring(struct nx_flowswitch *fsw, struct flow_entry *fe)
1607 {
1608 return flow_get_ring(fsw, fe, NR_RX);
1609 }
1610
1611 static inline struct __kern_channel_ring *
fsw_flow_get_tx_ring(struct nx_flowswitch * fsw,struct flow_entry * fe)1612 fsw_flow_get_tx_ring(struct nx_flowswitch *fsw, struct flow_entry *fe)
1613 {
1614 return flow_get_ring(fsw, fe, NR_TX);
1615 }
1616
1617 static bool
dp_flow_route_process(struct nx_flowswitch * fsw,struct flow_entry * fe)1618 dp_flow_route_process(struct nx_flowswitch *fsw, struct flow_entry *fe)
1619 {
1620 struct flow_route *fr = fe->fe_route;
1621 struct ifnet *ifp = fsw->fsw_ifp;
1622
1623 if (__improbable(!(fe->fe_flags & FLOWENTF_NONVIABLE) &&
1624 !fe->fe_want_nonviable && (fe->fe_key.fk_mask & FKMASK_SRC) &&
1625 fe->fe_laddr_gencnt != ifp->if_nx_flowswitch.if_fsw_ipaddr_gencnt &&
1626 !flow_route_key_validate(&fe->fe_key, ifp, &fe->fe_laddr_gencnt))) {
1627 /*
1628 * The source address is no longer around; we want this
1629 * flow to be nonviable, but that requires holding the lock
1630 * as writer (which isn't the case now.) Indicate that
1631 * we need to finalize the nonviable later down below.
1632 *
1633 * We also request that the flow route be re-configured,
1634 * if this is a connected mode flow.
1635 *
1636 */
1637 if (!(fe->fe_flags & FLOWENTF_NONVIABLE)) {
1638 /*
1639 * fsw_pending_nonviable is a hint for reaper thread;
1640 * due to the fact that setting fe_want_nonviable and
1641 * incrementing fsw_pending_nonviable counter is not
1642 * atomic, let the increment happen first, and the
1643 * thread losing the CAS does decrement.
1644 */
1645 os_atomic_inc(&fsw->fsw_pending_nonviable, relaxed);
1646 if (os_atomic_cmpxchg(&fe->fe_want_nonviable, 0, 1, acq_rel)) {
1647 fsw_reap_sched(fsw);
1648 } else {
1649 os_atomic_dec(&fsw->fsw_pending_nonviable, relaxed);
1650 }
1651 }
1652 if (fr != NULL) {
1653 os_atomic_inc(&fr->fr_want_configure, relaxed);
1654 }
1655 }
1656
1657 /* if flow was (or is going to be) marked as nonviable, drop it */
1658 if (__improbable(fe->fe_want_nonviable ||
1659 (fe->fe_flags & FLOWENTF_NONVIABLE) != 0)) {
1660 SK_DF(SK_VERB_FSW_DP | SK_VERB_FLOW, "flow 0x%llx non-viable",
1661 SK_KVA(fe));
1662 return false;
1663 }
1664 return true;
1665 }
1666
1667 bool
dp_flow_rx_route_process(struct nx_flowswitch * fsw,struct flow_entry * fe)1668 dp_flow_rx_route_process(struct nx_flowswitch *fsw, struct flow_entry *fe)
1669 {
1670 bool okay;
1671 okay = dp_flow_route_process(fsw, fe);
1672 #if (DEVELOPMENT || DEBUG)
1673 if (okay) {
1674 _FSW_INJECT_ERROR(5, okay, false, null_func);
1675 }
1676 #endif /* DEVELOPMENT || DEBUG */
1677
1678 return okay;
1679 }
1680
1681 void
dp_flow_rx_process(struct nx_flowswitch * fsw,struct flow_entry * fe,uint32_t flags)1682 dp_flow_rx_process(struct nx_flowswitch *fsw, struct flow_entry *fe,
1683 uint32_t flags)
1684 {
1685 #pragma unused(flags)
1686 struct pktq dpkts; /* dst pool alloc'ed packets */
1687 struct pktq disposed_pkts; /* done src packets */
1688 struct pktq dropped_pkts; /* dropped src packets */
1689 struct pktq transferred_pkts; /* dst packet ready for ring */
1690 struct __kern_packet *pkt, *tpkt;
1691 struct kern_pbufpool *dpp;
1692 uint32_t n_pkts = KPKTQ_LEN(&fe->fe_rx_pktq);
1693 uint64_t buf_array[RX_BUFLET_BATCH_COUNT];
1694 uint16_t buf_array_iter = 0;
1695 uint32_t cnt, buf_cnt = 0;
1696 int err;
1697
1698 KPKTQ_INIT(&dpkts);
1699 KPKTQ_INIT(&dropped_pkts);
1700 KPKTQ_INIT(&disposed_pkts);
1701 KPKTQ_INIT(&transferred_pkts);
1702
1703 if (__improbable(!dp_flow_rx_route_process(fsw, fe))) {
1704 SK_ERR("Rx route bad");
1705 fsw_snoop_and_dequeue(fe, &dropped_pkts, true);
1706 FSW_STATS_ADD(FSW_STATS_RX_FLOW_NONVIABLE, n_pkts);
1707 goto done;
1708 }
1709
1710 if (fe->fe_nx_port == FSW_VP_HOST) {
1711 /*
1712 * The host ring does not exist anymore so we can't take
1713 * the enqueue path below. This path should only be hit
1714 * for the rare tcp fragmentation case.
1715 */
1716 fsw_host_rx(fsw, &fe->fe_rx_pktq);
1717 return;
1718 }
1719
1720 /* find the ring */
1721 struct __kern_channel_ring *r;
1722 r = fsw_flow_get_rx_ring(fsw, fe);
1723 if (__improbable(r == NULL)) {
1724 fsw_snoop_and_dequeue(fe, &dropped_pkts, true);
1725 goto done;
1726 }
1727
1728 /* snoop before L2 is stripped */
1729 if (__improbable(pktap_total_tap_count != 0)) {
1730 fsw_snoop(fsw, fe, true);
1731 }
1732
1733 dpp = r->ckr_pp;
1734 /* batch allocate enough packets */
1735 err = pp_alloc_pktq(dpp, 1, &dpkts, n_pkts, NULL, NULL,
1736 SKMEM_NOSLEEP);
1737 if (__improbable(err == ENOMEM)) {
1738 ASSERT(KPKTQ_EMPTY(&dpkts));
1739 KPKTQ_CONCAT(&dropped_pkts, &fe->fe_rx_pktq);
1740 FSW_STATS_ADD(FSW_STATS_DROP_NOMEM_PKT, n_pkts);
1741 SK_ERR("failed to alloc %u pkts for kr %s, 0x%llu", n_pkts,
1742 r->ckr_name, SK_KVA(r));
1743 goto done;
1744 }
1745
1746 /*
1747 * estimate total number of buflets for the packet chain.
1748 */
1749 cnt = howmany(fe->fe_rx_pktq_bytes, PP_BUF_SIZE_DEF(dpp));
1750 if (cnt > n_pkts) {
1751 ASSERT(dpp->pp_max_frags > 1);
1752 cnt -= n_pkts;
1753 buf_cnt = MIN(RX_BUFLET_BATCH_COUNT, cnt);
1754 err = pp_alloc_buflet_batch(dpp, buf_array, &buf_cnt,
1755 SKMEM_NOSLEEP, false);
1756 if (__improbable(buf_cnt == 0)) {
1757 KPKTQ_CONCAT(&dropped_pkts, &fe->fe_rx_pktq);
1758 FSW_STATS_ADD(FSW_STATS_DROP_NOMEM_PKT, n_pkts);
1759 SK_ERR("failed to alloc %d buflets (err %d) for kr %s, "
1760 "0x%llu", cnt, err, r->ckr_name, SK_KVA(r));
1761 goto done;
1762 }
1763 err = 0;
1764 }
1765
1766 /* extra processing for user flow */
1767 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_rx_pktq, tpkt) {
1768 err = 0;
1769 KPKTQ_REMOVE(&fe->fe_rx_pktq, pkt);
1770 if (fe->fe_rx_pktq_bytes > pkt->pkt_flow_ulen) {
1771 fe->fe_rx_pktq_bytes -= pkt->pkt_flow_ulen;
1772 } else {
1773 fe->fe_rx_pktq_bytes = 0;
1774 }
1775 err = flow_pkt_track(fe, pkt, true);
1776 _FSW_INJECT_ERROR(33, err, EPROTO, null_func);
1777 if (__improbable(err != 0)) {
1778 SK_ERR("flow_pkt_track failed (err %d)", err);
1779 FSW_STATS_INC(FSW_STATS_RX_FLOW_TRACK_ERR);
1780 /* if need to trigger RST */
1781 if (err == ENETRESET) {
1782 flow_track_abort_tcp(fe, pkt, NULL);
1783 }
1784 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
1785 continue;
1786 }
1787
1788 /* transfer to dpkt */
1789 if (pkt->pkt_qum.qum_pp != dpp) {
1790 struct __kern_buflet *bprev, *bnew;
1791 struct __kern_packet *dpkt = NULL;
1792 uint32_t n_bufs, i;
1793
1794 KPKTQ_DEQUEUE(&dpkts, dpkt);
1795 if (__improbable(dpkt == NULL)) {
1796 FSW_STATS_INC(FSW_STATS_DROP_NOMEM_PKT);
1797 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
1798 continue;
1799 }
1800 n_bufs = howmany(pkt->pkt_length, PP_BUF_SIZE_DEF(dpp));
1801 n_bufs--;
1802 for (i = 0; i < n_bufs; i++) {
1803 if (__improbable(buf_cnt == 0)) {
1804 ASSERT(dpp->pp_max_frags > 1);
1805 buf_array_iter = 0;
1806 cnt = howmany(fe->fe_rx_pktq_bytes,
1807 PP_BUF_SIZE_DEF(dpp));
1808 n_pkts = KPKTQ_LEN(&fe->fe_rx_pktq);
1809 if (cnt >= n_pkts) {
1810 cnt -= n_pkts;
1811 } else {
1812 cnt = 0;
1813 }
1814 cnt += (n_bufs - i);
1815 buf_cnt = MIN(RX_BUFLET_BATCH_COUNT,
1816 cnt);
1817 cnt = buf_cnt;
1818 err = pp_alloc_buflet_batch(dpp,
1819 buf_array, &buf_cnt,
1820 SKMEM_NOSLEEP, false);
1821 if (__improbable(buf_cnt == 0)) {
1822 FSW_STATS_INC(FSW_STATS_DROP_NOMEM_PKT);
1823 KPKTQ_ENQUEUE(&dropped_pkts,
1824 pkt);
1825 pkt = NULL;
1826 pp_free_packet_single(dpkt);
1827 dpkt = NULL;
1828 SK_ERR("failed to alloc %d "
1829 "buflets (err %d) for "
1830 "kr %s, 0x%llu", cnt, err,
1831 r->ckr_name, SK_KVA(r));
1832 break;
1833 }
1834 err = 0;
1835 }
1836 ASSERT(buf_cnt != 0);
1837 if (i == 0) {
1838 PKT_GET_FIRST_BUFLET(dpkt, 1, bprev);
1839 }
1840 bnew = (kern_buflet_t)buf_array[buf_array_iter];
1841 buf_array[buf_array_iter] = 0;
1842 buf_array_iter++;
1843 buf_cnt--;
1844 VERIFY(kern_packet_add_buflet(SK_PKT2PH(dpkt),
1845 bprev, bnew) == 0);
1846 bprev = bnew;
1847 }
1848 if (__improbable(err != 0)) {
1849 continue;
1850 }
1851 err = copy_packet_from_dev(fsw, pkt, dpkt);
1852 _FSW_INJECT_ERROR(43, err, EINVAL, null_func);
1853 if (__improbable(err != 0)) {
1854 SK_ERR("copy packet failed (err %d)", err);
1855 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
1856 pp_free_packet_single(dpkt);
1857 dpkt = NULL;
1858 continue;
1859 }
1860 KPKTQ_ENQUEUE(&disposed_pkts, pkt);
1861 pkt = dpkt;
1862 }
1863 _UUID_COPY(pkt->pkt_flow_id, fe->fe_uuid);
1864 _UUID_COPY(pkt->pkt_policy_euuid, fe->fe_eproc_uuid);
1865 pkt->pkt_policy_id = fe->fe_policy_id;
1866 pkt->pkt_transport_protocol = fe->fe_transport_protocol;
1867 if (pkt->pkt_bufs_cnt > 1) {
1868 pkt->pkt_aggr_type = PKT_AGGR_SINGLE_IP;
1869 pkt->pkt_seg_cnt = 1;
1870 }
1871 KPKTQ_ENQUEUE(&transferred_pkts, pkt);
1872 }
1873 KPKTQ_FINI(&fe->fe_rx_pktq);
1874 KPKTQ_CONCAT(&fe->fe_rx_pktq, &transferred_pkts);
1875 KPKTQ_FINI(&transferred_pkts);
1876
1877 fsw_ring_enqueue_tail_drop(fsw, r, &fe->fe_rx_pktq);
1878
1879 done:
1880 /* Free unused buflets */
1881 while (buf_cnt > 0) {
1882 pp_free_buflet(dpp, (kern_buflet_t)(buf_array[buf_array_iter]));
1883 buf_array[buf_array_iter] = 0;
1884 buf_array_iter++;
1885 buf_cnt--;
1886 }
1887 dp_free_pktq(fsw, &dpkts);
1888 dp_free_pktq(fsw, &disposed_pkts);
1889 dp_drop_pktq(fsw, &dropped_pkts);
1890 }
1891
1892 static inline void
rx_flow_process(struct nx_flowswitch * fsw,struct flow_entry * fe,uint32_t flags)1893 rx_flow_process(struct nx_flowswitch *fsw, struct flow_entry *fe,
1894 uint32_t flags)
1895 {
1896 ASSERT(!KPKTQ_EMPTY(&fe->fe_rx_pktq));
1897 ASSERT(KPKTQ_LEN(&fe->fe_rx_pktq) != 0);
1898
1899 SK_DF(SK_VERB_FSW_DP | SK_VERB_RX, "Rx %d pkts for fe %p port %d",
1900 KPKTQ_LEN(&fe->fe_rx_pktq), fe, fe->fe_nx_port);
1901
1902 /* flow related processing (default, agg, fpd, etc.) */
1903 fe->fe_rx_process(fsw, fe, flags);
1904
1905 if (__improbable(fe->fe_want_withdraw)) {
1906 fsw_reap_sched(fsw);
1907 }
1908
1909 KPKTQ_FINI(&fe->fe_rx_pktq);
1910 }
1911
1912 static inline void
dp_rx_process_wake_packet(struct nx_flowswitch * fsw,struct __kern_packet * pkt)1913 dp_rx_process_wake_packet(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
1914 {
1915 /*
1916 * We only care about wake packets of flows that belong the flow switch
1917 * as wake packets for the host stack are handled by the host input
1918 * function
1919 */
1920 #if (DEBUG || DEVELOPMENT)
1921 if (__improbable(fsw->fsw_ifp->if_xflags & IFXF_MARK_WAKE_PKT)) {
1922 /*
1923 * This is a one shot command
1924 */
1925 fsw->fsw_ifp->if_xflags &= ~IFXF_MARK_WAKE_PKT;
1926
1927 pkt->pkt_pflags |= PKT_F_WAKE_PKT;
1928 }
1929 #endif /* (DEBUG || DEVELOPMENT) */
1930 if (__improbable(pkt->pkt_pflags & PKT_F_WAKE_PKT)) {
1931 if_ports_used_match_pkt(fsw->fsw_ifp, pkt);
1932 }
1933 }
1934
1935 static void
_fsw_receive_locked(struct nx_flowswitch * fsw,struct pktq * pktq)1936 _fsw_receive_locked(struct nx_flowswitch *fsw, struct pktq *pktq)
1937 {
1938 struct __kern_packet *pkt, *tpkt;
1939 struct flow_entry_list fes = TAILQ_HEAD_INITIALIZER(fes);
1940 struct flow_entry *fe, *prev_fe;
1941 sa_family_t af;
1942 struct pktq host_pkts, dropped_pkts;
1943 int err;
1944
1945 KPKTQ_INIT(&host_pkts);
1946 KPKTQ_INIT(&dropped_pkts);
1947
1948 if (__improbable(FSW_QUIESCED(fsw))) {
1949 DTRACE_SKYWALK1(rx__quiesced, struct nx_flowswitch *, fsw);
1950 KPKTQ_CONCAT(&dropped_pkts, pktq);
1951 goto done;
1952 }
1953 if (__improbable(fsw->fsw_demux == NULL)) {
1954 KPKTQ_CONCAT(&dropped_pkts, pktq);
1955 goto done;
1956 }
1957
1958 prev_fe = NULL;
1959 KPKTQ_FOREACH_SAFE(pkt, pktq, tpkt) {
1960 if (__probable(tpkt)) {
1961 void *baddr;
1962 MD_BUFLET_ADDR_ABS_PKT(tpkt, baddr);
1963 SK_PREFETCH(baddr, 0);
1964 /* prefetch L3 and L4 flow structs */
1965 SK_PREFETCHW(tpkt->pkt_flow, 0);
1966 SK_PREFETCHW(tpkt->pkt_flow, 128);
1967 }
1968
1969 KPKTQ_REMOVE(pktq, pkt);
1970
1971 pkt = rx_prepare_packet(fsw, pkt);
1972
1973 af = fsw->fsw_demux(fsw, pkt);
1974 if (__improbable(af == AF_UNSPEC)) {
1975 KPKTQ_ENQUEUE(&host_pkts, pkt);
1976 continue;
1977 }
1978
1979 err = flow_pkt_classify(pkt, fsw->fsw_ifp, af, TRUE);
1980 _FSW_INJECT_ERROR(1, err, ENXIO, null_func);
1981 if (__improbable(err != 0)) {
1982 FSW_STATS_INC(FSW_STATS_RX_FLOW_EXTRACT_ERR);
1983 KPKTQ_ENQUEUE(&host_pkts, pkt);
1984 continue;
1985 }
1986
1987 if (__improbable(pkt->pkt_flow_ip_is_frag)) {
1988 pkt = rx_process_ip_frag(fsw, pkt);
1989 if (pkt == NULL) {
1990 continue;
1991 }
1992 }
1993
1994 prev_fe = fe = rx_lookup_flow(fsw, pkt, prev_fe);
1995 if (__improbable(fe == NULL)) {
1996 KPKTQ_ENQUEUE_LIST(&host_pkts, pkt);
1997 continue;
1998 }
1999
2000 fe->fe_rx_pktq_bytes += pkt->pkt_flow_ulen;
2001
2002 dp_rx_process_wake_packet(fsw, pkt);
2003
2004 rx_flow_batch_packet(&fes, fe, pkt);
2005 prev_fe = fe;
2006 }
2007
2008 struct flow_entry *tfe = NULL;
2009 TAILQ_FOREACH_SAFE(fe, &fes, fe_rx_link, tfe) {
2010 rx_flow_process(fsw, fe, 0);
2011 TAILQ_REMOVE(&fes, fe, fe_rx_link);
2012 fe->fe_rx_pktq_bytes = 0;
2013 fe->fe_rx_frag_count = 0;
2014 flow_entry_release(&fe);
2015 }
2016
2017 if (!KPKTQ_EMPTY(&host_pkts)) {
2018 fsw_host_rx(fsw, &host_pkts);
2019 }
2020
2021 done:
2022 dp_drop_pktq(fsw, &dropped_pkts);
2023 }
2024
2025 #if (DEVELOPMENT || DEBUG)
2026 static void
fsw_rps_rx(struct nx_flowswitch * fsw,uint32_t id,struct __kern_packet * pkt)2027 fsw_rps_rx(struct nx_flowswitch *fsw, uint32_t id,
2028 struct __kern_packet *pkt)
2029 {
2030 struct fsw_rps_thread *frt = &fsw->fsw_rps_threads[id];
2031
2032 lck_mtx_lock_spin(&frt->frt_lock);
2033 KPKTQ_ENQUEUE(&frt->frt_pktq, pkt);
2034 lck_mtx_unlock(&frt->frt_lock);
2035 }
2036
2037 static void
fsw_rps_thread_schedule(struct nx_flowswitch * fsw,uint32_t id)2038 fsw_rps_thread_schedule(struct nx_flowswitch *fsw, uint32_t id)
2039 {
2040 struct fsw_rps_thread *frt = &fsw->fsw_rps_threads[id];
2041
2042 ASSERT(frt->frt_thread != THREAD_NULL);
2043 lck_mtx_lock_spin(&frt->frt_lock);
2044 ASSERT(!(frt->frt_flags & (FRT_TERMINATING | FRT_TERMINATED)));
2045
2046 frt->frt_requests++;
2047 if (!(frt->frt_flags & FRT_RUNNING)) {
2048 thread_wakeup((caddr_t)frt);
2049 }
2050 lck_mtx_unlock(&frt->frt_lock);
2051 }
2052
2053 __attribute__((noreturn))
2054 static void
fsw_rps_thread_cont(void * v,wait_result_t w)2055 fsw_rps_thread_cont(void *v, wait_result_t w)
2056 {
2057 struct fsw_rps_thread *frt = v;
2058 struct nx_flowswitch *fsw = frt->frt_fsw;
2059
2060 lck_mtx_lock(&frt->frt_lock);
2061 if (__improbable(w == THREAD_INTERRUPTIBLE ||
2062 (frt->frt_flags & FRT_TERMINATING) != 0)) {
2063 goto terminate;
2064 }
2065 if (KPKTQ_EMPTY(&frt->frt_pktq)) {
2066 goto done;
2067 }
2068 frt->frt_flags |= FRT_RUNNING;
2069
2070 for (;;) {
2071 uint32_t requests = frt->frt_requests;
2072 struct pktq pkts;
2073
2074 KPKTQ_INIT(&pkts);
2075 KPKTQ_CONCAT(&pkts, &frt->frt_pktq);
2076 lck_mtx_unlock(&frt->frt_lock);
2077
2078 sk_protect_t protect;
2079 protect = sk_sync_protect();
2080 FSW_RLOCK(fsw);
2081 _fsw_receive_locked(fsw, &pkts);
2082 FSW_RUNLOCK(fsw);
2083 sk_sync_unprotect(protect);
2084
2085 lck_mtx_lock(&frt->frt_lock);
2086 if ((frt->frt_flags & FRT_TERMINATING) != 0 ||
2087 requests == frt->frt_requests) {
2088 frt->frt_requests = 0;
2089 break;
2090 }
2091 }
2092
2093 done:
2094 lck_mtx_unlock(&frt->frt_lock);
2095 if (!(frt->frt_flags & FRT_TERMINATING)) {
2096 frt->frt_flags &= ~FRT_RUNNING;
2097 assert_wait(frt, THREAD_UNINT);
2098 thread_block_parameter(fsw_rps_thread_cont, frt);
2099 __builtin_unreachable();
2100 } else {
2101 terminate:
2102 LCK_MTX_ASSERT(&frt->frt_lock, LCK_MTX_ASSERT_OWNED);
2103 frt->frt_flags &= ~(FRT_RUNNING | FRT_TERMINATING);
2104 frt->frt_flags |= FRT_TERMINATED;
2105
2106 if (frt->frt_flags & FRT_TERMINATEBLOCK) {
2107 thread_wakeup((caddr_t)&frt);
2108 }
2109 lck_mtx_unlock(&frt->frt_lock);
2110
2111 SK_D("fsw_rx_%s_%d terminated", if_name(fsw->fsw_ifp),
2112 frt->frt_idx);
2113
2114 /* for the extra refcnt from kernel_thread_start() */
2115 thread_deallocate(current_thread());
2116 /* this is the end */
2117 thread_terminate(current_thread());
2118 /* NOTREACHED */
2119 __builtin_unreachable();
2120 }
2121
2122 /* must never get here */
2123 VERIFY(0);
2124 /* NOTREACHED */
2125 __builtin_unreachable();
2126 }
2127
2128 __attribute__((noreturn))
2129 static void
fsw_rps_thread_func(void * v,wait_result_t w)2130 fsw_rps_thread_func(void *v, wait_result_t w)
2131 {
2132 #pragma unused(w)
2133 struct fsw_rps_thread *frt = v;
2134 struct nx_flowswitch *fsw = frt->frt_fsw;
2135
2136 char thread_name[MAXTHREADNAMESIZE];
2137 bzero(thread_name, sizeof(thread_name));
2138 (void) snprintf(thread_name, sizeof(thread_name), "fsw_rx_%s_%d",
2139 if_name(fsw->fsw_ifp), frt->frt_idx);
2140 thread_set_thread_name(frt->frt_thread, thread_name);
2141 SK_D("%s spawned", thread_name);
2142
2143 net_thread_marks_push(NET_THREAD_SYNC_RX);
2144 assert_wait(frt, THREAD_UNINT);
2145 (void) thread_block_parameter(fsw_rps_thread_cont, frt);
2146
2147 __builtin_unreachable();
2148 }
2149
2150 static void
fsw_rps_thread_join(struct nx_flowswitch * fsw,uint32_t i)2151 fsw_rps_thread_join(struct nx_flowswitch *fsw, uint32_t i)
2152 {
2153 struct fsw_rps_thread *frt = &fsw->fsw_rps_threads[i];
2154 uint64_t f = (1 * NSEC_PER_MSEC);
2155 uint64_t s = (1000 * NSEC_PER_SEC);
2156 uint32_t c = 0;
2157
2158 lck_mtx_lock(&frt->frt_lock);
2159 frt->frt_flags |= FRT_TERMINATING;
2160
2161 while (!(frt->frt_flags & FRT_TERMINATED)) {
2162 uint64_t t = 0;
2163 nanoseconds_to_absolutetime((c++ == 0) ? f : s, &t);
2164 clock_absolutetime_interval_to_deadline(t, &t);
2165 ASSERT(t != 0);
2166
2167 frt->frt_flags |= FRT_TERMINATEBLOCK;
2168 if (!(frt->frt_flags & FRT_RUNNING)) {
2169 thread_wakeup_one((caddr_t)frt);
2170 }
2171 (void) assert_wait_deadline(&frt->frt_thread, THREAD_UNINT, t);
2172 lck_mtx_unlock(&frt->frt_lock);
2173 thread_block(THREAD_CONTINUE_NULL);
2174 lck_mtx_lock(&frt->frt_lock);
2175 frt->frt_flags &= ~FRT_TERMINATEBLOCK;
2176 }
2177 ASSERT(frt->frt_flags & FRT_TERMINATED);
2178 lck_mtx_unlock(&frt->frt_lock);
2179 frt->frt_thread = THREAD_NULL;
2180 }
2181
2182 static void
fsw_rps_thread_spawn(struct nx_flowswitch * fsw,uint32_t i)2183 fsw_rps_thread_spawn(struct nx_flowswitch *fsw, uint32_t i)
2184 {
2185 kern_return_t error;
2186 struct fsw_rps_thread *frt = &fsw->fsw_rps_threads[i];
2187 lck_mtx_init(&frt->frt_lock, &nexus_lock_group, &nexus_lock_attr);
2188 frt->frt_idx = i;
2189 frt->frt_fsw = fsw;
2190 error = kernel_thread_start(fsw_rps_thread_func, frt, &frt->frt_thread);
2191 ASSERT(!error);
2192 KPKTQ_INIT(&frt->frt_pktq);
2193 }
2194
2195 int
fsw_rps_set_nthreads(struct nx_flowswitch * fsw,uint32_t n)2196 fsw_rps_set_nthreads(struct nx_flowswitch* fsw, uint32_t n)
2197 {
2198 if (n > FSW_RPS_MAX_NTHREADS) {
2199 SK_ERR("rps nthreads %d, max %d", n, FSW_RPS_MAX_NTHREADS);
2200 return EINVAL;
2201 }
2202
2203 FSW_WLOCK(fsw);
2204 if (n < fsw->fsw_rps_nthreads) {
2205 for (uint32_t i = n; i < fsw->fsw_rps_nthreads; i++) {
2206 fsw_rps_thread_join(fsw, i);
2207 }
2208 fsw->fsw_rps_threads = krealloc_type(struct fsw_rps_thread,
2209 fsw->fsw_rps_nthreads, n, fsw->fsw_rps_threads,
2210 Z_WAITOK | Z_ZERO | Z_NOFAIL);
2211 } else if (n > fsw->fsw_rps_nthreads) {
2212 fsw->fsw_rps_threads = krealloc_type(struct fsw_rps_thread,
2213 fsw->fsw_rps_nthreads, n, fsw->fsw_rps_threads,
2214 Z_WAITOK | Z_ZERO | Z_NOFAIL);
2215 for (uint32_t i = fsw->fsw_rps_nthreads; i < n; i++) {
2216 fsw_rps_thread_spawn(fsw, i);
2217 }
2218 }
2219 fsw->fsw_rps_nthreads = n;
2220 FSW_WUNLOCK(fsw);
2221 return 0;
2222 }
2223
2224 static uint32_t
get_rps_id(struct nx_flowswitch * fsw,struct __kern_packet * pkt)2225 get_rps_id(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
2226 {
2227 sa_family_t af = fsw->fsw_demux(fsw, pkt);
2228 if (__improbable(af == AF_UNSPEC)) {
2229 return 0;
2230 }
2231
2232 flow_pkt_classify(pkt, fsw->fsw_ifp, af, true);
2233
2234 if (__improbable((pkt->pkt_qum_qflags &
2235 QUM_F_FLOW_CLASSIFIED) == 0)) {
2236 return 0;
2237 }
2238
2239 struct flow_key key;
2240 flow_pkt2key(pkt, true, &key);
2241 key.fk_mask = FKMASK_5TUPLE;
2242
2243 uint32_t id = flow_key_hash(&key) % fsw->fsw_rps_nthreads;
2244
2245 return id;
2246 }
2247
2248 #endif /* !DEVELOPMENT && !DEBUG */
2249
2250 void
fsw_receive(struct nx_flowswitch * fsw,struct pktq * pktq)2251 fsw_receive(struct nx_flowswitch *fsw, struct pktq *pktq)
2252 {
2253 FSW_RLOCK(fsw);
2254 #if (DEVELOPMENT || DEBUG)
2255 if (fsw->fsw_rps_nthreads != 0) {
2256 struct __kern_packet *pkt, *tpkt;
2257 bitmap_t map = 0;
2258
2259 _CASSERT(BITMAP_LEN(FSW_RPS_MAX_NTHREADS) == 1);
2260 KPKTQ_FOREACH_SAFE(pkt, pktq, tpkt) {
2261 uint32_t id = get_rps_id(fsw, pkt);
2262 KPKTQ_REMOVE(pktq, pkt);
2263 fsw_rps_rx(fsw, id, pkt);
2264 bitmap_set(&map, id);
2265 }
2266 for (int i = bitmap_first(&map, 64); i >= 0;
2267 i = bitmap_next(&map, i)) {
2268 fsw_rps_thread_schedule(fsw, i);
2269 }
2270 } else
2271 #endif /* !DEVELOPMENT && !DEBUG */
2272 {
2273 _fsw_receive_locked(fsw, pktq);
2274 }
2275 FSW_RUNLOCK(fsw);
2276 }
2277
2278 int
fsw_dev_input_netem_dequeue(void * handle,pktsched_pkt_t * pkts,uint32_t n_pkts)2279 fsw_dev_input_netem_dequeue(void *handle, pktsched_pkt_t * pkts,
2280 uint32_t n_pkts)
2281 {
2282 #pragma unused(handle)
2283 struct nx_flowswitch *fsw = handle;
2284 struct __kern_packet *kpkts[FSW_VP_DEV_BATCH_MAX];
2285 struct pktq pktq;
2286 sk_protect_t protect;
2287 uint32_t i;
2288
2289 ASSERT(n_pkts <= FSW_VP_DEV_BATCH_MAX);
2290
2291 for (i = 0; i < n_pkts; i++) {
2292 ASSERT(pkts[i].pktsched_ptype == QP_PACKET);
2293 ASSERT(pkts[i].pktsched_pkt_kpkt != NULL);
2294 kpkts[i] = pkts[i].pktsched_pkt_kpkt;
2295 }
2296
2297 protect = sk_sync_protect();
2298 KPKTQ_INIT(&pktq);
2299 pkts_to_pktq(kpkts, n_pkts, &pktq);
2300
2301 fsw_receive(fsw, &pktq);
2302 KPKTQ_FINI(&pktq);
2303 sk_sync_unprotect(protect);
2304
2305 return 0;
2306 }
2307
2308 static void
fsw_dev_input_netem_enqueue(struct nx_flowswitch * fsw,struct pktq * q)2309 fsw_dev_input_netem_enqueue(struct nx_flowswitch *fsw, struct pktq *q)
2310 {
2311 classq_pkt_t p;
2312 struct netem *ne;
2313 struct __kern_packet *pkt, *tpkt;
2314
2315 ASSERT(fsw->fsw_ifp != NULL);
2316 ne = fsw->fsw_ifp->if_input_netem;
2317 ASSERT(ne != NULL);
2318 KPKTQ_FOREACH_SAFE(pkt, q, tpkt) {
2319 bool pdrop;
2320 KPKTQ_REMOVE(q, pkt);
2321 CLASSQ_PKT_INIT_PACKET(&p, pkt);
2322 netem_enqueue(ne, &p, &pdrop);
2323 }
2324 }
2325
2326 void
fsw_devna_rx(struct nexus_adapter * devna,struct __kern_packet * pkt_head,struct nexus_pkt_stats * out_stats)2327 fsw_devna_rx(struct nexus_adapter *devna, struct __kern_packet *pkt_head,
2328 struct nexus_pkt_stats *out_stats)
2329 {
2330 struct __kern_packet *pkt = pkt_head, *next;
2331 struct nx_flowswitch *fsw;
2332 uint32_t n_bytes = 0, n_pkts = 0;
2333 uint64_t total_pkts = 0, total_bytes = 0;
2334 struct pktq q;
2335
2336 KPKTQ_INIT(&q);
2337 if (__improbable(devna->na_ifp == NULL ||
2338 (fsw = fsw_ifp_to_fsw(devna->na_ifp)) == NULL)) {
2339 SK_ERR("fsw not attached, dropping %d pkts", KPKTQ_LEN(&q));
2340 pp_free_packet_chain(pkt_head, NULL);
2341 return;
2342 }
2343 while (pkt != NULL) {
2344 if (__improbable(pkt->pkt_trace_id != 0)) {
2345 KDBG(SK_KTRACE_PKT_RX_DRV | DBG_FUNC_END, pkt->pkt_trace_id);
2346 KDBG(SK_KTRACE_PKT_RX_FSW | DBG_FUNC_START, pkt->pkt_trace_id);
2347 }
2348 next = pkt->pkt_nextpkt;
2349 pkt->pkt_nextpkt = NULL;
2350
2351 if (__probable((pkt->pkt_qum_qflags & QUM_F_DROPPED) == 0)) {
2352 KPKTQ_ENQUEUE(&q, pkt);
2353 n_bytes += pkt->pkt_length;
2354 } else {
2355 DTRACE_SKYWALK1(non__finalized__drop,
2356 struct __kern_packet *, pkt);
2357 FSW_STATS_INC(FSW_STATS_RX_PKT_NOT_FINALIZED);
2358 pp_free_packet_single(pkt);
2359 pkt = NULL;
2360 }
2361 n_pkts = KPKTQ_LEN(&q);
2362 if (n_pkts == fsw_rx_batch || (next == NULL && n_pkts > 0)) {
2363 if (__improbable(fsw->fsw_ifp->if_input_netem != NULL)) {
2364 fsw_dev_input_netem_enqueue(fsw, &q);
2365 } else {
2366 fsw_receive(fsw, &q);
2367 }
2368 total_pkts += n_pkts;
2369 total_bytes += n_bytes;
2370 n_pkts = 0;
2371 n_bytes = 0;
2372 KPKTQ_FINI(&q);
2373 }
2374 pkt = next;
2375 }
2376 ASSERT(KPKTQ_LEN(&q) == 0);
2377 FSW_STATS_ADD(FSW_STATS_RX_PACKETS, total_pkts);
2378 if (out_stats != NULL) {
2379 out_stats->nps_pkts = total_pkts;
2380 out_stats->nps_bytes = total_bytes;
2381 }
2382 KDBG(SK_KTRACE_FSW_DEV_RING_FLUSH, SK_KVA(devna), total_pkts, total_bytes);
2383 }
2384
2385 static int
dp_copy_to_dev_mbuf(struct nx_flowswitch * fsw,struct __kern_packet * spkt,struct __kern_packet * dpkt)2386 dp_copy_to_dev_mbuf(struct nx_flowswitch *fsw, struct __kern_packet *spkt,
2387 struct __kern_packet *dpkt)
2388 {
2389 struct mbuf *m = NULL;
2390 uint32_t bdlen, bdlim, bdoff;
2391 uint8_t *bdaddr;
2392 unsigned int one = 1;
2393 int err = 0;
2394
2395 err = mbuf_allocpacket(MBUF_DONTWAIT,
2396 (fsw->fsw_frame_headroom + spkt->pkt_length), &one, &m);
2397 #if (DEVELOPMENT || DEBUG)
2398 if (m != NULL) {
2399 _FSW_INJECT_ERROR(11, m, NULL, m_freem, m);
2400 }
2401 #endif /* DEVELOPMENT || DEBUG */
2402 if (__improbable(m == NULL)) {
2403 FSW_STATS_INC(FSW_STATS_DROP_NOMEM_MBUF);
2404 err = ENOBUFS;
2405 goto done;
2406 }
2407
2408 MD_BUFLET_ADDR_ABS_DLEN(dpkt, bdaddr, bdlen, bdlim, bdoff);
2409 if (fsw->fsw_frame_headroom > bdlim) {
2410 SK_ERR("not enough space in buffer for headroom");
2411 err = EINVAL;
2412 goto done;
2413 }
2414
2415 dpkt->pkt_headroom = fsw->fsw_frame_headroom;
2416 dpkt->pkt_mbuf = m;
2417 dpkt->pkt_pflags |= PKT_F_MBUF_DATA;
2418
2419 /* packet copy into mbuf */
2420 fsw->fsw_pkt_copy_to_mbuf(NR_TX, SK_PTR_ENCODE(spkt,
2421 METADATA_TYPE(spkt), METADATA_SUBTYPE(spkt)), 0, m,
2422 fsw->fsw_frame_headroom, spkt->pkt_length,
2423 PACKET_HAS_PARTIAL_CHECKSUM(spkt),
2424 spkt->pkt_csum_tx_start_off);
2425 FSW_STATS_INC(FSW_STATS_TX_COPY_PKT2MBUF);
2426
2427 /* header copy into dpkt buffer for classification */
2428 kern_packet_t sph = SK_PTR_ENCODE(spkt,
2429 METADATA_TYPE(spkt), METADATA_SUBTYPE(spkt));
2430 kern_packet_t dph = SK_PTR_ENCODE(dpkt,
2431 METADATA_TYPE(dpkt), METADATA_SUBTYPE(dpkt));
2432 uint32_t copy_len = MIN(spkt->pkt_length, bdlim - dpkt->pkt_headroom);
2433 fsw->fsw_pkt_copy_from_pkt(NR_TX, dph, dpkt->pkt_headroom,
2434 sph, spkt->pkt_headroom, copy_len, FALSE, 0, 0, 0);
2435
2436 /*
2437 * fsw->fsw_frame_headroom is after m_data, thus we treat m_data same as
2438 * buflet baddr m_data always points to the beginning of packet and
2439 * should represents the same as baddr + headroom
2440 */
2441 ASSERT((uintptr_t)m->m_data ==
2442 ((uintptr_t)mbuf_datastart(m) + fsw->fsw_frame_headroom));
2443
2444 done:
2445 return err;
2446 }
2447
2448 static int
dp_copy_to_dev_pkt(struct nx_flowswitch * fsw,struct __kern_packet * spkt,struct __kern_packet * dpkt)2449 dp_copy_to_dev_pkt(struct nx_flowswitch *fsw, struct __kern_packet *spkt,
2450 struct __kern_packet *dpkt)
2451 {
2452 struct ifnet *ifp = fsw->fsw_ifp;
2453 uint16_t headroom = fsw->fsw_frame_headroom + ifp->if_tx_headroom;
2454
2455 if (headroom > UINT8_MAX) {
2456 SK_ERR("headroom too large %d", headroom);
2457 return ERANGE;
2458 }
2459 dpkt->pkt_headroom = (uint8_t)headroom;
2460 ASSERT((dpkt->pkt_headroom & 0x7) == 0);
2461 dpkt->pkt_l2_len = 0;
2462 dpkt->pkt_link_flags = spkt->pkt_link_flags;
2463
2464 kern_packet_t sph = SK_PTR_ENCODE(spkt,
2465 METADATA_TYPE(spkt), METADATA_SUBTYPE(spkt));
2466 kern_packet_t dph = SK_PTR_ENCODE(dpkt,
2467 METADATA_TYPE(dpkt), METADATA_SUBTYPE(dpkt));
2468 fsw->fsw_pkt_copy_from_pkt(NR_TX, dph,
2469 dpkt->pkt_headroom, sph, spkt->pkt_headroom,
2470 spkt->pkt_length, PACKET_HAS_PARTIAL_CHECKSUM(spkt),
2471 (spkt->pkt_csum_tx_start_off - spkt->pkt_headroom),
2472 (spkt->pkt_csum_tx_stuff_off - spkt->pkt_headroom),
2473 (spkt->pkt_csum_flags & PACKET_CSUM_ZERO_INVERT));
2474
2475 FSW_STATS_INC(FSW_STATS_TX_COPY_PKT2PKT);
2476
2477 return 0;
2478 }
2479
2480 #if SK_LOG
2481 /* Hoisted out of line to reduce kernel stack footprint */
2482 SK_LOG_ATTRIBUTE
2483 static void
dp_copy_to_dev_log(struct nx_flowswitch * fsw,const struct kern_pbufpool * pp,struct __kern_packet * spkt,struct __kern_packet * dpkt,int error)2484 dp_copy_to_dev_log(struct nx_flowswitch *fsw, const struct kern_pbufpool *pp,
2485 struct __kern_packet *spkt, struct __kern_packet *dpkt, int error)
2486 {
2487 struct proc *p = current_proc();
2488 struct ifnet *ifp = fsw->fsw_ifp;
2489 uint64_t logflags = (SK_VERB_FSW_DP | SK_VERB_TX);
2490
2491 if (error == ERANGE) {
2492 SK_ERR("packet too long, hr(fr+tx)+slen (%u+%u)+%u > "
2493 "dev_pp_max %u", (uint32_t)fsw->fsw_frame_headroom,
2494 (uint32_t)ifp->if_tx_headroom, spkt->pkt_length,
2495 (uint32_t)pp->pp_max_frags * PP_BUF_SIZE_DEF(pp));
2496 } else if (error == ENOBUFS) {
2497 SK_DF(logflags, "%s(%d) packet allocation failure",
2498 sk_proc_name_address(p), sk_proc_pid(p));
2499 } else if (error == 0) {
2500 ASSERT(dpkt != NULL);
2501 char *daddr;
2502 MD_BUFLET_ADDR_ABS(dpkt, daddr);
2503 SK_DF(logflags, "%s(%d) splen %u dplen %u hr %u (fr/tx %u/%u)",
2504 sk_proc_name_address(p), sk_proc_pid(p), spkt->pkt_length,
2505 dpkt->pkt_length, (uint32_t)dpkt->pkt_headroom,
2506 (uint32_t)fsw->fsw_frame_headroom,
2507 (uint32_t)ifp->if_tx_headroom);
2508 SK_DF(logflags | SK_VERB_DUMP, "%s",
2509 sk_dump("buf", daddr, dpkt->pkt_length, 128, NULL, 0));
2510 } else {
2511 SK_DF(logflags, "%s(%d) error %d", error);
2512 }
2513 }
2514 #else
2515 #define dp_copy_to_dev_log(...)
2516 #endif /* SK_LOG */
2517
2518 static void
fsw_pkt_copy_metadata(struct __kern_packet * spkt,struct __kern_packet * dpkt)2519 fsw_pkt_copy_metadata(struct __kern_packet *spkt, struct __kern_packet *dpkt)
2520 {
2521 ASSERT(!(spkt->pkt_pflags & PKT_F_MBUF_MASK));
2522 ASSERT(!(spkt->pkt_pflags & PKT_F_PKT_MASK));
2523
2524 SK_PREFETCHW(dpkt->pkt_qum_buf.buf_addr, 0);
2525 /* Copy packet metadata */
2526 _QUM_COPY(&(spkt)->pkt_qum, &(dpkt)->pkt_qum);
2527 _PKT_COPY(spkt, dpkt);
2528 _PKT_COPY_TX_PORT_DATA(spkt, dpkt);
2529 ASSERT((dpkt->pkt_qum.qum_qflags & QUM_F_KERNEL_ONLY) ||
2530 !PP_KERNEL_ONLY(dpkt->pkt_qum.qum_pp));
2531 ASSERT(dpkt->pkt_mbuf == NULL);
2532
2533 /* Copy AQM metadata */
2534 dpkt->pkt_flowsrc_type = spkt->pkt_flowsrc_type;
2535 dpkt->pkt_flowsrc_fidx = spkt->pkt_flowsrc_fidx;
2536 _CASSERT((offsetof(struct __flow, flow_src_id) % 8) == 0);
2537 _UUID_COPY(dpkt->pkt_flowsrc_id, spkt->pkt_flowsrc_id);
2538 _UUID_COPY(dpkt->pkt_policy_euuid, spkt->pkt_policy_euuid);
2539 dpkt->pkt_policy_id = spkt->pkt_policy_id;
2540 }
2541
2542 static int
dp_copy_to_dev(struct nx_flowswitch * fsw,struct __kern_packet * spkt,struct __kern_packet * dpkt)2543 dp_copy_to_dev(struct nx_flowswitch *fsw, struct __kern_packet *spkt,
2544 struct __kern_packet *dpkt)
2545 {
2546 const struct kern_pbufpool *pp = dpkt->pkt_qum.qum_pp;
2547 struct ifnet *ifp = fsw->fsw_ifp;
2548 uint32_t dev_pkt_len;
2549 int err = 0;
2550
2551 fsw_pkt_copy_metadata(spkt, dpkt);
2552 switch (fsw->fsw_classq_enq_ptype) {
2553 case QP_MBUF:
2554 err = dp_copy_to_dev_mbuf(fsw, spkt, dpkt);
2555 break;
2556
2557 case QP_PACKET:
2558 dev_pkt_len = fsw->fsw_frame_headroom + ifp->if_tx_headroom +
2559 spkt->pkt_length;
2560 if (dev_pkt_len > pp->pp_max_frags * PP_BUF_SIZE_DEF(pp)) {
2561 FSW_STATS_INC(FSW_STATS_TX_COPY_BAD_LEN);
2562 err = ERANGE;
2563 goto done;
2564 }
2565 err = dp_copy_to_dev_pkt(fsw, spkt, dpkt);
2566 break;
2567
2568 default:
2569 VERIFY(0);
2570 __builtin_unreachable();
2571 }
2572 done:
2573 dp_copy_to_dev_log(fsw, pp, spkt, dpkt, err);
2574 return err;
2575 }
2576
2577 static int
dp_copy_headers_to_dev(struct nx_flowswitch * fsw,struct __kern_packet * spkt,struct __kern_packet * dpkt)2578 dp_copy_headers_to_dev(struct nx_flowswitch *fsw, struct __kern_packet *spkt,
2579 struct __kern_packet *dpkt)
2580 {
2581 uint8_t *sbaddr, *dbaddr;
2582 uint16_t headroom = fsw->fsw_frame_headroom + fsw->fsw_ifp->if_tx_headroom;
2583 uint16_t hdrs_len_estimate = (uint16_t)MIN(spkt->pkt_length, 128);
2584
2585 fsw_pkt_copy_metadata(spkt, dpkt);
2586
2587 MD_BUFLET_ADDR_ABS(spkt, sbaddr);
2588 ASSERT(sbaddr != NULL);
2589 sbaddr += spkt->pkt_headroom;
2590
2591 MD_BUFLET_ADDR_ABS(dpkt, dbaddr);
2592 ASSERT(dbaddr != NULL);
2593 dpkt->pkt_headroom = (uint8_t)headroom;
2594 dbaddr += headroom;
2595
2596 pkt_copy(sbaddr, dbaddr, hdrs_len_estimate);
2597 METADATA_SET_LEN(dpkt, hdrs_len_estimate, headroom);
2598
2599 /* packet length is set to the full length */
2600 dpkt->pkt_length = spkt->pkt_length;
2601 dpkt->pkt_pflags |= PKT_F_TRUNCATED;
2602 return 0;
2603 }
2604
2605 static struct mbuf *
convert_pkt_to_mbuf(struct __kern_packet * pkt)2606 convert_pkt_to_mbuf(struct __kern_packet *pkt)
2607 {
2608 ASSERT(pkt->pkt_pflags & PKT_F_MBUF_DATA);
2609 ASSERT(pkt->pkt_mbuf != NULL);
2610 struct mbuf *m = pkt->pkt_mbuf;
2611
2612 /* pass additional metadata generated from flow parse/lookup */
2613 _CASSERT(sizeof(m->m_pkthdr.pkt_flowid) ==
2614 sizeof(pkt->pkt_flow_token));
2615 _CASSERT(sizeof(m->m_pkthdr.pkt_mpriv_srcid) ==
2616 sizeof(pkt->pkt_flowsrc_token));
2617 _CASSERT(sizeof(m->m_pkthdr.pkt_mpriv_fidx) ==
2618 sizeof(pkt->pkt_flowsrc_fidx));
2619 m->m_pkthdr.pkt_svc = pkt->pkt_svc_class;
2620 m->m_pkthdr.pkt_proto = pkt->pkt_flow->flow_ip_proto;
2621 m->m_pkthdr.pkt_flowid = pkt->pkt_flow_token;
2622 m->m_pkthdr.comp_gencnt = pkt->pkt_comp_gencnt;
2623 m->m_pkthdr.pkt_flowsrc = pkt->pkt_flowsrc_type;
2624 m->m_pkthdr.pkt_mpriv_srcid = pkt->pkt_flowsrc_token;
2625 m->m_pkthdr.pkt_mpriv_fidx = pkt->pkt_flowsrc_fidx;
2626
2627 if (pkt->pkt_transport_protocol == IPPROTO_QUIC) {
2628 m->m_pkthdr.pkt_ext_flags |= PKTF_EXT_QUIC;
2629 }
2630
2631 /* The packet should have a timestamp by the time we get here. */
2632 m->m_pkthdr.pkt_timestamp = pkt->pkt_timestamp;
2633 m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID;
2634
2635 m->m_pkthdr.pkt_flags &= ~PKT_F_COMMON_MASK;
2636 m->m_pkthdr.pkt_flags |= (pkt->pkt_pflags & PKT_F_COMMON_MASK);
2637 /* set pkt_hdr so that AQM can find IP header and mark ECN bits */
2638 m->m_pkthdr.pkt_hdr = m->m_data + pkt->pkt_l2_len;
2639
2640 if ((pkt->pkt_pflags & PKT_F_START_SEQ) != 0) {
2641 m->m_pkthdr.tx_start_seq = ntohl(pkt->pkt_flow_tcp_seq);
2642 }
2643 KPKT_CLEAR_MBUF_DATA(pkt);
2644
2645 /* mbuf has been consumed, release packet as well */
2646 ASSERT(pkt->pkt_qum.qum_ksd == NULL);
2647 pp_free_packet_single(pkt);
2648 return m;
2649 }
2650
2651 static void
convert_pkt_to_mbuf_list(struct __kern_packet * pkt_list,struct mbuf ** head,struct mbuf ** tail,uint32_t * cnt,uint32_t * bytes)2652 convert_pkt_to_mbuf_list(struct __kern_packet *pkt_list,
2653 struct mbuf **head, struct mbuf **tail,
2654 uint32_t *cnt, uint32_t *bytes)
2655 {
2656 struct __kern_packet *pkt = pkt_list, *next;
2657 struct mbuf *m_head = NULL, **m_tailp = &m_head, *m = NULL;
2658 uint32_t c = 0, b = 0;
2659
2660 while (pkt != NULL) {
2661 next = pkt->pkt_nextpkt;
2662 pkt->pkt_nextpkt = NULL;
2663 m = convert_pkt_to_mbuf(pkt);
2664 ASSERT(m != NULL);
2665
2666 *m_tailp = m;
2667 m_tailp = &m->m_nextpkt;
2668 c++;
2669 b += m_pktlen(m);
2670 pkt = next;
2671 }
2672 if (head != NULL) {
2673 *head = m_head;
2674 }
2675 if (tail != NULL) {
2676 *tail = m;
2677 }
2678 if (cnt != NULL) {
2679 *cnt = c;
2680 }
2681 if (bytes != NULL) {
2682 *bytes = b;
2683 }
2684 }
2685
2686 SK_NO_INLINE_ATTRIBUTE
2687 static int
classq_enqueue_flow_single(struct nx_flowswitch * fsw,struct __kern_packet * pkt)2688 classq_enqueue_flow_single(struct nx_flowswitch *fsw,
2689 struct __kern_packet *pkt)
2690 {
2691 struct ifnet *ifp = fsw->fsw_ifp;
2692 boolean_t pkt_drop = FALSE;
2693 int err;
2694
2695 FSW_LOCK_ASSERT_HELD(fsw);
2696 ASSERT(fsw->fsw_classq_enabled);
2697 ASSERT(pkt->pkt_flow_token != 0);
2698 fsw_ifp_inc_traffic_class_out_pkt(ifp, pkt->pkt_svc_class,
2699 1, pkt->pkt_length);
2700
2701 if (__improbable(pkt->pkt_trace_id != 0)) {
2702 KDBG(SK_KTRACE_PKT_TX_FSW | DBG_FUNC_END, pkt->pkt_trace_id);
2703 KDBG(SK_KTRACE_PKT_TX_AQM | DBG_FUNC_START, pkt->pkt_trace_id);
2704 }
2705
2706 switch (fsw->fsw_classq_enq_ptype) {
2707 case QP_MBUF: { /* compat interface */
2708 struct mbuf *m;
2709
2710 m = convert_pkt_to_mbuf(pkt);
2711 ASSERT(m != NULL);
2712 pkt = NULL;
2713
2714 /* ifnet_enqueue consumes mbuf */
2715 err = ifnet_enqueue_mbuf(ifp, m, false, &pkt_drop);
2716 m = NULL;
2717 #if (DEVELOPMENT || DEBUG)
2718 if (__improbable(!pkt_drop)) {
2719 _FSW_INJECT_ERROR(14, pkt_drop, TRUE, null_func);
2720 }
2721 #endif /* DEVELOPMENT || DEBUG */
2722 if (pkt_drop) {
2723 FSW_STATS_INC(FSW_STATS_DROP);
2724 FSW_STATS_INC(FSW_STATS_TX_AQM_DROP);
2725 }
2726 break;
2727 }
2728 case QP_PACKET: { /* native interface */
2729 /* ifnet_enqueue consumes packet */
2730 err = ifnet_enqueue_pkt(ifp, pkt, false, &pkt_drop);
2731 pkt = NULL;
2732 #if (DEVELOPMENT || DEBUG)
2733 if (__improbable(!pkt_drop)) {
2734 _FSW_INJECT_ERROR(14, pkt_drop, TRUE, null_func);
2735 }
2736 #endif /* DEVELOPMENT || DEBUG */
2737 if (pkt_drop) {
2738 FSW_STATS_INC(FSW_STATS_DROP);
2739 FSW_STATS_INC(FSW_STATS_TX_AQM_DROP);
2740 }
2741 break;
2742 }
2743 default:
2744 err = EINVAL;
2745 VERIFY(0);
2746 /* NOTREACHED */
2747 __builtin_unreachable();
2748 }
2749
2750 return err;
2751 }
2752
2753 static int
classq_enqueue_flow_chain(struct nx_flowswitch * fsw,struct __kern_packet * pkt_head,struct __kern_packet * pkt_tail,uint32_t cnt,uint32_t bytes)2754 classq_enqueue_flow_chain(struct nx_flowswitch *fsw,
2755 struct __kern_packet *pkt_head, struct __kern_packet *pkt_tail,
2756 uint32_t cnt, uint32_t bytes)
2757 {
2758 struct ifnet *ifp = fsw->fsw_ifp;
2759 boolean_t pkt_drop = FALSE;
2760 uint32_t svc;
2761 int err;
2762
2763 FSW_LOCK_ASSERT_HELD(fsw);
2764 ASSERT(fsw->fsw_classq_enabled);
2765 ASSERT(pkt_head->pkt_flow_token != 0);
2766
2767 /*
2768 * All packets in the flow should have the same svc.
2769 */
2770 svc = pkt_head->pkt_svc_class;
2771 fsw_ifp_inc_traffic_class_out_pkt(ifp, svc, cnt, bytes);
2772
2773 switch (fsw->fsw_classq_enq_ptype) {
2774 case QP_MBUF: { /* compat interface */
2775 struct mbuf *m_head = NULL, *m_tail = NULL;
2776 uint32_t c = 0, b = 0;
2777
2778 convert_pkt_to_mbuf_list(pkt_head, &m_head, &m_tail, &c, &b);
2779 ASSERT(m_head != NULL && m_tail != NULL);
2780 ASSERT(c == cnt);
2781 ASSERT(b == bytes);
2782 pkt_head = NULL;
2783
2784 /* ifnet_enqueue consumes mbuf */
2785 err = ifnet_enqueue_mbuf_chain(ifp, m_head, m_tail, cnt,
2786 bytes, FALSE, &pkt_drop);
2787 m_head = NULL;
2788 m_tail = NULL;
2789 #if (DEVELOPMENT || DEBUG)
2790 if (__improbable(!pkt_drop)) {
2791 _FSW_INJECT_ERROR(14, pkt_drop, TRUE, null_func);
2792 }
2793 #endif /* DEVELOPMENT || DEBUG */
2794 if (pkt_drop) {
2795 STATS_ADD(&fsw->fsw_stats, FSW_STATS_DROP, cnt);
2796 STATS_ADD(&fsw->fsw_stats, FSW_STATS_TX_AQM_DROP,
2797 cnt);
2798 }
2799 break;
2800 }
2801 case QP_PACKET: { /* native interface */
2802 /* ifnet_enqueue consumes packet */
2803 err = ifnet_enqueue_pkt_chain(ifp, pkt_head, pkt_tail, cnt,
2804 bytes, FALSE, &pkt_drop);
2805 pkt_head = NULL;
2806 #if (DEVELOPMENT || DEBUG)
2807 if (__improbable(!pkt_drop)) {
2808 _FSW_INJECT_ERROR(14, pkt_drop, TRUE, null_func);
2809 }
2810 #endif /* DEVELOPMENT || DEBUG */
2811 if (pkt_drop) {
2812 STATS_ADD(&fsw->fsw_stats, FSW_STATS_DROP, cnt);
2813 STATS_ADD(&fsw->fsw_stats, FSW_STATS_TX_AQM_DROP,
2814 cnt);
2815 }
2816 break;
2817 }
2818 default:
2819 err = EINVAL;
2820 VERIFY(0);
2821 /* NOTREACHED */
2822 __builtin_unreachable();
2823 }
2824
2825 return err;
2826 }
2827
2828 /*
2829 * This code path needs to be kept for interfaces without logical link support.
2830 */
2831 static void
classq_enqueue_flow(struct nx_flowswitch * fsw,struct flow_entry * fe,bool chain,uint32_t cnt,uint32_t bytes)2832 classq_enqueue_flow(struct nx_flowswitch *fsw, struct flow_entry *fe,
2833 bool chain, uint32_t cnt, uint32_t bytes)
2834 {
2835 bool flowadv_is_set = false;
2836 struct __kern_packet *pkt, *tail, *tpkt;
2837 flowadv_idx_t flow_adv_idx;
2838 bool flowadv_cap;
2839 flowadv_token_t flow_adv_token;
2840 int err;
2841
2842 SK_DF(SK_VERB_FSW_DP | SK_VERB_AQM, "%s classq enqueued %d pkts",
2843 if_name(fsw->fsw_ifp), KPKTQ_LEN(&fe->fe_tx_pktq));
2844
2845 if (chain) {
2846 pkt = KPKTQ_FIRST(&fe->fe_tx_pktq);
2847 tail = KPKTQ_LAST(&fe->fe_tx_pktq);
2848 KPKTQ_INIT(&fe->fe_tx_pktq);
2849 if (pkt == NULL) {
2850 return;
2851 }
2852 flow_adv_idx = pkt->pkt_flowsrc_fidx;
2853 flowadv_cap = ((pkt->pkt_pflags & PKT_F_FLOW_ADV) != 0);
2854 flow_adv_token = pkt->pkt_flow_token;
2855
2856 err = classq_enqueue_flow_chain(fsw, pkt, tail, cnt, bytes);
2857
2858 /* set flow advisory if needed */
2859 if (__improbable((err == EQFULL || err == EQSUSPENDED) &&
2860 flowadv_cap)) {
2861 flowadv_is_set = na_flowadv_set(flow_get_na(fsw, fe),
2862 flow_adv_idx, flow_adv_token);
2863 }
2864 DTRACE_SKYWALK3(chain__enqueue, uint32_t, cnt, uint32_t, bytes,
2865 bool, flowadv_is_set);
2866 } else {
2867 uint32_t c = 0, b = 0;
2868
2869 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_tx_pktq, tpkt) {
2870 KPKTQ_REMOVE(&fe->fe_tx_pktq, pkt);
2871
2872 flow_adv_idx = pkt->pkt_flowsrc_fidx;
2873 flowadv_cap = ((pkt->pkt_pflags & PKT_F_FLOW_ADV) != 0);
2874 flow_adv_token = pkt->pkt_flow_token;
2875
2876 c++;
2877 b += pkt->pkt_length;
2878 err = classq_enqueue_flow_single(fsw, pkt);
2879
2880 /* set flow advisory if needed */
2881 if (__improbable(!flowadv_is_set &&
2882 ((err == EQFULL || err == EQSUSPENDED) &&
2883 flowadv_cap))) {
2884 flowadv_is_set = na_flowadv_set(
2885 flow_get_na(fsw, fe), flow_adv_idx,
2886 flow_adv_token);
2887 }
2888 }
2889 ASSERT(c == cnt);
2890 ASSERT(b == bytes);
2891 DTRACE_SKYWALK3(non__chain__enqueue, uint32_t, cnt, uint32_t, bytes,
2892 bool, flowadv_is_set);
2893 }
2894
2895 /* notify flow advisory event */
2896 if (__improbable(flowadv_is_set)) {
2897 struct __kern_channel_ring *r = fsw_flow_get_tx_ring(fsw, fe);
2898 if (__probable(r)) {
2899 na_flowadv_event(r);
2900 SK_DF(SK_VERB_FLOW_ADVISORY | SK_VERB_TX,
2901 "%s(%d) notified of flow update",
2902 sk_proc_name_address(current_proc()),
2903 sk_proc_pid(current_proc()));
2904 }
2905 }
2906 }
2907
2908 /*
2909 * Logical link code path
2910 */
2911 static void
classq_qset_enqueue_flow(struct nx_flowswitch * fsw,struct flow_entry * fe,bool chain,uint32_t cnt,uint32_t bytes)2912 classq_qset_enqueue_flow(struct nx_flowswitch *fsw, struct flow_entry *fe,
2913 bool chain, uint32_t cnt, uint32_t bytes)
2914 {
2915 #pragma unused(chain)
2916 struct __kern_packet *pkt, *tail;
2917 flowadv_idx_t flow_adv_idx;
2918 bool flowadv_is_set = false;
2919 bool flowadv_cap;
2920 flowadv_token_t flow_adv_token;
2921 uint32_t flowctl = 0, dropped = 0;
2922 int err;
2923
2924 SK_DF(SK_VERB_FSW_DP | SK_VERB_AQM, "%s classq enqueued %d pkts",
2925 if_name(fsw->fsw_ifp), KPKTQ_LEN(&fe->fe_tx_pktq));
2926
2927 pkt = KPKTQ_FIRST(&fe->fe_tx_pktq);
2928 tail = KPKTQ_LAST(&fe->fe_tx_pktq);
2929 KPKTQ_INIT(&fe->fe_tx_pktq);
2930 if (pkt == NULL) {
2931 return;
2932 }
2933 flow_adv_idx = pkt->pkt_flowsrc_fidx;
2934 flowadv_cap = ((pkt->pkt_pflags & PKT_F_FLOW_ADV) != 0);
2935 flow_adv_token = pkt->pkt_flow_token;
2936
2937 err = netif_qset_enqueue(fe->fe_qset, pkt, tail, cnt, bytes,
2938 &flowctl, &dropped);
2939
2940 if (__improbable(err != 0)) {
2941 /* set flow advisory if needed */
2942 if (flowctl > 0 && flowadv_cap) {
2943 flowadv_is_set = na_flowadv_set(flow_get_na(fsw, fe),
2944 flow_adv_idx, flow_adv_token);
2945
2946 /* notify flow advisory event */
2947 if (flowadv_is_set) {
2948 struct __kern_channel_ring *r =
2949 fsw_flow_get_tx_ring(fsw, fe);
2950 if (__probable(r)) {
2951 na_flowadv_event(r);
2952 SK_DF(SK_VERB_FLOW_ADVISORY |
2953 SK_VERB_TX,
2954 "%s(%d) notified of flow update",
2955 sk_proc_name_address(current_proc()),
2956 sk_proc_pid(current_proc()));
2957 }
2958 }
2959 }
2960 if (dropped > 0) {
2961 STATS_ADD(&fsw->fsw_stats, FSW_STATS_DROP, dropped);
2962 STATS_ADD(&fsw->fsw_stats, FSW_STATS_TX_AQM_DROP,
2963 dropped);
2964 }
2965 }
2966 }
2967
2968 static void
tx_finalize_packet(struct nx_flowswitch * fsw,struct __kern_packet * pkt)2969 tx_finalize_packet(struct nx_flowswitch *fsw, struct __kern_packet *pkt)
2970 {
2971 #pragma unused(fsw)
2972 /* finalize here; no more changes to buflets after classq */
2973 if (__probable(!(pkt->pkt_pflags & PKT_F_MBUF_DATA))) {
2974 kern_packet_t ph = SK_PTR_ENCODE(pkt,
2975 METADATA_TYPE(pkt), METADATA_SUBTYPE(pkt));
2976 int err = __packet_finalize(ph);
2977 VERIFY(err == 0);
2978 }
2979 }
2980
2981 static bool
dp_flow_tx_route_process(struct nx_flowswitch * fsw,struct flow_entry * fe)2982 dp_flow_tx_route_process(struct nx_flowswitch *fsw, struct flow_entry *fe)
2983 {
2984 struct flow_route *fr = fe->fe_route;
2985 int err;
2986
2987 ASSERT(fr != NULL);
2988
2989 if (__improbable(!dp_flow_route_process(fsw, fe))) {
2990 return false;
2991 }
2992 if (fe->fe_qset_select == FE_QSET_SELECT_DYNAMIC) {
2993 flow_qset_select_dynamic(fsw, fe, TRUE);
2994 }
2995
2996 _FSW_INJECT_ERROR(35, fr->fr_flags, fr->fr_flags,
2997 _fsw_error35_handler, 1, fr, NULL, NULL);
2998 _FSW_INJECT_ERROR(36, fr->fr_flags, fr->fr_flags,
2999 _fsw_error36_handler, 1, fr, NULL);
3000
3001 /*
3002 * See if we need to resolve the flow route; note the test against
3003 * fr_flags here is done without any lock for performance. Thus
3004 * it's possible that we race against the thread performing route
3005 * event updates for a packet (which is OK). In any case we should
3006 * not have any assertion on fr_flags value(s) due to the lack of
3007 * serialization.
3008 */
3009 if (fr->fr_flags & FLOWRTF_RESOLVED) {
3010 goto frame;
3011 }
3012
3013 struct __kern_packet *pkt, *tpkt;
3014 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_tx_pktq, tpkt) {
3015 err = fsw->fsw_resolve(fsw, fr, pkt);
3016 _FSW_INJECT_ERROR_SET(35, _fsw_error35_handler, 2, fr, pkt, &err);
3017 _FSW_INJECT_ERROR_SET(36, _fsw_error36_handler, 2, fr, &err);
3018 /*
3019 * If resolver returns EJUSTRETURN then we drop the pkt as the
3020 * resolver should have converted the pkt into mbuf (or
3021 * detached the attached mbuf from pkt) and added it to the
3022 * llinfo queue. If we do have a cached llinfo, then proceed
3023 * to using it even though it may be stale (very unlikely)
3024 * while the resolution is in progress.
3025 * Otherwise, any other error results in dropping pkt.
3026 */
3027 if (err == EJUSTRETURN) {
3028 KPKTQ_REMOVE(&fe->fe_tx_pktq, pkt);
3029 pp_free_packet_single(pkt);
3030 FSW_STATS_INC(FSW_STATS_TX_RESOLV_PENDING);
3031 continue;
3032 } else if (err != 0 && (fr->fr_flags & FLOWRTF_HAS_LLINFO)) {
3033 /* use existing llinfo */
3034 FSW_STATS_INC(FSW_STATS_TX_RESOLV_STALE);
3035 } else if (err != 0) {
3036 KPKTQ_REMOVE(&fe->fe_tx_pktq, pkt);
3037 pp_free_packet_single(pkt);
3038 FSW_STATS_INC(FSW_STATS_TX_RESOLV_FAIL);
3039 continue;
3040 }
3041 }
3042
3043 frame:
3044 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_tx_pktq, tpkt) {
3045 if (fsw->fsw_frame != NULL) {
3046 fsw->fsw_frame(fsw, fr, pkt);
3047 }
3048 }
3049
3050 return true;
3051 }
3052
3053 static void
dp_listener_flow_tx_process(struct nx_flowswitch * fsw,struct flow_entry * fe)3054 dp_listener_flow_tx_process(struct nx_flowswitch *fsw, struct flow_entry *fe)
3055 {
3056 #pragma unused(fsw)
3057 struct __kern_packet *pkt, *tpkt;
3058 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_tx_pktq, tpkt) {
3059 KPKTQ_REMOVE(&fe->fe_tx_pktq, pkt);
3060 /* listener is only allowed TCP RST */
3061 if (pkt->pkt_flow_ip_proto == IPPROTO_TCP &&
3062 (pkt->pkt_flow_tcp_flags & TH_RST) != 0) {
3063 flow_track_abort_tcp(fe, NULL, pkt);
3064 } else {
3065 char *addr;
3066 MD_BUFLET_ADDR_ABS(pkt, addr);
3067 SK_ERR("listener flow sends non-RST packet %s",
3068 sk_dump(sk_proc_name_address(current_proc()),
3069 addr, pkt->pkt_length, 128, NULL, 0));
3070 }
3071 pp_free_packet_single(pkt);
3072 }
3073 }
3074
3075 static void
fsw_update_timestamps(struct __kern_packet * pkt,volatile uint64_t * fg_ts,volatile uint64_t * rt_ts,ifnet_t ifp)3076 fsw_update_timestamps(struct __kern_packet *pkt, volatile uint64_t *fg_ts,
3077 volatile uint64_t *rt_ts, ifnet_t ifp)
3078 {
3079 struct timespec now;
3080 uint64_t now_nsec = 0;
3081
3082 if (!(pkt->pkt_pflags & PKT_F_TS_VALID) || pkt->pkt_timestamp == 0) {
3083 nanouptime(&now);
3084 net_timernsec(&now, &now_nsec);
3085 pkt->pkt_timestamp = now_nsec;
3086 }
3087 pkt->pkt_pflags &= ~PKT_F_TS_VALID;
3088
3089 /*
3090 * If the packet service class is not background,
3091 * update the timestamps on the interface, as well as
3092 * the ones in nexus-wide advisory to indicate recent
3093 * activity on a foreground flow.
3094 */
3095 if (!(pkt->pkt_pflags & PKT_F_BACKGROUND)) {
3096 ifp->if_fg_sendts = (uint32_t)_net_uptime;
3097 if (fg_ts != NULL) {
3098 *fg_ts = _net_uptime;
3099 }
3100 }
3101 if (pkt->pkt_pflags & PKT_F_REALTIME) {
3102 ifp->if_rt_sendts = (uint32_t)_net_uptime;
3103 if (rt_ts != NULL) {
3104 *rt_ts = _net_uptime;
3105 }
3106 }
3107 }
3108
3109 static bool
fsw_chain_enqueue_enabled(struct nx_flowswitch * fsw,bool gso_enabled)3110 fsw_chain_enqueue_enabled(struct nx_flowswitch *fsw, bool gso_enabled)
3111 {
3112 return fsw_chain_enqueue != 0 &&
3113 fsw->fsw_ifp->if_output_netem == NULL &&
3114 (fsw->fsw_ifp->if_eflags & IFEF_ENQUEUE_MULTI) == 0 &&
3115 gso_enabled;
3116 }
3117
3118 void
dp_flow_tx_process(struct nx_flowswitch * fsw,struct flow_entry * fe,uint32_t flags)3119 dp_flow_tx_process(struct nx_flowswitch *fsw, struct flow_entry *fe,
3120 uint32_t flags)
3121 {
3122 struct pktq dropped_pkts;
3123 bool chain, gso = ((flags & FLOW_PROC_FLAG_GSO) != 0);
3124 uint32_t cnt = 0, bytes = 0;
3125 volatile struct sk_nexusadv *nxadv = NULL;
3126 volatile uint64_t *fg_ts = NULL;
3127 volatile uint64_t *rt_ts = NULL;
3128 uint8_t qset_idx = (fe->fe_qset != NULL) ? fe->fe_qset->nqs_idx : 0;
3129
3130 KPKTQ_INIT(&dropped_pkts);
3131 ASSERT(!KPKTQ_EMPTY(&fe->fe_tx_pktq));
3132 if (__improbable(fe->fe_flags & FLOWENTF_LISTENER)) {
3133 dp_listener_flow_tx_process(fsw, fe);
3134 return;
3135 }
3136 if (__improbable(!dp_flow_tx_route_process(fsw, fe))) {
3137 SK_RDERR(5, "Tx route bad");
3138 FSW_STATS_ADD(FSW_STATS_TX_FLOW_NONVIABLE,
3139 KPKTQ_LEN(&fe->fe_tx_pktq));
3140 KPKTQ_CONCAT(&dropped_pkts, &fe->fe_tx_pktq);
3141 goto done;
3142 }
3143 chain = fsw_chain_enqueue_enabled(fsw, gso);
3144 if (chain) {
3145 nxadv = fsw->fsw_nx->nx_adv.flowswitch_nxv_adv;
3146 if (nxadv != NULL) {
3147 fg_ts = &nxadv->nxadv_fg_sendts;
3148 rt_ts = &nxadv->nxadv_rt_sendts;
3149 }
3150 }
3151 struct __kern_packet *pkt, *tpkt;
3152 KPKTQ_FOREACH_SAFE(pkt, &fe->fe_tx_pktq, tpkt) {
3153 int err = 0;
3154
3155 err = flow_pkt_track(fe, pkt, false);
3156 if (__improbable(err != 0)) {
3157 SK_RDERR(5, "flow_pkt_track failed (err %d)", err);
3158 FSW_STATS_INC(FSW_STATS_TX_FLOW_TRACK_ERR);
3159 KPKTQ_REMOVE(&fe->fe_tx_pktq, pkt);
3160 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3161 continue;
3162 }
3163 _UUID_COPY(pkt->pkt_policy_euuid, fe->fe_eproc_uuid);
3164 pkt->pkt_transport_protocol = fe->fe_transport_protocol;
3165
3166 /* set AQM related values for outgoing packet */
3167 if (fe->fe_adv_idx != FLOWADV_IDX_NONE) {
3168 pkt->pkt_pflags |= PKT_F_FLOW_ADV;
3169 pkt->pkt_flowsrc_type = FLOWSRC_CHANNEL;
3170 pkt->pkt_flowsrc_fidx = fe->fe_adv_idx;
3171 } else {
3172 pkt->pkt_pflags &= ~PKT_F_FLOW_ADV;
3173 }
3174 _UUID_CLEAR(pkt->pkt_flow_id);
3175 pkt->pkt_flow_token = fe->fe_flowid;
3176 pkt->pkt_pflags |= PKT_F_FLOW_ID;
3177 pkt->pkt_qset_idx = qset_idx;
3178 /*
3179 * The same code is exercised per packet for the non-chain case
3180 * (see ifnet_enqueue_ifclassq()). It's replicated here to avoid
3181 * re-walking the chain later.
3182 */
3183 if (chain) {
3184 fsw_update_timestamps(pkt, fg_ts, rt_ts, fsw->fsw_ifp);
3185 }
3186 /* mark packet tos/svc_class */
3187 fsw_qos_mark(fsw, fe, pkt);
3188
3189 tx_finalize_packet(fsw, pkt);
3190 bytes += pkt->pkt_length;
3191 cnt++;
3192 }
3193
3194 /* snoop after it's finalized */
3195 if (__improbable(pktap_total_tap_count != 0)) {
3196 fsw_snoop(fsw, fe, false);
3197 }
3198 if (fe->fe_qset != NULL) {
3199 classq_qset_enqueue_flow(fsw, fe, chain, cnt, bytes);
3200 } else {
3201 classq_enqueue_flow(fsw, fe, chain, cnt, bytes);
3202 }
3203 done:
3204 dp_drop_pktq(fsw, &dropped_pkts);
3205 }
3206
3207 static struct flow_entry *
tx_process_continuous_ip_frag(struct nx_flowswitch * fsw,struct flow_entry * prev_fe,struct __kern_packet * pkt)3208 tx_process_continuous_ip_frag(struct nx_flowswitch *fsw,
3209 struct flow_entry *prev_fe, struct __kern_packet *pkt)
3210 {
3211 ASSERT(!pkt->pkt_flow_ip_is_first_frag);
3212
3213 if (__improbable(pkt->pkt_flow_ip_frag_id == 0)) {
3214 FSW_STATS_INC(FSW_STATS_TX_FRAG_BAD_ID);
3215 SK_ERR("%s(%d) invalid zero fragment id",
3216 sk_proc_name_address(current_proc()),
3217 sk_proc_pid(current_proc()));
3218 return NULL;
3219 }
3220
3221 SK_DF(SK_VERB_FSW_DP | SK_VERB_TX,
3222 "%s(%d) continuation frag, id %u",
3223 sk_proc_name_address(current_proc()),
3224 sk_proc_pid(current_proc()),
3225 pkt->pkt_flow_ip_frag_id);
3226 if (__improbable(prev_fe == NULL ||
3227 !prev_fe->fe_tx_is_cont_frag)) {
3228 SK_ERR("%s(%d) unexpected continuation frag",
3229 sk_proc_name_address(current_proc()),
3230 sk_proc_pid(current_proc()),
3231 pkt->pkt_flow_ip_frag_id);
3232 FSW_STATS_INC(FSW_STATS_TX_FRAG_BAD_CONT);
3233 return NULL;
3234 }
3235 if (__improbable(pkt->pkt_flow_ip_frag_id !=
3236 prev_fe->fe_tx_frag_id)) {
3237 FSW_STATS_INC(FSW_STATS_TX_FRAG_BAD_CONT);
3238 SK_ERR("%s(%d) wrong continuation frag id %u expecting %u",
3239 sk_proc_name_address(current_proc()),
3240 sk_proc_pid(current_proc()),
3241 pkt->pkt_flow_ip_frag_id,
3242 prev_fe->fe_tx_frag_id);
3243 return NULL;
3244 }
3245
3246 return prev_fe;
3247 }
3248
3249 static struct flow_entry *
tx_lookup_flow(struct nx_flowswitch * fsw,struct __kern_packet * pkt,struct flow_entry * prev_fe)3250 tx_lookup_flow(struct nx_flowswitch *fsw, struct __kern_packet *pkt,
3251 struct flow_entry *prev_fe)
3252 {
3253 struct flow_entry *fe;
3254
3255 fe = lookup_flow_with_pkt(fsw, pkt, false, prev_fe);
3256 if (__improbable(fe == NULL)) {
3257 goto done;
3258 }
3259
3260 if (__improbable(fe->fe_flags & FLOWENTF_TORN_DOWN)) {
3261 SK_RDERR(5, "Tx flow torn down");
3262 FSW_STATS_INC(FSW_STATS_TX_FLOW_TORNDOWN);
3263 flow_entry_release(&fe);
3264 goto done;
3265 }
3266
3267 _FSW_INJECT_ERROR(34, pkt->pkt_flow_id[0], fe->fe_uuid[0] + 1,
3268 null_func);
3269
3270 if (__improbable(!_UUID_MATCH(pkt->pkt_flow_id, fe->fe_uuid))) {
3271 uuid_string_t flow_id_str, pkt_id_str;
3272 sk_uuid_unparse(fe->fe_uuid, flow_id_str);
3273 sk_uuid_unparse(pkt->pkt_flow_id, pkt_id_str);
3274 SK_ERR("pkt flow id %s != flow id %s", pkt_id_str, flow_id_str);
3275 flow_entry_release(&fe);
3276 FSW_STATS_INC(FSW_STATS_TX_FLOW_BAD_ID);
3277 }
3278
3279 done:
3280 return fe;
3281 }
3282
3283 static inline void
tx_flow_process(struct nx_flowswitch * fsw,struct flow_entry * fe,uint32_t flags)3284 tx_flow_process(struct nx_flowswitch *fsw, struct flow_entry *fe,
3285 uint32_t flags)
3286 {
3287 ASSERT(!KPKTQ_EMPTY(&fe->fe_tx_pktq));
3288 ASSERT(KPKTQ_LEN(&fe->fe_tx_pktq) != 0);
3289
3290 SK_DF(SK_VERB_FSW_DP | SK_VERB_TX, "TX %d pkts from fe %p port %d",
3291 KPKTQ_LEN(&fe->fe_tx_pktq), fe, fe->fe_nx_port);
3292
3293 /* flow related processing (default, agg, etc.) */
3294 fe->fe_tx_process(fsw, fe, flags);
3295
3296 KPKTQ_FINI(&fe->fe_tx_pktq);
3297 }
3298
3299 #if SK_LOG
3300 static void
dp_tx_log_pkt(uint64_t verb,char * desc,struct __kern_packet * pkt)3301 dp_tx_log_pkt(uint64_t verb, char *desc, struct __kern_packet *pkt)
3302 {
3303 char *pkt_buf;
3304 MD_BUFLET_ADDR_ABS(pkt, pkt_buf);
3305 SK_DF(verb, "%s(%d) %s %s", sk_proc_name_address(current_proc()),
3306 sk_proc_pid(current_proc()), desc, sk_dump("buf", pkt_buf,
3307 pkt->pkt_length, 128, NULL, 0));
3308 }
3309 #else /* !SK_LOG */
3310 #define dp_tx_log_pkt(...)
3311 #endif /* !SK_LOG */
3312
3313 static void
dp_tx_pktq(struct nx_flowswitch * fsw,struct pktq * spktq)3314 dp_tx_pktq(struct nx_flowswitch *fsw, struct pktq *spktq)
3315 {
3316 struct __kern_packet *spkt, *pkt;
3317 struct flow_entry_list fes = TAILQ_HEAD_INITIALIZER(fes);
3318 struct flow_entry *fe, *prev_fe;
3319 struct pktq dropped_pkts, dpktq;
3320 struct nexus_adapter *dev_na;
3321 struct kern_pbufpool *dev_pp;
3322 struct ifnet *ifp;
3323 sa_family_t af;
3324 uint32_t n_pkts, n_flows = 0;
3325 boolean_t do_pacing = FALSE;
3326
3327 int err;
3328 KPKTQ_INIT(&dpktq);
3329 KPKTQ_INIT(&dropped_pkts);
3330 n_pkts = KPKTQ_LEN(spktq);
3331
3332 FSW_RLOCK(fsw);
3333 if (__improbable(FSW_QUIESCED(fsw))) {
3334 DTRACE_SKYWALK1(tx__quiesced, struct nx_flowswitch *, fsw);
3335 SK_ERR("flowswitch detached, dropping %d pkts", n_pkts);
3336 KPKTQ_CONCAT(&dropped_pkts, spktq);
3337 goto done;
3338 }
3339 dev_na = fsw->fsw_dev_ch->ch_na;
3340 if (__improbable(dev_na == NULL)) {
3341 SK_ERR("dev port not attached, dropping %d pkts", n_pkts);
3342 FSW_STATS_ADD(FSW_STATS_DST_NXPORT_INACTIVE, n_pkts);
3343 KPKTQ_CONCAT(&dropped_pkts, spktq);
3344 goto done;
3345 }
3346 /*
3347 * fsw_ifp should still be valid at this point. If fsw is detached
3348 * after fsw_lock is released, this ifp will remain valid and
3349 * netif_transmit() will behave properly even if the ifp is in
3350 * detached state.
3351 */
3352 ifp = fsw->fsw_ifp;
3353
3354 /* batch allocate enough packets */
3355 dev_pp = na_kr_get_pp(dev_na, NR_TX);
3356
3357 err = pp_alloc_pktq(dev_pp, dev_pp->pp_max_frags, &dpktq, n_pkts, NULL,
3358 NULL, SKMEM_NOSLEEP);
3359 #if DEVELOPMENT || DEBUG
3360 if (__probable(err != ENOMEM)) {
3361 _FSW_INJECT_ERROR(12, err, ENOMEM, pp_free_pktq, &dpktq);
3362 }
3363 #endif /* DEVELOPMENT || DEBUG */
3364 if (__improbable(err == ENOMEM)) {
3365 ASSERT(KPKTQ_EMPTY(&dpktq));
3366 KPKTQ_CONCAT(&dropped_pkts, spktq);
3367 FSW_STATS_ADD(FSW_STATS_DROP_NOMEM_PKT, n_pkts);
3368 SK_ERR("failed to alloc %u pkts from device pool", n_pkts);
3369 goto done;
3370 } else if (__improbable(err == EAGAIN)) {
3371 FSW_STATS_ADD(FSW_STATS_DROP_NOMEM_PKT,
3372 (n_pkts - KPKTQ_LEN(&dpktq)));
3373 FSW_STATS_ADD(FSW_STATS_DROP,
3374 (n_pkts - KPKTQ_LEN(&dpktq)));
3375 }
3376
3377 n_pkts = KPKTQ_LEN(&dpktq);
3378 prev_fe = NULL;
3379 KPKTQ_FOREACH(spkt, spktq) {
3380 if (n_pkts == 0) {
3381 break;
3382 }
3383 --n_pkts;
3384
3385 KPKTQ_DEQUEUE(&dpktq, pkt);
3386 ASSERT(pkt != NULL);
3387 err = dp_copy_to_dev(fsw, spkt, pkt);
3388 if (__improbable(err != 0)) {
3389 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3390 continue;
3391 }
3392
3393 do_pacing |= ((pkt->pkt_pflags & PKT_F_OPT_TX_TIMESTAMP) != 0);
3394 af = fsw_ip_demux(fsw, pkt);
3395 if (__improbable(af == AF_UNSPEC)) {
3396 dp_tx_log_pkt(SK_VERB_ERROR, "demux err", pkt);
3397 FSW_STATS_INC(FSW_STATS_TX_DEMUX_ERR);
3398 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3399 continue;
3400 }
3401
3402 err = flow_pkt_classify(pkt, ifp, af, false);
3403 if (__improbable(err != 0)) {
3404 dp_tx_log_pkt(SK_VERB_ERROR, "flow extract err", pkt);
3405 FSW_STATS_INC(FSW_STATS_TX_FLOW_EXTRACT_ERR);
3406 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3407 continue;
3408 }
3409
3410 if (__improbable(pkt->pkt_flow_ip_is_frag &&
3411 !pkt->pkt_flow_ip_is_first_frag)) {
3412 fe = tx_process_continuous_ip_frag(fsw, prev_fe, pkt);
3413 if (__probable(fe != NULL)) {
3414 flow_entry_retain(fe);
3415 goto flow_batch;
3416 } else {
3417 FSW_STATS_INC(FSW_STATS_TX_FRAG_BAD_CONT);
3418 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3419 continue;
3420 }
3421 }
3422
3423 fe = tx_lookup_flow(fsw, pkt, prev_fe);
3424 if (__improbable(fe == NULL)) {
3425 FSW_STATS_INC(FSW_STATS_TX_FLOW_NOT_FOUND);
3426 KPKTQ_ENQUEUE(&dropped_pkts, pkt);
3427 prev_fe = NULL;
3428 continue;
3429 }
3430 flow_batch:
3431 tx_flow_batch_packet(&fes, fe, pkt);
3432 prev_fe = fe;
3433 }
3434
3435 struct flow_entry *tfe = NULL;
3436 TAILQ_FOREACH_SAFE(fe, &fes, fe_tx_link, tfe) {
3437 tx_flow_process(fsw, fe, 0);
3438 TAILQ_REMOVE(&fes, fe, fe_tx_link);
3439 fe->fe_tx_is_cont_frag = false;
3440 fe->fe_tx_frag_id = 0;
3441 flow_entry_release(&fe);
3442 n_flows++;
3443 }
3444
3445 done:
3446 FSW_RUNLOCK(fsw);
3447 if (n_flows > 0) {
3448 netif_transmit(ifp, NETIF_XMIT_FLAG_CHANNEL | (do_pacing ? NETIF_XMIT_FLAG_PACING : 0));
3449 }
3450 dp_drop_pktq(fsw, &dropped_pkts);
3451 KPKTQ_FINI(&dropped_pkts);
3452 KPKTQ_FINI(&dpktq);
3453 }
3454
3455 static sa_family_t
get_tso_af(struct __kern_packet * pkt)3456 get_tso_af(struct __kern_packet *pkt)
3457 {
3458 packet_tso_flags_t tso_flags;
3459
3460 tso_flags = pkt->pkt_csum_flags & PACKET_CSUM_TSO_FLAGS;
3461 if (tso_flags == PACKET_TSO_IPV4) {
3462 return AF_INET;
3463 } else if (tso_flags == PACKET_TSO_IPV6) {
3464 return AF_INET6;
3465 } else {
3466 panic("invalid tso flags: 0x%x\n", tso_flags);
3467 /* NOTREACHED */
3468 __builtin_unreachable();
3469 }
3470 }
3471
3472 static inline void
update_flow_info(struct __kern_packet * pkt,void * iphdr,void * tcphdr,uint16_t payload_sz)3473 update_flow_info(struct __kern_packet *pkt, void *iphdr, void *tcphdr,
3474 uint16_t payload_sz)
3475 {
3476 struct tcphdr *tcp = tcphdr;
3477
3478 DTRACE_SKYWALK4(update__flow__info, struct __kern_packet *, pkt,
3479 void *, iphdr, void *, tcphdr, uint16_t, payload_sz);
3480 pkt->pkt_flow_ip_hdr = (mach_vm_address_t)iphdr;
3481 pkt->pkt_flow_tcp_hdr = (mach_vm_address_t)tcphdr;
3482 pkt->pkt_flow_tcp_flags = tcp->th_flags;
3483 pkt->pkt_flow_tcp_seq = tcp->th_seq;
3484 pkt->pkt_flow_ulen = payload_sz;
3485 }
3486
3487 static int
do_gso(struct nx_flowswitch * fsw,int af,struct __kern_packet * orig_pkt,struct __kern_packet * first_pkt,struct pktq * dev_pktq,struct pktq * gso_pktq)3488 do_gso(struct nx_flowswitch *fsw, int af, struct __kern_packet *orig_pkt,
3489 struct __kern_packet *first_pkt, struct pktq *dev_pktq,
3490 struct pktq *gso_pktq)
3491 {
3492 ifnet_t ifp = fsw->fsw_ifp;
3493 struct __kern_packet *pkt = first_pkt;
3494 uint8_t proto = pkt->pkt_flow_ip_proto;
3495 uint16_t ip_hlen = pkt->pkt_flow_ip_hlen;
3496 uint16_t tcp_hlen = pkt->pkt_flow_tcp_hlen;
3497 uint16_t total_hlen = ip_hlen + tcp_hlen;
3498 uint16_t mtu = (uint16_t)ifp->if_mtu;
3499 uint16_t mss = pkt->pkt_proto_seg_sz, payload_sz;
3500 uint32_t n, n_pkts, off = 0, total_len = orig_pkt->pkt_length;
3501 uint16_t headroom = fsw->fsw_frame_headroom + ifp->if_tx_headroom;
3502 kern_packet_t orig_ph = SK_PKT2PH(orig_pkt);
3503 uint8_t *orig_pkt_baddr;
3504 struct tcphdr *tcp;
3505 struct ip *ip;
3506 struct ip6_hdr *ip6;
3507 uint32_t tcp_seq;
3508 uint16_t ipid;
3509 uint32_t pseudo_hdr_csum, bufsz;
3510
3511 ASSERT(headroom <= UINT8_MAX);
3512 if (proto != IPPROTO_TCP) {
3513 SK_ERR("invalid proto: %d", proto);
3514 DTRACE_SKYWALK3(invalid__proto, struct nx_flowswitch *,
3515 fsw, ifnet_t, ifp, uint8_t, proto);
3516 return EINVAL;
3517 }
3518 if (mss == 0 || mss > (mtu - total_hlen)) {
3519 SK_ERR("invalid args: mss %d, mtu %d, total_hlen %d",
3520 mss, mtu, total_hlen);
3521 DTRACE_SKYWALK5(invalid__args1, struct nx_flowswitch *,
3522 fsw, ifnet_t, ifp, uint16_t, mss, uint16_t, mtu,
3523 uint32_t, total_hlen);
3524 return EINVAL;
3525 }
3526 bufsz = PP_BUF_SIZE_DEF(pkt->pkt_qum.qum_pp);
3527 if ((headroom + total_hlen + mss) > bufsz) {
3528 SK_ERR("invalid args: headroom %d, total_hlen %d, "
3529 "mss %d, bufsz %d", headroom, total_hlen, mss, bufsz);
3530 DTRACE_SKYWALK6(invalid__args2, struct nx_flowswitch *,
3531 fsw, ifnet_t, ifp, uint16_t, headroom, uint16_t,
3532 total_hlen, uint16_t, mss, uint32_t, bufsz);
3533 return EINVAL;
3534 }
3535 n_pkts = (uint32_t)(SK_ROUNDUP((total_len - total_hlen), mss) / mss);
3536
3537 ASSERT(pkt->pkt_headroom == headroom);
3538 ASSERT(pkt->pkt_length == total_len);
3539 ASSERT(pkt->pkt_l2_len == 0);
3540 ASSERT((pkt->pkt_qum.qum_qflags & QUM_F_FINALIZED) == 0);
3541 ASSERT((pkt->pkt_pflags & PKT_F_TRUNCATED) != 0);
3542 pkt->pkt_pflags &= ~PKT_F_TRUNCATED;
3543 pkt->pkt_proto_seg_sz = 0;
3544 pkt->pkt_csum_flags = 0;
3545 MD_BUFLET_ADDR_ABS(orig_pkt, orig_pkt_baddr);
3546 orig_pkt_baddr += orig_pkt->pkt_headroom;
3547
3548 if (af == AF_INET) {
3549 ip = (struct ip *)pkt->pkt_flow_ip_hdr;
3550 tcp = (struct tcphdr *)pkt->pkt_flow_tcp_hdr;
3551 ipid = ip->ip_id;
3552 pseudo_hdr_csum = in_pseudo(pkt->pkt_flow_ipv4_src.s_addr,
3553 pkt->pkt_flow_ipv4_dst.s_addr, 0);
3554 } else {
3555 ASSERT(af == AF_INET6);
3556 tcp = (struct tcphdr *)pkt->pkt_flow_tcp_hdr;
3557 pseudo_hdr_csum = in6_pseudo(&pkt->pkt_flow_ipv6_src,
3558 &pkt->pkt_flow_ipv6_dst, 0);
3559 }
3560 tcp_seq = ntohl(tcp->th_seq);
3561
3562 for (n = 1, payload_sz = mss, off = total_hlen; off < total_len;
3563 off += payload_sz) {
3564 uint8_t *baddr, *baddr0;
3565 uint32_t partial;
3566
3567 if (pkt == NULL) {
3568 n++;
3569 KPKTQ_DEQUEUE(dev_pktq, pkt);
3570 ASSERT(pkt != NULL);
3571 }
3572 MD_BUFLET_ADDR_ABS(pkt, baddr0);
3573 baddr = baddr0;
3574 baddr += headroom;
3575
3576 /* Copy headers from the original packet */
3577 if (n != 1) {
3578 ASSERT(pkt != first_pkt);
3579 pkt_copy(orig_pkt_baddr, baddr, total_hlen);
3580 fsw_pkt_copy_metadata(first_pkt, pkt);
3581
3582 ASSERT((pkt->pkt_qum_qflags & QUM_F_FLOW_CLASSIFIED) != 0);
3583 /* flow info still needs to be updated below */
3584 bcopy(first_pkt->pkt_flow, pkt->pkt_flow,
3585 sizeof(*pkt->pkt_flow));
3586 pkt->pkt_trace_id = 0;
3587 ASSERT(pkt->pkt_headroom == headroom);
3588 } else {
3589 METADATA_SET_LEN(pkt, 0, 0);
3590 }
3591 baddr += total_hlen;
3592
3593 /* Copy/checksum the payload from the original packet */
3594 if (off + payload_sz > total_len) {
3595 payload_sz = (uint16_t)(total_len - off);
3596 }
3597 pkt_copypkt_sum(orig_ph,
3598 (uint16_t)(orig_pkt->pkt_headroom + off),
3599 SK_PKT2PH(pkt), headroom + total_hlen, payload_sz,
3600 &partial, TRUE);
3601
3602 DTRACE_SKYWALK6(copy__csum, struct nx_flowswitch *, fsw,
3603 ifnet_t, ifp, uint8_t *, baddr, uint16_t, payload_sz,
3604 uint16_t, mss, uint32_t, partial);
3605 FSW_STATS_INC(FSW_STATS_TX_COPY_PKT2PKT);
3606
3607 /*
3608 * Adjust header information and fill in the missing fields.
3609 */
3610 if (af == AF_INET) {
3611 ip = (struct ip *)(void *)(baddr0 + pkt->pkt_headroom);
3612 tcp = (struct tcphdr *)(void *)((caddr_t)ip + ip_hlen);
3613
3614 if (n != n_pkts) {
3615 tcp->th_flags &= ~(TH_FIN | TH_PUSH);
3616 }
3617 if (n != 1) {
3618 tcp->th_flags &= ~TH_CWR;
3619 tcp->th_seq = htonl(tcp_seq);
3620 }
3621 update_flow_info(pkt, ip, tcp, payload_sz);
3622
3623 ip->ip_id = htons((ipid)++);
3624 ip->ip_len = htons(ip_hlen + tcp_hlen + payload_sz);
3625 ip->ip_sum = 0;
3626 ip->ip_sum = inet_cksum_buffer(ip, 0, 0, ip_hlen);
3627 tcp->th_sum = 0;
3628 partial = __packet_cksum(tcp, tcp_hlen, partial);
3629 partial += htons(tcp_hlen + IPPROTO_TCP + payload_sz);
3630 partial += pseudo_hdr_csum;
3631 ADDCARRY(partial);
3632 tcp->th_sum = ~(uint16_t)partial;
3633 } else {
3634 ASSERT(af == AF_INET6);
3635 ip6 = (struct ip6_hdr *)(baddr0 + pkt->pkt_headroom);
3636 tcp = (struct tcphdr *)(void *)((caddr_t)ip6 + ip_hlen);
3637
3638 if (n != n_pkts) {
3639 tcp->th_flags &= ~(TH_FIN | TH_PUSH);
3640 }
3641 if (n != 1) {
3642 tcp->th_flags &= ~TH_CWR;
3643 tcp->th_seq = htonl(tcp_seq);
3644 }
3645 update_flow_info(pkt, ip6, tcp, payload_sz);
3646
3647 ip6->ip6_plen = htons(tcp_hlen + payload_sz);
3648 tcp->th_sum = 0;
3649 partial = __packet_cksum(tcp, tcp_hlen, partial);
3650 partial += htonl(tcp_hlen + IPPROTO_TCP + payload_sz);
3651 partial += pseudo_hdr_csum;
3652 ADDCARRY(partial);
3653 tcp->th_sum = ~(uint16_t)partial;
3654 }
3655 tcp_seq += payload_sz;
3656 METADATA_ADJUST_LEN(pkt, total_hlen, headroom);
3657 #if (DEVELOPMENT || DEBUG)
3658 struct __kern_buflet *bft;
3659 uint32_t blen;
3660 PKT_GET_FIRST_BUFLET(pkt, 1, bft);
3661 blen = __buflet_get_data_length(bft);
3662 if (blen != total_hlen + payload_sz) {
3663 panic("blen (%d) != total_len + payload_sz (%d)\n",
3664 blen, total_hlen + payload_sz);
3665 }
3666 #endif /* DEVELOPMENT || DEBUG */
3667
3668 pkt->pkt_length = total_hlen + payload_sz;
3669 KPKTQ_ENQUEUE(gso_pktq, pkt);
3670 pkt = NULL;
3671
3672 /*
3673 * Note that at this point the packet is not yet finalized.
3674 * The finalization happens in dp_flow_tx_process() after
3675 * the framing is done.
3676 */
3677 }
3678 ASSERT(n == n_pkts);
3679 ASSERT(off == total_len);
3680 DTRACE_SKYWALK7(gso__done, struct nx_flowswitch *, fsw, ifnet_t, ifp,
3681 uint32_t, n_pkts, uint32_t, total_len, uint16_t, ip_hlen,
3682 uint16_t, tcp_hlen, uint8_t *, orig_pkt_baddr);
3683 return 0;
3684 }
3685
3686 static void
tx_flow_enqueue_gso_pktq(struct flow_entry_list * fes,struct flow_entry * fe,struct pktq * gso_pktq)3687 tx_flow_enqueue_gso_pktq(struct flow_entry_list *fes, struct flow_entry *fe,
3688 struct pktq *gso_pktq)
3689 {
3690 if (KPKTQ_EMPTY(&fe->fe_tx_pktq)) {
3691 ASSERT(KPKTQ_LEN(&fe->fe_tx_pktq) == 0);
3692 TAILQ_INSERT_TAIL(fes, fe, fe_tx_link);
3693 KPKTQ_ENQUEUE_MULTI(&fe->fe_tx_pktq, KPKTQ_FIRST(gso_pktq),
3694 KPKTQ_LAST(gso_pktq), KPKTQ_LEN(gso_pktq));
3695 KPKTQ_INIT(gso_pktq);
3696 } else {
3697 ASSERT(!TAILQ_EMPTY(fes));
3698 KPKTQ_ENQUEUE_MULTI(&fe->fe_tx_pktq, KPKTQ_FIRST(gso_pktq),
3699 KPKTQ_LAST(gso_pktq), KPKTQ_LEN(gso_pktq));
3700 KPKTQ_INIT(gso_pktq);
3701 flow_entry_release(&fe);
3702 }
3703 }
3704
3705 static void
dp_gso_pktq(struct nx_flowswitch * fsw,struct pktq * spktq,uint32_t gso_pkts_estimate)3706 dp_gso_pktq(struct nx_flowswitch *fsw, struct pktq *spktq,
3707 uint32_t gso_pkts_estimate)
3708 {
3709 struct __kern_packet *spkt, *pkt;
3710 struct flow_entry_list fes = TAILQ_HEAD_INITIALIZER(fes);
3711 struct flow_entry *fe, *prev_fe;
3712 struct pktq dpktq;
3713 struct nexus_adapter *dev_na;
3714 struct kern_pbufpool *dev_pp;
3715 struct ifnet *ifp;
3716 sa_family_t af;
3717 uint32_t n_pkts, n_flows = 0;
3718 int err;
3719
3720 KPKTQ_INIT(&dpktq);
3721 n_pkts = KPKTQ_LEN(spktq);
3722
3723 FSW_RLOCK(fsw);
3724 if (__improbable(FSW_QUIESCED(fsw))) {
3725 DTRACE_SKYWALK1(tx__quiesced, struct nx_flowswitch *, fsw);
3726 SK_ERR("flowswitch detached, dropping %d pkts", n_pkts);
3727 dp_drop_pktq(fsw, spktq);
3728 goto done;
3729 }
3730 dev_na = fsw->fsw_dev_ch->ch_na;
3731 if (__improbable(dev_na == NULL)) {
3732 SK_ERR("dev port not attached, dropping %d pkts", n_pkts);
3733 FSW_STATS_ADD(FSW_STATS_DST_NXPORT_INACTIVE, n_pkts);
3734 dp_drop_pktq(fsw, spktq);
3735 goto done;
3736 }
3737 /*
3738 * fsw_ifp should still be valid at this point. If fsw is detached
3739 * after fsw_lock is released, this ifp will remain valid and
3740 * netif_transmit() will behave properly even if the ifp is in
3741 * detached state.
3742 */
3743 ifp = fsw->fsw_ifp;
3744 dev_pp = na_kr_get_pp(dev_na, NR_TX);
3745
3746 /*
3747 * Batch allocate enough packets to perform GSO on all
3748 * packets in spktq.
3749 */
3750 err = pp_alloc_pktq(dev_pp, dev_pp->pp_max_frags, &dpktq,
3751 gso_pkts_estimate, NULL, NULL, SKMEM_NOSLEEP);
3752 #if DEVELOPMENT || DEBUG
3753 if (__probable(err != ENOMEM)) {
3754 _FSW_INJECT_ERROR(12, err, ENOMEM, pp_free_pktq, &dpktq);
3755 }
3756 #endif /* DEVELOPMENT || DEBUG */
3757 /*
3758 * We either get all packets or none. No partial allocations.
3759 */
3760 if (__improbable(err != 0)) {
3761 if (err == ENOMEM) {
3762 ASSERT(KPKTQ_EMPTY(&dpktq));
3763 } else {
3764 dp_free_pktq(fsw, &dpktq);
3765 }
3766 DTRACE_SKYWALK1(gso__no__mem, int, err);
3767 dp_drop_pktq(fsw, spktq);
3768 FSW_STATS_ADD(FSW_STATS_DROP_NOMEM_PKT, n_pkts);
3769 SK_ERR("failed to alloc %u pkts from device pool",
3770 gso_pkts_estimate);
3771 goto done;
3772 }
3773 prev_fe = NULL;
3774 KPKTQ_FOREACH(spkt, spktq) {
3775 KPKTQ_DEQUEUE(&dpktq, pkt);
3776 ASSERT(pkt != NULL);
3777 /*
3778 * Copy only headers to the first packet of the GSO chain.
3779 * The headers will be used for classification below.
3780 */
3781 err = dp_copy_headers_to_dev(fsw, spkt, pkt);
3782 if (__improbable(err != 0)) {
3783 pp_free_packet_single(pkt);
3784 DTRACE_SKYWALK2(copy__headers__failed,
3785 struct nx_flowswitch *, fsw,
3786 struct __kern_packet *, spkt);
3787 continue;
3788 }
3789 af = get_tso_af(pkt);
3790 ASSERT(af == AF_INET || af == AF_INET6);
3791
3792 err = flow_pkt_classify(pkt, ifp, af, false);
3793 if (__improbable(err != 0)) {
3794 dp_tx_log_pkt(SK_VERB_ERROR, "flow extract err", pkt);
3795 FSW_STATS_INC(FSW_STATS_TX_FLOW_EXTRACT_ERR);
3796 pp_free_packet_single(pkt);
3797 DTRACE_SKYWALK4(classify__failed,
3798 struct nx_flowswitch *, fsw,
3799 struct __kern_packet *, spkt,
3800 struct __kern_packet *, pkt,
3801 int, err);
3802 continue;
3803 }
3804 /*
3805 * GSO cannot be done on a fragment and it's a bug in user
3806 * space to mark a fragment as needing GSO.
3807 */
3808 if (__improbable(pkt->pkt_flow_ip_is_frag)) {
3809 FSW_STATS_INC(FSW_STATS_TX_FRAG_BAD_CONT);
3810 pp_free_packet_single(pkt);
3811 DTRACE_SKYWALK3(is__frag,
3812 struct nx_flowswitch *, fsw,
3813 struct __kern_packet *, spkt,
3814 struct __kern_packet *, pkt);
3815 continue;
3816 }
3817 fe = tx_lookup_flow(fsw, pkt, prev_fe);
3818 if (__improbable(fe == NULL)) {
3819 FSW_STATS_INC(FSW_STATS_TX_FLOW_NOT_FOUND);
3820 pp_free_packet_single(pkt);
3821 DTRACE_SKYWALK3(lookup__failed,
3822 struct nx_flowswitch *, fsw,
3823 struct __kern_packet *, spkt,
3824 struct __kern_packet *, pkt);
3825 prev_fe = NULL;
3826 continue;
3827 }
3828 /*
3829 * Perform GSO on spkt using the flow information
3830 * obtained above.
3831 */
3832 struct pktq gso_pktq;
3833 KPKTQ_INIT(&gso_pktq);
3834 err = do_gso(fsw, af, spkt, pkt, &dpktq, &gso_pktq);
3835 if (__probable(err == 0)) {
3836 tx_flow_enqueue_gso_pktq(&fes, fe, &gso_pktq);
3837 prev_fe = fe;
3838 } else {
3839 DTRACE_SKYWALK1(gso__error, int, err);
3840 /* TODO: increment error stat */
3841 pp_free_packet_single(pkt);
3842 flow_entry_release(&fe);
3843 prev_fe = NULL;
3844 }
3845 KPKTQ_FINI(&gso_pktq);
3846 }
3847 struct flow_entry *tfe = NULL;
3848 TAILQ_FOREACH_SAFE(fe, &fes, fe_tx_link, tfe) {
3849 /* Chain-enqueue can be used for GSO chains */
3850 tx_flow_process(fsw, fe, FLOW_PROC_FLAG_GSO);
3851 TAILQ_REMOVE(&fes, fe, fe_tx_link);
3852 flow_entry_release(&fe);
3853 n_flows++;
3854 }
3855 done:
3856 FSW_RUNLOCK(fsw);
3857 if (n_flows > 0) {
3858 netif_transmit(ifp, NETIF_XMIT_FLAG_CHANNEL);
3859 }
3860
3861 /*
3862 * It's possible for packets to be left in dpktq because
3863 * gso_pkts_estimate is only an estimate. The actual number
3864 * of packets needed could be less.
3865 */
3866 uint32_t dpktq_len;
3867 if ((dpktq_len = KPKTQ_LEN(&dpktq)) > 0) {
3868 DTRACE_SKYWALK2(leftover__dev__pkts,
3869 struct nx_flowswitch *, fsw, uint32_t, dpktq_len);
3870 dp_free_pktq(fsw, &dpktq);
3871 }
3872 KPKTQ_FINI(&dpktq);
3873 }
3874
3875 static inline void
fsw_dev_ring_flush(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,struct proc * p)3876 fsw_dev_ring_flush(struct nx_flowswitch *fsw, struct __kern_channel_ring *r,
3877 struct proc *p)
3878 {
3879 #pragma unused(p)
3880 uint32_t total_pkts = 0, total_bytes = 0;
3881
3882 for (;;) {
3883 struct pktq pktq;
3884 KPKTQ_INIT(&pktq);
3885 uint32_t n_bytes;
3886 fsw_rx_ring_dequeue_pktq(fsw, r, fsw_rx_batch, &pktq, &n_bytes);
3887 if (n_bytes == 0) {
3888 break;
3889 }
3890 total_pkts += KPKTQ_LEN(&pktq);
3891 total_bytes += n_bytes;
3892
3893 if (__probable(fsw->fsw_ifp->if_input_netem == NULL)) {
3894 fsw_receive(fsw, &pktq);
3895 } else {
3896 fsw_dev_input_netem_enqueue(fsw, &pktq);
3897 }
3898 KPKTQ_FINI(&pktq);
3899 }
3900
3901 KDBG(SK_KTRACE_FSW_DEV_RING_FLUSH, SK_KVA(r), total_pkts, total_bytes);
3902 DTRACE_SKYWALK2(fsw__dp__dev__ring__flush, uint32_t, total_pkts,
3903 uint32_t, total_bytes);
3904
3905 /* compute mitigation rate for delivered traffic */
3906 if (__probable(r->ckr_netif_mit_stats != NULL)) {
3907 r->ckr_netif_mit_stats(r, total_pkts, total_bytes);
3908 }
3909 }
3910
3911 static inline void
fsw_user_ring_flush(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,struct proc * p)3912 fsw_user_ring_flush(struct nx_flowswitch *fsw, struct __kern_channel_ring *r,
3913 struct proc *p)
3914 {
3915 #pragma unused(p)
3916 static packet_trace_id_t trace_id = 0;
3917 uint32_t total_pkts = 0, total_bytes = 0;
3918
3919 for (;;) {
3920 struct pktq pktq;
3921 KPKTQ_INIT(&pktq);
3922 uint32_t n_bytes;
3923 uint32_t gso_pkts_estimate = 0;
3924
3925 fsw_tx_ring_dequeue_pktq(fsw, r, fsw_tx_batch, &pktq, &n_bytes,
3926 &gso_pkts_estimate);
3927 if (n_bytes == 0) {
3928 break;
3929 }
3930 total_pkts += KPKTQ_LEN(&pktq);
3931 total_bytes += n_bytes;
3932
3933 KPKTQ_FIRST(&pktq)->pkt_trace_id = ++trace_id;
3934 KDBG(SK_KTRACE_PKT_TX_FSW | DBG_FUNC_START,
3935 KPKTQ_FIRST(&pktq)->pkt_trace_id);
3936
3937 if (gso_pkts_estimate > 0) {
3938 dp_gso_pktq(fsw, &pktq, gso_pkts_estimate);
3939 } else {
3940 dp_tx_pktq(fsw, &pktq);
3941 }
3942 dp_free_pktq(fsw, &pktq);
3943 KPKTQ_FINI(&pktq);
3944 }
3945 kr_update_stats(r, total_pkts, total_bytes);
3946
3947 KDBG(SK_KTRACE_FSW_USER_RING_FLUSH, SK_KVA(r), total_pkts, total_bytes);
3948 DTRACE_SKYWALK2(fsw__dp__user__ring__flush, uint32_t, total_pkts,
3949 uint32_t, total_bytes);
3950 }
3951
3952 void
fsw_ring_flush(struct nx_flowswitch * fsw,struct __kern_channel_ring * r,struct proc * p)3953 fsw_ring_flush(struct nx_flowswitch *fsw, struct __kern_channel_ring *r,
3954 struct proc *p)
3955 {
3956 struct nexus_vp_adapter *vpna = VPNA(KRNA(r));
3957
3958 ASSERT(sk_is_sync_protected());
3959 ASSERT(vpna->vpna_nx_port != FSW_VP_HOST);
3960 ASSERT(vpna->vpna_up.na_md_type == NEXUS_META_TYPE_PACKET);
3961
3962 if (vpna->vpna_nx_port == FSW_VP_DEV) {
3963 fsw_dev_ring_flush(fsw, r, p);
3964 } else {
3965 fsw_user_ring_flush(fsw, r, p);
3966 }
3967 }
3968
3969 int
fsw_dp_ctor(struct nx_flowswitch * fsw)3970 fsw_dp_ctor(struct nx_flowswitch *fsw)
3971 {
3972 uint32_t fe_cnt = fsw_fe_table_size;
3973 uint32_t fob_cnt = fsw_flow_owner_buckets;
3974 uint32_t frb_cnt = fsw_flow_route_buckets;
3975 uint32_t frib_cnt = fsw_flow_route_id_buckets;
3976 struct kern_nexus *nx = fsw->fsw_nx;
3977 char name[64];
3978 int error = 0;
3979
3980 /* just in case */
3981 if (fe_cnt == 0) {
3982 fe_cnt = NX_FSW_FE_TABLESZ;
3983 ASSERT(fe_cnt != 0);
3984 }
3985 if (fob_cnt == 0) {
3986 fob_cnt = NX_FSW_FOB_HASHSZ;
3987 ASSERT(fob_cnt != 0);
3988 }
3989 if (frb_cnt == 0) {
3990 frb_cnt = NX_FSW_FRB_HASHSZ;
3991 ASSERT(frb_cnt != 0);
3992 }
3993 if (frib_cnt == 0) {
3994 frib_cnt = NX_FSW_FRIB_HASHSZ;
3995 ASSERT(frib_cnt != 0);
3996 }
3997
3998 /* make sure fe_cnt is a power of two, else round up */
3999 if ((fe_cnt & (fe_cnt - 1)) != 0) {
4000 fe_cnt--;
4001 fe_cnt |= (fe_cnt >> 1);
4002 fe_cnt |= (fe_cnt >> 2);
4003 fe_cnt |= (fe_cnt >> 4);
4004 fe_cnt |= (fe_cnt >> 8);
4005 fe_cnt |= (fe_cnt >> 16);
4006 fe_cnt++;
4007 }
4008
4009 /* make sure frb_cnt is a power of two, else round up */
4010 if ((frb_cnt & (frb_cnt - 1)) != 0) {
4011 frb_cnt--;
4012 frb_cnt |= (frb_cnt >> 1);
4013 frb_cnt |= (frb_cnt >> 2);
4014 frb_cnt |= (frb_cnt >> 4);
4015 frb_cnt |= (frb_cnt >> 8);
4016 frb_cnt |= (frb_cnt >> 16);
4017 frb_cnt++;
4018 }
4019
4020 lck_mtx_init(&fsw->fsw_detach_barrier_lock, &nexus_lock_group,
4021 &nexus_lock_attr);
4022 lck_mtx_init(&fsw->fsw_reap_lock, &nexus_lock_group, &nexus_lock_attr);
4023 lck_mtx_init(&fsw->fsw_linger_lock, &nexus_lock_group, &nexus_lock_attr);
4024 TAILQ_INIT(&fsw->fsw_linger_head);
4025
4026 (void) snprintf(name, sizeof(name), "%s_%llu", NX_FSW_NAME, nx->nx_id);
4027 error = nx_advisory_alloc(nx, name,
4028 &NX_PROV(nx)->nxprov_region_params[SKMEM_REGION_NEXUSADV],
4029 NEXUS_ADVISORY_TYPE_FLOWSWITCH);
4030 if (error != 0) {
4031 fsw_dp_dtor(fsw);
4032 return error;
4033 }
4034
4035 fsw->fsw_flow_mgr = flow_mgr_create(fe_cnt, fob_cnt, frb_cnt, frib_cnt);
4036 if (fsw->fsw_flow_mgr == NULL) {
4037 fsw_dp_dtor(fsw);
4038 return error;
4039 }
4040
4041 /* generic name; will be customized upon ifattach */
4042 (void) snprintf(fsw->fsw_reap_name, sizeof(fsw->fsw_reap_name),
4043 FSW_REAP_THREADNAME, name, "");
4044
4045 if (kernel_thread_start(fsw_reap_thread_func, fsw,
4046 &fsw->fsw_reap_thread) != KERN_SUCCESS) {
4047 panic_plain("%s: can't create thread", __func__);
4048 /* NOTREACHED */
4049 __builtin_unreachable();
4050 }
4051 /* this must not fail */
4052 VERIFY(fsw->fsw_reap_thread != NULL);
4053
4054 SK_DF(SK_VERB_MEM, "fsw 0x%llx ALLOC", SK_KVA(fsw));
4055
4056
4057 return error;
4058 }
4059
4060 void
fsw_dp_dtor(struct nx_flowswitch * fsw)4061 fsw_dp_dtor(struct nx_flowswitch *fsw)
4062 {
4063 uint64_t f = (1 * NSEC_PER_MSEC); /* 1 ms */
4064 uint64_t s = (1000 * NSEC_PER_SEC); /* 1 sec */
4065 uint32_t i = 0;
4066
4067 #if (DEVELOPMENT || DEBUG)
4068 if (fsw->fsw_rps_threads != NULL) {
4069 for (i = 0; i < fsw->fsw_rps_nthreads; i++) {
4070 fsw_rps_thread_join(fsw, i);
4071 }
4072 kfree_type(struct fsw_rps_thread, fsw->fsw_rps_threads);
4073 }
4074 #endif /* !DEVELOPMENT && !DEBUG */
4075
4076 nx_advisory_free(fsw->fsw_nx);
4077
4078 if (fsw->fsw_reap_thread != THREAD_NULL) {
4079 /* signal thread to begin self-termination */
4080 lck_mtx_lock(&fsw->fsw_reap_lock);
4081 fsw->fsw_reap_flags |= FSW_REAPF_TERMINATING;
4082
4083 /*
4084 * And wait for thread to terminate; use another
4085 * wait channel here other than fsw_reap_flags to
4086 * make it more explicit. In the event the reaper
4087 * thread misses a wakeup, we'll try again once
4088 * every second (except for the first time).
4089 */
4090 while (!(fsw->fsw_reap_flags & FSW_REAPF_TERMINATED)) {
4091 uint64_t t = 0;
4092
4093 nanoseconds_to_absolutetime((i++ == 0) ? f : s, &t);
4094 clock_absolutetime_interval_to_deadline(t, &t);
4095 ASSERT(t != 0);
4096
4097 fsw->fsw_reap_flags |= FSW_REAPF_TERMINATEBLOCK;
4098 if (!(fsw->fsw_reap_flags & FSW_REAPF_RUNNING)) {
4099 thread_wakeup((caddr_t)&fsw->fsw_reap_flags);
4100 }
4101 (void) assert_wait_deadline(&fsw->fsw_reap_thread,
4102 THREAD_UNINT, t);
4103 lck_mtx_unlock(&fsw->fsw_reap_lock);
4104 thread_block(THREAD_CONTINUE_NULL);
4105 lck_mtx_lock(&fsw->fsw_reap_lock);
4106 fsw->fsw_reap_flags &= ~FSW_REAPF_TERMINATEBLOCK;
4107 }
4108 ASSERT(fsw->fsw_reap_flags & FSW_REAPF_TERMINATED);
4109 lck_mtx_unlock(&fsw->fsw_reap_lock);
4110 fsw->fsw_reap_thread = THREAD_NULL;
4111 }
4112
4113 /* free any remaining flow entries in the linger list */
4114 fsw_linger_purge(fsw);
4115
4116 if (fsw->fsw_flow_mgr != NULL) {
4117 flow_mgr_destroy(fsw->fsw_flow_mgr);
4118 fsw->fsw_flow_mgr = NULL;
4119 }
4120
4121
4122 lck_mtx_destroy(&fsw->fsw_detach_barrier_lock, &nexus_lock_group);
4123 lck_mtx_destroy(&fsw->fsw_reap_lock, &nexus_lock_group);
4124 lck_mtx_destroy(&fsw->fsw_linger_lock, &nexus_lock_group);
4125 }
4126
4127 void
fsw_linger_insert(struct flow_entry * fe)4128 fsw_linger_insert(struct flow_entry *fe)
4129 {
4130 struct nx_flowswitch *fsw = fe->fe_fsw;
4131 SK_LOG_VAR(char dbgbuf[FLOWENTRY_DBGBUF_SIZE]);
4132 SK_DF(SK_VERB_FLOW, "entry \"%s\" fe 0x%llx flags 0x%b",
4133 fe_as_string(fe, dbgbuf, sizeof(dbgbuf)), SK_KVA(fe),
4134 fe->fe_flags, FLOWENTF_BITS);
4135
4136 net_update_uptime();
4137
4138 ASSERT(flow_entry_refcnt(fe) >= 1);
4139 ASSERT(fe->fe_flags & FLOWENTF_TORN_DOWN);
4140 ASSERT(fe->fe_flags & FLOWENTF_DESTROYED);
4141 ASSERT(!(fe->fe_flags & FLOWENTF_LINGERING));
4142 ASSERT(fe->fe_flags & FLOWENTF_WAIT_CLOSE);
4143 ASSERT(fe->fe_linger_wait != 0);
4144 fe->fe_linger_expire = (_net_uptime + fe->fe_linger_wait);
4145 os_atomic_or(&fe->fe_flags, FLOWENTF_LINGERING, relaxed);
4146
4147 lck_mtx_lock_spin(&fsw->fsw_linger_lock);
4148 TAILQ_INSERT_TAIL(&fsw->fsw_linger_head, fe, fe_linger_link);
4149 fsw->fsw_linger_cnt++;
4150 VERIFY(fsw->fsw_linger_cnt != 0);
4151 lck_mtx_unlock(&fsw->fsw_linger_lock);
4152
4153 fsw_reap_sched(fsw);
4154 }
4155
4156 static void
fsw_linger_remove_internal(struct flow_entry_linger_head * linger_head,struct flow_entry * fe)4157 fsw_linger_remove_internal(struct flow_entry_linger_head *linger_head,
4158 struct flow_entry *fe)
4159 {
4160 SK_LOG_VAR(char dbgbuf[FLOWENTRY_DBGBUF_SIZE]);
4161 SK_DF(SK_VERB_FLOW, "entry \"%s\" fe 0x%llx flags 0x%b",
4162 fe_as_string(fe, dbgbuf, sizeof(dbgbuf)), SK_KVA(fe),
4163 fe->fe_flags, FLOWENTF_BITS);
4164
4165 ASSERT(fe->fe_flags & FLOWENTF_TORN_DOWN);
4166 ASSERT(fe->fe_flags & FLOWENTF_DESTROYED);
4167 ASSERT(fe->fe_flags & FLOWENTF_LINGERING);
4168 os_atomic_andnot(&fe->fe_flags, FLOWENTF_LINGERING, relaxed);
4169
4170 TAILQ_REMOVE(linger_head, fe, fe_linger_link);
4171 flow_entry_release(&fe);
4172 }
4173
4174 static void
fsw_linger_remove(struct flow_entry * fe)4175 fsw_linger_remove(struct flow_entry *fe)
4176 {
4177 struct nx_flowswitch *fsw = fe->fe_fsw;
4178
4179 LCK_MTX_ASSERT(&fsw->fsw_linger_lock, LCK_MTX_ASSERT_OWNED);
4180
4181 fsw_linger_remove_internal(&fsw->fsw_linger_head, fe);
4182 VERIFY(fsw->fsw_linger_cnt != 0);
4183 fsw->fsw_linger_cnt--;
4184 }
4185
4186 void
fsw_linger_purge(struct nx_flowswitch * fsw)4187 fsw_linger_purge(struct nx_flowswitch *fsw)
4188 {
4189 struct flow_entry *fe, *tfe;
4190
4191 lck_mtx_lock(&fsw->fsw_linger_lock);
4192 TAILQ_FOREACH_SAFE(fe, &fsw->fsw_linger_head, fe_linger_link, tfe) {
4193 fsw_linger_remove(fe);
4194 }
4195 ASSERT(fsw->fsw_linger_cnt == 0);
4196 ASSERT(TAILQ_EMPTY(&fsw->fsw_linger_head));
4197 lck_mtx_unlock(&fsw->fsw_linger_lock);
4198 }
4199
4200 void
fsw_reap_sched(struct nx_flowswitch * fsw)4201 fsw_reap_sched(struct nx_flowswitch *fsw)
4202 {
4203 ASSERT(fsw->fsw_reap_thread != THREAD_NULL);
4204 lck_mtx_lock_spin(&fsw->fsw_reap_lock);
4205 if (!(fsw->fsw_reap_flags & FSW_REAPF_RUNNING) &&
4206 !(fsw->fsw_reap_flags & (FSW_REAPF_TERMINATING | FSW_REAPF_TERMINATED))) {
4207 thread_wakeup((caddr_t)&fsw->fsw_reap_flags);
4208 }
4209 lck_mtx_unlock(&fsw->fsw_reap_lock);
4210 }
4211
4212 __attribute__((noreturn))
4213 static void
fsw_reap_thread_func(void * v,wait_result_t w)4214 fsw_reap_thread_func(void *v, wait_result_t w)
4215 {
4216 #pragma unused(w)
4217 struct nx_flowswitch *fsw = v;
4218
4219 ASSERT(fsw->fsw_reap_thread == current_thread());
4220 thread_set_thread_name(current_thread(), fsw->fsw_reap_name);
4221
4222 net_update_uptime();
4223
4224 lck_mtx_lock(&fsw->fsw_reap_lock);
4225 VERIFY(!(fsw->fsw_reap_flags & FSW_REAPF_RUNNING));
4226 (void) assert_wait(&fsw->fsw_reap_flags, THREAD_UNINT);
4227 lck_mtx_unlock(&fsw->fsw_reap_lock);
4228 thread_block_parameter(fsw_reap_thread_cont, fsw);
4229 /* NOTREACHED */
4230 __builtin_unreachable();
4231 }
4232
4233 __attribute__((noreturn))
4234 static void
fsw_reap_thread_cont(void * v,wait_result_t wres)4235 fsw_reap_thread_cont(void *v, wait_result_t wres)
4236 {
4237 struct nx_flowswitch *fsw = v;
4238 boolean_t low;
4239 uint64_t t = 0;
4240
4241 SK_DF(SK_VERB_FLOW, "%s: running", fsw->fsw_reap_name);
4242
4243 lck_mtx_lock(&fsw->fsw_reap_lock);
4244 if (__improbable(wres == THREAD_INTERRUPTED ||
4245 (fsw->fsw_reap_flags & FSW_REAPF_TERMINATING) != 0)) {
4246 goto terminate;
4247 }
4248
4249 ASSERT(!(fsw->fsw_reap_flags & FSW_REAPF_TERMINATED));
4250 fsw->fsw_reap_flags |= FSW_REAPF_RUNNING;
4251 lck_mtx_unlock(&fsw->fsw_reap_lock);
4252
4253 net_update_uptime();
4254
4255 /* prevent detach from happening while we're here */
4256 if (!fsw_detach_barrier_add(fsw)) {
4257 SK_ERR("%s: netagent detached", fsw->fsw_reap_name);
4258 t = 0;
4259 } else {
4260 uint32_t fe_nonviable, fe_freed, fe_aborted;
4261 uint32_t fr_freed, fr_resid = 0;
4262 struct ifnet *ifp = fsw->fsw_ifp;
4263 uint64_t i = FSW_REAP_IVAL;
4264 uint64_t now = _net_uptime;
4265 uint64_t last;
4266
4267 ASSERT(fsw->fsw_ifp != NULL);
4268
4269 /*
4270 * Pass 1: process any deferred {withdrawn,nonviable} requests.
4271 */
4272 fe_nonviable = fsw_process_deferred(fsw);
4273
4274 /*
4275 * Pass 2: remove any expired lingering flows.
4276 */
4277 fe_freed = fsw_process_linger(fsw, &fe_aborted);
4278
4279 /*
4280 * Pass 3: prune idle flow routes.
4281 */
4282 fr_freed = flow_route_prune(fsw->fsw_flow_mgr,
4283 ifp, &fr_resid);
4284
4285 /*
4286 * Pass 4: prune flow table
4287 *
4288 */
4289 cuckoo_hashtable_try_shrink(fsw->fsw_flow_mgr->fm_flow_table);
4290
4291 SK_DF(SK_VERB_FLOW, "%s: fe_nonviable %u/%u fe_freed %u/%u "
4292 "fe_aborted %u fr_freed %u/%u",
4293 fsw->fsw_flow_mgr->fm_name, fe_nonviable,
4294 (fe_nonviable + fsw->fsw_pending_nonviable),
4295 fe_freed, fsw->fsw_linger_cnt, fe_aborted, fe_freed,
4296 (fe_freed + fr_resid));
4297
4298 /* see if VM memory level is critical */
4299 low = skmem_lowmem_check();
4300
4301 /*
4302 * If things appear to be idle, we can prune away cached
4303 * object that have fallen out of the working sets (this
4304 * is different than purging). Every once in a while, we
4305 * also purge the caches. Note that this is done across
4306 * all flowswitch instances, and so we limit this to no
4307 * more than once every FSW_REAP_SK_THRES seconds.
4308 */
4309 last = os_atomic_load(&fsw_reap_last, relaxed);
4310 if ((low || (last != 0 && (now - last) >= FSW_REAP_SK_THRES)) &&
4311 os_atomic_cmpxchg(&fsw_reap_last, last, now, acq_rel)) {
4312 fsw_purge_cache(fsw, low);
4313
4314 /* increase sleep interval if idle */
4315 if (kdebug_enable == 0 && fsw->fsw_linger_cnt == 0 &&
4316 fsw->fsw_pending_nonviable == 0 && fr_resid == 0) {
4317 i <<= 3;
4318 }
4319 } else if (last == 0) {
4320 os_atomic_store(&fsw_reap_last, now, release);
4321 }
4322
4323 /*
4324 * Additionally, run thru the list of channels and prune
4325 * or purge away cached objects on "idle" channels. This
4326 * check is rate limited to no more than once every
4327 * FSW_DRAIN_CH_THRES seconds.
4328 */
4329 last = fsw->fsw_drain_channel_chk_last;
4330 if (low || (last != 0 && (now - last) >= FSW_DRAIN_CH_THRES)) {
4331 SK_DF(SK_VERB_FLOW, "%s: pruning channels",
4332 fsw->fsw_flow_mgr->fm_name);
4333
4334 fsw->fsw_drain_channel_chk_last = now;
4335 fsw_drain_channels(fsw, now, low);
4336 } else if (__improbable(last == 0)) {
4337 fsw->fsw_drain_channel_chk_last = now;
4338 }
4339
4340 /*
4341 * Finally, invoke the interface's reap callback to
4342 * tell it to prune or purge away cached objects if
4343 * it is idle. This check is rate limited to no more
4344 * than once every FSW_REAP_IF_THRES seconds.
4345 */
4346 last = fsw->fsw_drain_netif_chk_last;
4347 if (low || (last != 0 && (now - last) >= FSW_REAP_IF_THRES)) {
4348 ASSERT(fsw->fsw_nifna != NULL);
4349
4350 if (ifp->if_na_ops != NULL &&
4351 ifp->if_na_ops->ni_reap != NULL) {
4352 SK_DF(SK_VERB_FLOW, "%s: pruning netif",
4353 fsw->fsw_flow_mgr->fm_name);
4354 ifp->if_na_ops->ni_reap(ifp->if_na, ifp,
4355 FSW_REAP_IF_THRES, low);
4356 }
4357
4358 fsw->fsw_drain_netif_chk_last = now;
4359 } else if (__improbable(last == 0)) {
4360 fsw->fsw_drain_netif_chk_last = now;
4361 }
4362
4363 /* emit periodic interface stats ktrace */
4364 last = fsw->fsw_reap_last;
4365 if (last != 0 && (now - last) >= FSW_IFSTATS_THRES) {
4366 KDBG(SK_KTRACE_AON_IF_STATS, ifp->if_data.ifi_ipackets,
4367 ifp->if_data.ifi_ibytes * 8,
4368 ifp->if_data.ifi_opackets,
4369 ifp->if_data.ifi_obytes * 8);
4370
4371 fsw->fsw_reap_last = now;
4372 } else if (__improbable(last == 0)) {
4373 fsw->fsw_reap_last = now;
4374 }
4375
4376 nanoseconds_to_absolutetime(i * NSEC_PER_SEC, &t);
4377 clock_absolutetime_interval_to_deadline(t, &t);
4378 ASSERT(t != 0);
4379
4380 /* allow any pending detach to proceed */
4381 fsw_detach_barrier_remove(fsw);
4382 }
4383
4384 lck_mtx_lock(&fsw->fsw_reap_lock);
4385 if (!(fsw->fsw_reap_flags & FSW_REAPF_TERMINATING)) {
4386 fsw->fsw_reap_flags &= ~FSW_REAPF_RUNNING;
4387 (void) assert_wait_deadline(&fsw->fsw_reap_flags,
4388 THREAD_UNINT, t);
4389 lck_mtx_unlock(&fsw->fsw_reap_lock);
4390 thread_block_parameter(fsw_reap_thread_cont, fsw);
4391 /* NOTREACHED */
4392 __builtin_unreachable();
4393 } else {
4394 terminate:
4395 LCK_MTX_ASSERT(&fsw->fsw_reap_lock, LCK_MTX_ASSERT_OWNED);
4396 fsw->fsw_reap_flags &= ~(FSW_REAPF_RUNNING | FSW_REAPF_TERMINATING);
4397 fsw->fsw_reap_flags |= FSW_REAPF_TERMINATED;
4398 /*
4399 * And signal any thread waiting for us to terminate;
4400 * wait channel here other than fsw_reap_flags to make
4401 * it more explicit.
4402 */
4403 if (fsw->fsw_reap_flags & FSW_REAPF_TERMINATEBLOCK) {
4404 thread_wakeup((caddr_t)&fsw->fsw_reap_thread);
4405 }
4406 lck_mtx_unlock(&fsw->fsw_reap_lock);
4407
4408 SK_DF(SK_VERB_FLOW, "%s: terminating", fsw->fsw_reap_name);
4409
4410 /* for the extra refcnt from kernel_thread_start() */
4411 thread_deallocate(current_thread());
4412 /* this is the end */
4413 thread_terminate(current_thread());
4414 /* NOTREACHED */
4415 __builtin_unreachable();
4416 }
4417
4418 /* must never get here */
4419 VERIFY(0);
4420 /* NOTREACHED */
4421 __builtin_unreachable();
4422 }
4423
4424 static void
fsw_drain_channels(struct nx_flowswitch * fsw,uint64_t now,boolean_t low)4425 fsw_drain_channels(struct nx_flowswitch *fsw, uint64_t now, boolean_t low)
4426 {
4427 struct kern_nexus *nx = fsw->fsw_nx;
4428
4429 /* flowswitch protects NA via fsw_lock, see fsw_port_alloc/free */
4430 FSW_RLOCK(fsw);
4431
4432 /* uncrustify doesn't handle C blocks properly */
4433 /* BEGIN IGNORE CODESTYLE */
4434 nx_port_foreach(nx, ^(nexus_port_t p) {
4435 struct nexus_adapter *na = nx_port_get_na(nx, p);
4436 if (na == NULL || na->na_work_ts == 0 || na->na_rx_rings == NULL) {
4437 return;
4438 }
4439
4440 boolean_t purge;
4441
4442 /*
4443 * If some activity happened in the last FSW_DRAIN_CH_THRES
4444 * seconds on this channel, we reclaim memory if the channel
4445 * throughput is less than the reap threshold value.
4446 */
4447 if ((now - na->na_work_ts) < FSW_DRAIN_CH_THRES) {
4448 struct __kern_channel_ring *ring;
4449 channel_ring_stats *stats;
4450 uint64_t bps;
4451
4452 ring = na->na_rx_rings;
4453 stats = &ring->ckr_stats;
4454 bps = stats->crs_bytes_per_second;
4455
4456 if (bps < fsw_channel_reap_thresh) {
4457 purge = FALSE;
4458 na_drain(na, purge);
4459 }
4460 return;
4461 }
4462
4463 /*
4464 * If NA has been inactive for some time (twice the drain
4465 * threshold), we clear the work timestamp to temporarily skip
4466 * this channel until it's active again. Purging cached objects
4467 * can be expensive since we'd need to allocate and construct
4468 * them again, so we do it only when necessary.
4469 */
4470 if (low || ((now - na->na_work_ts) >= (FSW_DRAIN_CH_THRES << 1))) {
4471 na->na_work_ts = 0;
4472 purge = TRUE;
4473 } else {
4474 purge = FALSE;
4475 }
4476
4477 na_drain(na, purge); /* purge/prune caches */
4478 });
4479 /* END IGNORE CODESTYLE */
4480
4481 FSW_RUNLOCK(fsw);
4482 }
4483
4484 static void
fsw_purge_cache(struct nx_flowswitch * fsw,boolean_t low)4485 fsw_purge_cache(struct nx_flowswitch *fsw, boolean_t low)
4486 {
4487 #pragma unused(fsw)
4488 uint64_t o = os_atomic_inc_orig(&fsw_want_purge, relaxed);
4489 uint32_t p = fsw_flow_purge_thresh;
4490 boolean_t purge = (low || (o != 0 && p != 0 && (o % p) == 0));
4491
4492 SK_DF(SK_VERB_FLOW, "%s: %s caches",
4493 fsw->fsw_flow_mgr->fm_name,
4494 (purge ? "purge" : "prune"));
4495
4496 skmem_cache_reap_now(sk_fo_cache, purge);
4497 skmem_cache_reap_now(sk_fe_cache, purge);
4498 skmem_cache_reap_now(sk_fab_cache, purge);
4499 skmem_cache_reap_now(flow_route_cache, purge);
4500 skmem_cache_reap_now(flow_stats_cache, purge);
4501 netns_reap_caches(purge);
4502 skmem_reap_caches(purge);
4503
4504 #if CONFIG_MBUF_MCACHE
4505 if (if_is_fsw_transport_netagent_enabled() && purge) {
4506 mbuf_drain(FALSE);
4507 }
4508 #endif /* CONFIG_MBUF_MCACHE */
4509 }
4510
4511 static void
fsw_flow_handle_low_power(struct nx_flowswitch * fsw,struct flow_entry * fe)4512 fsw_flow_handle_low_power(struct nx_flowswitch *fsw, struct flow_entry *fe)
4513 {
4514 /* When the interface is in low power mode, the flow is nonviable */
4515 if (!(fe->fe_flags & FLOWENTF_NONVIABLE) &&
4516 os_atomic_cmpxchg(&fe->fe_want_nonviable, 0, 1, acq_rel)) {
4517 os_atomic_inc(&fsw->fsw_pending_nonviable, relaxed);
4518 }
4519 }
4520
4521 static uint32_t
fsw_process_deferred(struct nx_flowswitch * fsw)4522 fsw_process_deferred(struct nx_flowswitch *fsw)
4523 {
4524 struct flow_entry_dead sfed __sk_aligned(8);
4525 struct flow_mgr *fm = fsw->fsw_flow_mgr;
4526 struct flow_entry_dead *fed, *tfed;
4527 LIST_HEAD(, flow_entry_dead) fed_head =
4528 LIST_HEAD_INITIALIZER(fed_head);
4529 uint32_t i, nonviable = 0;
4530 boolean_t lowpowermode = FALSE;
4531
4532 bzero(&sfed, sizeof(sfed));
4533
4534 /*
4535 * The flows become nonviable when the interface
4536 * is in low power mode (edge trigger)
4537 */
4538 if ((fsw->fsw_ifp->if_xflags & IFXF_LOW_POWER) &&
4539 fsw->fsw_ifp->if_low_power_gencnt != fsw->fsw_low_power_gencnt) {
4540 lowpowermode = TRUE;
4541 fsw->fsw_low_power_gencnt = fsw->fsw_ifp->if_low_power_gencnt;
4542 }
4543
4544 /*
4545 * Scan thru the flow entry tree, and commit any pending withdraw or
4546 * nonviable requests. We may need to push stats and/or unassign the
4547 * nexus from NECP, but we cannot do that while holding the locks;
4548 * build a temporary list for those entries.
4549 */
4550 for (i = 0; i < fm->fm_owner_buckets_cnt; i++) {
4551 struct flow_owner_bucket *fob = flow_mgr_get_fob_at_idx(fm, i);
4552 struct flow_owner *fo;
4553
4554 /*
4555 * Grab the lock at all costs when handling low power mode
4556 */
4557 if (__probable(!lowpowermode)) {
4558 if (!FOB_TRY_LOCK(fob)) {
4559 continue;
4560 }
4561 } else {
4562 FOB_LOCK(fob);
4563 }
4564
4565 FOB_LOCK_ASSERT_HELD(fob);
4566 RB_FOREACH(fo, flow_owner_tree, &fob->fob_owner_head) {
4567 struct flow_entry *fe;
4568
4569 RB_FOREACH(fe, flow_entry_id_tree,
4570 &fo->fo_flow_entry_id_head) {
4571 /* try first as reader; skip if we can't */
4572 if (__improbable(lowpowermode)) {
4573 fsw_flow_handle_low_power(fsw, fe);
4574 }
4575 if (__improbable(fe->fe_flags & FLOWENTF_HALF_CLOSED)) {
4576 os_atomic_andnot(&fe->fe_flags, FLOWENTF_HALF_CLOSED, relaxed);
4577 flow_namespace_half_close(&fe->fe_port_reservation);
4578 }
4579
4580 /* if not withdrawn/nonviable, skip */
4581 if (!fe->fe_want_withdraw &&
4582 !fe->fe_want_nonviable) {
4583 continue;
4584 }
4585 /*
4586 * Here we're holding the lock as writer;
4587 * don't spend too much time as we're
4588 * blocking the data path now.
4589 */
4590 ASSERT(!uuid_is_null(fe->fe_uuid));
4591 /* only need flow UUID and booleans */
4592 uuid_copy(sfed.fed_uuid, fe->fe_uuid);
4593 sfed.fed_want_clonotify =
4594 (fe->fe_flags & FLOWENTF_CLOSE_NOTIFY);
4595 sfed.fed_want_nonviable = fe->fe_want_nonviable;
4596 flow_entry_teardown(fo, fe);
4597
4598 /* do this outside the flow bucket lock */
4599 fed = flow_entry_dead_alloc(Z_WAITOK);
4600 ASSERT(fed != NULL);
4601 *fed = sfed;
4602 LIST_INSERT_HEAD(&fed_head, fed, fed_link);
4603 }
4604 }
4605 FOB_UNLOCK(fob);
4606 }
4607
4608 /*
4609 * These nonviable flows are no longer useful since we've lost
4610 * the source IP address; in the event the client monitors the
4611 * viability of the flow, explicitly mark it as nonviable so
4612 * that a new flow can be created.
4613 */
4614 LIST_FOREACH_SAFE(fed, &fed_head, fed_link, tfed) {
4615 LIST_REMOVE(fed, fed_link);
4616 ASSERT(fsw->fsw_agent_session != NULL);
4617
4618 /* if flow is closed early */
4619 if (fed->fed_want_clonotify) {
4620 necp_client_early_close(fed->fed_uuid);
4621 }
4622
4623 /* if nonviable, unassign nexus attributes */
4624 if (fed->fed_want_nonviable) {
4625 (void) netagent_assign_nexus(fsw->fsw_agent_session,
4626 fed->fed_uuid, NULL, 0);
4627 }
4628
4629 flow_entry_dead_free(fed);
4630 ++nonviable;
4631 }
4632 ASSERT(LIST_EMPTY(&fed_head));
4633
4634 return nonviable;
4635 }
4636
4637 static uint32_t
fsw_process_linger(struct nx_flowswitch * fsw,uint32_t * abort)4638 fsw_process_linger(struct nx_flowswitch *fsw, uint32_t *abort)
4639 {
4640 struct flow_entry_linger_head linger_head =
4641 TAILQ_HEAD_INITIALIZER(linger_head);
4642 struct flow_entry *fe, *tfe;
4643 uint64_t now = _net_uptime;
4644 uint32_t i = 0, cnt = 0, freed = 0;
4645
4646 ASSERT(fsw->fsw_ifp != NULL);
4647 ASSERT(abort != NULL);
4648 *abort = 0;
4649
4650 /*
4651 * We don't want to contend with the datapath, so move
4652 * everything that's in the linger list into a local list.
4653 * This allows us to generate RSTs or free the flow entry
4654 * outside the lock. Any remaining flow entry in the local
4655 * list will get re-added back to the head of the linger
4656 * list, in front of any new ones added since then.
4657 */
4658 lck_mtx_lock(&fsw->fsw_linger_lock);
4659 TAILQ_CONCAT(&linger_head, &fsw->fsw_linger_head, fe_linger_link);
4660 ASSERT(TAILQ_EMPTY(&fsw->fsw_linger_head));
4661 cnt = fsw->fsw_linger_cnt;
4662 fsw->fsw_linger_cnt = 0;
4663 lck_mtx_unlock(&fsw->fsw_linger_lock);
4664
4665 TAILQ_FOREACH_SAFE(fe, &linger_head, fe_linger_link, tfe) {
4666 ASSERT(fe->fe_flags & FLOWENTF_TORN_DOWN);
4667 ASSERT(fe->fe_flags & FLOWENTF_DESTROYED);
4668 ASSERT(fe->fe_flags & FLOWENTF_LINGERING);
4669
4670 /*
4671 * See if this is a TCP flow that needs to generate
4672 * a RST to the remote peer (if not already).
4673 */
4674 if (flow_track_tcp_want_abort(fe)) {
4675 VERIFY(fe->fe_flags & FLOWENTF_ABORTED);
4676 ASSERT(!uuid_is_null(fe->fe_uuid));
4677 flow_track_abort_tcp(fe, NULL, NULL);
4678 (*abort)++;
4679 SK_LOG_VAR(char dbgbuf[FLOWENTRY_DBGBUF_SIZE]);
4680 SK_DF(SK_VERB_FLOW, "entry \"%s\" fe 0x%llx "
4681 "flags 0x%b [RST]", fe_as_string(fe, dbgbuf,
4682 sizeof(dbgbuf)), SK_KVA(fe), fe->fe_flags,
4683 FLOWENTF_BITS);
4684 }
4685
4686 /*
4687 * If flow has expired, remove from list and free;
4688 * otherwise leave it around in the linger list.
4689 */
4690 if (fe->fe_linger_expire <= now) {
4691 freed++;
4692 fsw_linger_remove_internal(&linger_head, fe);
4693 fe = NULL;
4694 }
4695 ++i;
4696 }
4697 VERIFY(i == cnt && cnt >= freed);
4698
4699 /*
4700 * Add any remaining ones back into the linger list.
4701 */
4702 lck_mtx_lock(&fsw->fsw_linger_lock);
4703 if (!TAILQ_EMPTY(&linger_head)) {
4704 ASSERT(TAILQ_EMPTY(&fsw->fsw_linger_head) || fsw->fsw_linger_cnt);
4705 TAILQ_CONCAT(&linger_head, &fsw->fsw_linger_head, fe_linger_link);
4706 ASSERT(TAILQ_EMPTY(&fsw->fsw_linger_head));
4707 TAILQ_CONCAT(&fsw->fsw_linger_head, &linger_head, fe_linger_link);
4708 fsw->fsw_linger_cnt += (cnt - freed);
4709 }
4710 ASSERT(TAILQ_EMPTY(&linger_head));
4711 lck_mtx_unlock(&fsw->fsw_linger_lock);
4712
4713 return freed;
4714 }
4715
4716 __attribute__((always_inline))
4717 static inline void
fsw_ifp_inc_traffic_class_in_pkt(struct ifnet * ifp,kern_packet_t ph)4718 fsw_ifp_inc_traffic_class_in_pkt(struct ifnet *ifp, kern_packet_t ph)
4719 {
4720 switch (__packet_get_traffic_class(ph)) {
4721 case PKT_TC_BE:
4722 ifp->if_tc.ifi_ibepackets++;
4723 ifp->if_tc.ifi_ibebytes += SK_PTR_ADDR_KPKT(ph)->pkt_length;
4724 break;
4725 case PKT_TC_BK:
4726 ifp->if_tc.ifi_ibkpackets++;
4727 ifp->if_tc.ifi_ibkbytes += SK_PTR_ADDR_KPKT(ph)->pkt_length;
4728 break;
4729 case PKT_TC_VI:
4730 ifp->if_tc.ifi_ivipackets++;
4731 ifp->if_tc.ifi_ivibytes += SK_PTR_ADDR_KPKT(ph)->pkt_length;
4732 break;
4733 case PKT_TC_VO:
4734 ifp->if_tc.ifi_ivopackets++;
4735 ifp->if_tc.ifi_ivobytes += SK_PTR_ADDR_KPKT(ph)->pkt_length;
4736 break;
4737 default:
4738 break;
4739 }
4740 }
4741
4742 __attribute__((always_inline))
4743 static inline void
fsw_ifp_inc_traffic_class_out_pkt(struct ifnet * ifp,uint32_t svc,uint32_t cnt,uint32_t len)4744 fsw_ifp_inc_traffic_class_out_pkt(struct ifnet *ifp, uint32_t svc,
4745 uint32_t cnt, uint32_t len)
4746 {
4747 switch (svc) {
4748 case PKT_TC_BE:
4749 ifp->if_tc.ifi_obepackets += cnt;
4750 ifp->if_tc.ifi_obebytes += len;
4751 break;
4752 case PKT_TC_BK:
4753 ifp->if_tc.ifi_obkpackets += cnt;
4754 ifp->if_tc.ifi_obkbytes += len;
4755 break;
4756 case PKT_TC_VI:
4757 ifp->if_tc.ifi_ovipackets += cnt;
4758 ifp->if_tc.ifi_ovibytes += len;
4759 break;
4760 case PKT_TC_VO:
4761 ifp->if_tc.ifi_ovopackets += cnt;
4762 ifp->if_tc.ifi_ovobytes += len;
4763 break;
4764 default:
4765 break;
4766 }
4767 }
4768