1 /*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_
30 #define _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_
31
32 #include <skywalk/os_skywalk_private.h>
33 #include <skywalk/nexus/flowswitch/nx_flowswitch.h>
34 #include <skywalk/nexus/flowswitch/flow/flow_var.h>
35
36 #include <net/if_var.h>
37 #include <net/network_agent.h>
38 #include <net/necp.h>
39 #include <net/pktap.h>
40
41 #define FSW_VP_DEV 0 /* device port */
42 #define FSW_VP_HOST 1 /* host port (MS) */
43 #define FSW_VP_USER_MIN 2 /* start of user vp port num */
44 #define FSW_VP_USER_MAX NX_FSW_VP_MAX /* end of user vp port num */
45 /* max flush batch size for device port */
46 #if !XNU_TARGET_OS_OSX
47 #define FSW_VP_DEV_BATCH_MAX 16
48 #else /* XNU_TARGET_OS_OSX */
49 #define FSW_VP_DEV_BATCH_MAX 32
50 #endif /* XNU_TARGET_OS_OSX */
51
52 #define FSW_REAP_THREADNAME "skywalk_fsw_reap_%s%s"
53
54 struct fsw_ip_frag_mgr; /* forward declaration */
55 extern uint32_t fsw_ip_reass;
56 struct pktq;
57
58 #define FSW_DETACHF_DETACHING 0x10000000 /* detach in progress */
59 #define FSW_DETACHF_DETACHED 0x20000000 /* detached */
60
61 #define FSW_REAPF_RUNNING 0x00000001 /* thread is running */
62 #define FSW_REAPF_TERMINATEBLOCK 0x20000000 /* blocked waiting terminate */
63 #define FSW_REAPF_TERMINATING 0x40000000 /* thread is terminating */
64 #define FSW_REAPF_TERMINATED 0x80000000 /* thread is terminated */
65
66 extern kern_allocation_name_t skmem_tag_fsw_ports;
67 extern kern_allocation_name_t skmem_tag_fsw_ft;
68 extern kern_allocation_name_t skmem_tag_fsw_fb_hash;
69 extern kern_allocation_name_t skmem_tag_fsw_fob_hash;
70 extern kern_allocation_name_t skmem_tag_fsw_frb_hash;
71 extern kern_allocation_name_t skmem_tag_fsw_frib_hash;
72 extern kern_allocation_name_t skmem_tag_fsw_frag_mgr;
73
74 __BEGIN_DECLS
75
76 // generic
77 extern void fsw_init(void);
78 extern void fsw_uninit(void);
79 extern struct nx_flowswitch * fsw_alloc(zalloc_flags_t);
80 extern void fsw_free(struct nx_flowswitch *fsw);
81 extern int fsw_grow(struct nx_flowswitch *fsw, uint32_t grow);
82 extern int fsw_port_find(struct nx_flowswitch *fsw, nexus_port_t first,
83 nexus_port_t last, nexus_port_t *nx_port);
84 extern int fsw_port_bind(struct nx_flowswitch *fsw, nexus_port_t nx_port,
85 struct nxbind *nxb0);
86 extern int fsw_port_unbind(struct nx_flowswitch *fsw, nexus_port_t nx_port);
87 extern int fsw_port_na_defunct(struct nx_flowswitch *fsw,
88 struct nexus_vp_adapter *vpna);
89 extern size_t fsw_mib_get(struct nx_flowswitch *fsw,
90 struct nexus_mib_filter *filter, void *out, size_t len, struct proc *p);
91 extern int fsw_attach_vp(struct kern_nexus *nx, struct kern_channel *ch,
92 struct chreq *chr, struct nxbind *nxb, struct proc *p,
93 struct nexus_vp_adapter **vpna);
94 extern int fsw_ctl(struct kern_nexus *nx, nxcfg_cmd_t nc_cmd, struct proc *p,
95 void *data);
96 extern int fsw_ctl_detach(struct kern_nexus *nx, struct proc *p,
97 struct nx_spec_req *nsr);
98 extern boolean_t fsw_should_drop_packet(boolean_t is_input, sa_family_t af,
99 uint8_t proto, const char *ifname);
100 extern int fsw_port_alloc(struct nx_flowswitch *fsw, struct nxbind *nxb,
101 struct nexus_vp_adapter **vpna, nexus_port_t nx_port, struct proc *p,
102 boolean_t ifattach, boolean_t host);
103 extern void fsw_port_free(struct nx_flowswitch *fsw,
104 struct nexus_vp_adapter *vpna, nexus_port_t nx_port, boolean_t defunct);
105 extern int fsw_port_grow(struct nx_flowswitch *fsw, uint32_t num_ports);
106 extern int fsw_port_na_activate(struct nx_flowswitch *fsw,
107 struct nexus_vp_adapter *vpna, na_activate_mode_t mode);
108 extern boolean_t fsw_detach_barrier_add(struct nx_flowswitch *fsw);
109 extern void fsw_detach_barrier_remove(struct nx_flowswitch *fsw);
110
111 // vp related
112 extern int fsw_vp_na_activate(struct nexus_adapter *na,
113 na_activate_mode_t mode);
114 extern int fsw_vp_na_krings_create(struct nexus_adapter *na,
115 struct kern_channel *ch);
116 extern void fsw_vp_na_krings_delete(struct nexus_adapter *na,
117 struct kern_channel *ch, boolean_t defunct);
118 extern int fsw_vp_na_txsync(struct __kern_channel_ring *kring,
119 struct proc *p, uint32_t flags);
120 extern int fsw_vp_na_rxsync(struct __kern_channel_ring *kring,
121 struct proc *p, uint32_t flags);
122 extern int fsw_vp_na_attach(struct kern_nexus *nx, const char *cr_name,
123 struct nexus_adapter *na);
124 extern int fsw_vp_na_create(struct kern_nexus *nx, struct chreq *chr,
125 struct nexus_vp_adapter **ret);
126 extern void fsw_vp_channel_error_stats_fold(struct fsw_stats *fs,
127 struct __nx_stats_channel_errors *es);
128 extern errno_t fsw_vp_na_channel_event(struct nx_flowswitch *fsw,
129 uint32_t nx_port_id, struct __kern_channel_event *event,
130 uint16_t event_len);
131
132 // classq related
133 extern void fsw_classq_setup(struct nx_flowswitch *fsw,
134 struct nexus_adapter *hostna);
135 extern void fsw_classq_teardown(struct nx_flowswitch *fsw,
136 struct nexus_adapter *hostna);
137 extern struct mbuf * fsw_classq_kpkt_to_mbuf(struct nx_flowswitch *fsw,
138 struct __kern_packet *pkt);
139
140 // routing related
141 extern int fsw_generic_resolve(struct nx_flowswitch *fsw, struct flow_route *fr,
142 struct __kern_packet *pkt);
143
144 // data plane related
145 extern int fsw_dp_init(void);
146 extern void fsw_dp_uninit(void);
147 extern int fsw_dp_ctor(struct nx_flowswitch *fsw);
148 extern void fsw_dp_dtor(struct nx_flowswitch *fsw);
149 extern void fsw_ring_flush(struct nx_flowswitch *fsw,
150 struct __kern_channel_ring *skring, struct proc *p);
151 extern void fsw_ring_enqueue_tail_drop(struct nx_flowswitch *fsw,
152 struct __kern_channel_ring *ring, struct pktq *pktq);
153 extern boolean_t fsw_detach_barrier_add(struct nx_flowswitch *fsw);
154 extern void fsw_detach_barrier_remove(struct nx_flowswitch *fsw);
155 extern void fsw_linger_insert(struct flow_entry *fsw);
156 extern void fsw_linger_purge(struct nx_flowswitch *fsw);
157 extern void fsw_reap_sched(struct nx_flowswitch *fsw);
158
159 extern int fsw_dev_input_netem_dequeue(void *handle, pktsched_pkt_t *pkts,
160 uint32_t n_pkts);
161 extern void fsw_snoop(struct nx_flowswitch *fsw, struct flow_entry *fe,
162 bool input);
163
164 extern void fsw_receive(struct nx_flowswitch *fsw, struct pktq *pktq);
165 extern void dp_flow_tx_process(struct nx_flowswitch *fsw,
166 struct flow_entry *fe, uint32_t flags);
167 extern void dp_flow_rx_process(struct nx_flowswitch *fsw,
168 struct flow_entry *fe, uint32_t flags);
169
170 #if (DEVELOPMENT || DEBUG)
171 extern int fsw_rps_set_nthreads(struct nx_flowswitch* fsw, uint32_t n);
172 #endif /* !DEVELOPMENT && !DEBUG */
173
174 extern uint32_t fsw_tx_batch;
175 extern uint32_t fsw_rx_batch;
176 extern uint32_t fsw_chain_enqueue;
177 extern uint32_t fsw_use_dual_sized_pool;
178
179 // flow related
180 extern struct flow_owner * fsw_flow_add(struct nx_flowswitch *fsw,
181 struct nx_flow_req *req0, int *error);
182 extern int fsw_flow_del(struct nx_flowswitch *fsw, struct nx_flow_req *req,
183 bool nolinger, void *params);
184 extern int fsw_flow_config(struct nx_flowswitch *fsw, struct nx_flow_req *req);
185 extern void fsw_flow_abort_tcp(struct nx_flowswitch *fsw, struct flow_entry *fe,
186 struct __kern_packet *pkt);
187 extern void fsw_flow_abort_quic(struct flow_entry *fe, uint8_t *token);
188 extern struct __kern_channel_ring * fsw_flow_get_rx_ring(struct nx_flowswitch *fsw,
189 struct flow_entry *fe);
190 extern bool dp_flow_rx_route_process(struct nx_flowswitch *fsw,
191 struct flow_entry *fe);
192 // stats related
193 extern void fsw_fold_stats(struct nx_flowswitch *fsw, void *data,
194 nexus_stats_type_t type);
195
196 // netagent related
197 extern int fsw_netagent_add_remove(struct kern_nexus *nx, boolean_t add);
198 extern void fsw_netagent_update(struct kern_nexus *nx);
199 extern int fsw_netagent_register(struct nx_flowswitch *fsw, struct ifnet *ifp);
200 extern void fsw_netagent_unregister(struct nx_flowswitch *fsw,
201 struct ifnet *ifp);
202
203 // interface related
204 extern int fsw_ip_setup(struct nx_flowswitch *fsw, struct ifnet *ifp);
205 extern int fsw_cellular_setup(struct nx_flowswitch *fsw, struct ifnet *ifp);
206 extern int fsw_ethernet_setup(struct nx_flowswitch *fsw, struct ifnet *ifp);
207 extern void fsw_classq_setup(struct nx_flowswitch *fsw,
208 struct nexus_adapter *hostna);
209 extern void fsw_classq_teardown(struct nx_flowswitch *fsw,
210 struct nexus_adapter *hostna);
211 extern void fsw_qos_mark(struct nx_flowswitch *fsw, struct flow_entry *fe,
212 struct __kern_packet *pkt);
213 extern boolean_t fsw_qos_default_restricted(void);
214 extern struct mbuf * fsw_classq_kpkt_to_mbuf(struct nx_flowswitch *fsw,
215 struct __kern_packet *pkt);
216 extern sa_family_t fsw_ip_demux(struct nx_flowswitch *, struct __kern_packet *);
217
218 // fragment reassembly related
219 extern struct fsw_ip_frag_mgr * fsw_ip_frag_mgr_create(
220 struct nx_flowswitch *fsw, struct ifnet *ifp, size_t f_limit);
221 extern void fsw_ip_frag_mgr_destroy(struct fsw_ip_frag_mgr *mgr);
222 extern int fsw_ip_frag_reass_v4(struct fsw_ip_frag_mgr *mgr,
223 struct __kern_packet **pkt, struct ip *ip4, uint16_t *nfrags,
224 uint16_t *tlen);
225 extern int fsw_ip_frag_reass_v6(struct fsw_ip_frag_mgr *mgr,
226 struct __kern_packet **pkt, struct ip6_hdr *ip6, struct ip6_frag *ip6f,
227 uint16_t *nfrags, uint16_t *tlen);
228
229 __END_DECLS
230
231 __attribute__((always_inline))
232 static inline void
fsw_snoop_and_dequeue(struct flow_entry * fe,struct pktq * target,bool input)233 fsw_snoop_and_dequeue(struct flow_entry *fe, struct pktq *target, bool input)
234 {
235 if (pktap_total_tap_count != 0) {
236 fsw_snoop(fe->fe_fsw, fe, input);
237 }
238 KPKTQ_CONCAT(target, input ? &fe->fe_rx_pktq : &fe->fe_tx_pktq);
239 }
240
241 #define FSW_STATS_VAL(x) STATS_VAL(&fsw->fsw_stats, x)
242 #define FSW_STATS_INC(x) STATS_INC(&fsw->fsw_stats, x)
243 #define FSW_STATS_ADD(x, n) STATS_ADD(&fsw->fsw_stats, x, n)
244
245 #endif /* _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_ */
246