1 /* 2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 /* 30 * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. 31 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 52 * SUCH DAMAGE. 53 */ 54 55 #ifndef _SKYWALK_NEXUS_FLOWSWITCH_H_ 56 #define _SKYWALK_NEXUS_FLOWSWITCH_H_ 57 58 #include <skywalk/os_skywalk_private.h> 59 #include <net/ethernet.h> 60 #include <net/if_vlan_var.h> 61 #include <netinet/ip6.h> 62 63 #include <skywalk/nexus/flowswitch/flow/flow_var.h> 64 65 #if CONFIG_NEXUS_FLOWSWITCH 66 /* Shared declarations for the flow switch. */ 67 #define NX_FSW_NAME "fsw" /* prefix for flow switch port name */ 68 69 #define NX_FSW_MAXRINGS NX_MAX_NUM_RING_PAIR 70 #define NX_FSW_TXRINGSIZE 256 /* default TX ring size */ 71 #define NX_FSW_RXRINGSIZE 1024 /* default RX ring size */ 72 #define NX_FSW_AFRINGSIZE 256 /* default Alloc/Free ring size */ 73 74 #define NX_FSW_CHUNK 64 /* port chunk */ 75 #define NX_FSW_CHUNK_FREE 0xffffffffffffffff /* entire chunk is free */ 76 77 #define NX_FSW_VP_MIN NX_FSW_CHUNK 78 #define NX_FSW_VP_MAX 4096 /* up to 4k ports */ 79 #define NX_FSW_VP_NOPORT (NX_FSW_VP_MAX+1) 80 #define NX_FSW_VP_BROADCAST NX_FSW_VP_MAX 81 82 #define NX_FSW_MINSLOTS 2 /* XXX unclear how many */ 83 #define NX_FSW_MAXSLOTS NX_MAX_NUM_SLOT_PER_RING /* max # of slots */ 84 85 #define NX_FSW_TXBATCH 64 /* default TX batch size */ 86 #define NX_FSW_RXBATCH 32 /* default RX batch size */ 87 88 #define NX_FSW_BUFSIZE (2 * 1024) /* default buffer size */ 89 90 #define NX_FSW_MINBUFSIZE 512 /* min buffer size */ 91 #define NX_FSW_MAXBUFSIZE (32 * 1024) /* max buffer size */ 92 #define NX_FSW_MAXBUFFERS (4 * 1024) /* max number of buffers */ 93 /* max number of buffers for memory constrained device */ 94 #define NX_FSW_MAXBUFFERS_MEM_CONSTRAINED (2 * 1024) 95 /* default user buffer segment size for non memory-constrained device */ 96 #define NX_FSW_BUF_SEG_SIZE (32 * 1024) 97 98 /* {min, def, max} values for large buffer size */ 99 #define NX_FSW_MIN_LARGE_BUFSIZE 0 100 #define NX_FSW_DEF_LARGE_BUFSIZE (16 * 1024) 101 #define NX_FSW_MAX_LARGE_BUFSIZE (32 * 1024) 102 103 /* 104 * TODO: [email protected] -- minimum buflets for now; we will need to 105 * have a way to adjust this based on the underlying interface's 106 * parameters, e.g. jumbo MTU, large segment offload, etc. 107 */ 108 #define NX_FSW_UMD_SIZE _USER_PACKET_SIZE(BUFLETS_MIN) 109 #define NX_FSW_KMD_SIZE _KERN_PACKET_SIZE(BUFLETS_MIN) 110 111 #define NX_FSW_EVENT_RING_NUM 1 /* number of event rings */ 112 #define NX_FSW_EVENT_RING_SIZE 32 /* default event ring size */ 113 114 115 /* 116 * macro to check if TCP RX aggregation is enabled in flowswitch. 117 */ 118 #define NX_FSW_TCP_RX_AGG_ENABLED() (sk_fsw_rx_agg_tcp != 0) 119 120 struct nx_flowswitch; 121 122 /* 123 * Virtual port nexus adapter 124 */ 125 struct nexus_vp_adapter { 126 /* 127 * This is an overlay structure on nexus_adapter; 128 * make sure it contains 'up' as the first member. 129 */ 130 struct nexus_adapter vpna_up; 131 132 /* 133 * Flow switch support: 134 * 135 * If the adapter is associated with a nexus port, vpna_fsw points 136 * to the flow switch this NA is attached to; vpna_nx_port is the 137 * port number used in the flow switch. Otherwise, vpna_fsw would 138 * be NULL and vpna_nx_port would be NEXUS_PORT_ANY. 139 */ 140 struct nx_flowswitch *vpna_fsw; 141 nexus_port_t vpna_nx_port; 142 uint16_t vpna_gencnt; 143 boolean_t vpna_retry; 144 boolean_t vpna_pid_bound; 145 boolean_t vpna_defunct; 146 pid_t vpna_pid; 147 }; 148 149 #define VPNA(_na) __unsafe_forge_single(struct nexus_vp_adapter *, \ 150 ((struct nexus_vp_adapter *)(_na))) 151 152 #define NEXUS_PROVIDER_FLOW_SWITCH "com.apple.nexus.flowswitch" 153 154 /* fsw_state_flags */ 155 #define FSW_STATEF_QUIESCED 0x0001 156 #define FSW_STATEF_NETAGENT_ADDED 0x0002 157 #define FSW_STATEF_NETAGENT_ENABLED 0x0004 158 159 #define FSW_QUIESCED(_fsw) \ 160 (((_fsw)->fsw_state_flags & FSW_STATEF_QUIESCED) != 0) 161 162 #define FSW_NETAGENT_ADDED(_fsw) \ 163 (((_fsw)->fsw_state_flags & FSW_STATEF_NETAGENT_ADDED) != 0) 164 165 #define FSW_NETAGENT_ENABLED(_fsw) \ 166 (((_fsw)->fsw_state_flags & FSW_STATEF_NETAGENT_ENABLED) != 0) 167 168 #if (DEVELOPMENT || DEBUG) 169 #define FSW_RPS_MAX_NTHREADS 64 170 struct fsw_rps_thread { 171 lck_mtx_t frt_lock; 172 struct nx_flowswitch *frt_fsw; 173 struct thread *frt_thread; 174 struct pktq frt_pktq; 175 176 uint32_t frt_idx; 177 uint32_t frt_flags; 178 uint32_t frt_requests; 179 }; 180 181 #define FRT_RUNNING 0x00000001 182 #define FRT_TERMINATEBLOCK 0x10000000 183 #define FRT_TERMINATING 0x20000000 184 #define FRT_TERMINATED 0x40000000 185 #endif /* !DEVELOPMENT && !DEBUG */ 186 187 typedef enum { 188 FSW_TSO_MODE_NONE, 189 FSW_TSO_MODE_HW, 190 FSW_TSO_MODE_SW 191 } fsw_tso_mode_t; 192 193 /* 194 * nx_flowswitch is a descriptor for a flow switch instance. 195 * Interfaces for a flow switch are all in fsw_ports[]. 196 * The array has fixed size, an empty entry does not terminate 197 * the search, but lookups only occur on attach/detach so we 198 * don't mind if they are slow. 199 * 200 * The flow switch is non blocking on the transmit ports: excess 201 * packets are dropped if there is no room on the output port. 202 * 203 * fsw_lock protects accesses to the fsw_ports array. 204 * This is a rw lock (or equivalent). 205 */ 206 struct nx_flowswitch { 207 decl_lck_rw_data(, fsw_lock); 208 uint32_t fsw_tx_rings; 209 uint32_t fsw_rx_rings; 210 211 struct kern_nexus *fsw_nx; 212 213 /* packet type enqueued by the class queues */ 214 classq_pkt_type_t fsw_classq_enq_ptype; 215 boolean_t fsw_classq_enabled; 216 217 /* packet copy routines */ 218 pkt_copy_from_pkt_t *fsw_pkt_copy_from_pkt; 219 pkt_copy_from_mbuf_t *fsw_pkt_copy_from_mbuf; 220 pkt_copy_to_mbuf_t *fsw_pkt_copy_to_mbuf; 221 222 uint8_t fsw_frame_headroom; 223 uint32_t fsw_src_lla_gencnt; 224 uint32_t fsw_pending_nonviable; 225 uint32_t fsw_low_power_gencnt; 226 227 /* The following are protected by fsw_lock. */ 228 struct flow_mgr *fsw_flow_mgr; 229 netagent_session_t fsw_agent_session; 230 uuid_t fsw_agent_uuid; 231 struct ifnet *fsw_ifp; /* host interface */ 232 struct nexus_adapter *fsw_nifna; /* netif adapter */ 233 uint32_t fsw_state_flags; /* FSW_STATEF_* */ 234 235 236 union { 237 uint64_t _buf[1]; 238 uint8_t _eth_src[ETHER_ADDR_LEN]; 239 } __fsw_slladdr __attribute((aligned(sizeof(uint64_t)))); 240 241 #define fsw_slla __fsw_slladdr._buf 242 #define fsw_ether_shost __fsw_slladdr._eth_src 243 244 int (*fsw_resolve)(struct nx_flowswitch *, struct flow_route *, 245 struct __kern_packet *); 246 void (*fsw_frame)(struct nx_flowswitch *, struct flow_route *, 247 struct __kern_packet *); 248 sa_family_t (*fsw_demux)(struct nx_flowswitch *, 249 struct __kern_packet *); 250 251 struct fsw_stats fsw_stats; 252 253 /* 254 * The host interface attachment to the flowswitch (fsw_ifp), as well 255 * as the netagent registration, are guarded by the flowswitch's RW 256 * lock. During fsw_flow_bind() time, we need to make sure they are 257 * valid before proceeding forward, but holding that RW lock across 258 * the routine is not possible since the thread may block if there 259 * are other threads performing fsw_flow_{bind,unbind} on the same 260 * flow owner bucket. To prevent fsw_dtor() from happening while 261 * fsw_flow_bind() is in progress, we need to have it wait until all 262 * pending flow binds are done. To do this we add a busy counter 263 * incremented at flow bind time, and use the lock for synchronization. 264 */ 265 decl_lck_mtx_data(, fsw_detach_barrier_lock); 266 uint32_t fsw_detach_flags; /* see fsw_DETACHF_* */ 267 uint32_t fsw_detach_barriers; 268 uint32_t fsw_detach_waiters; 269 270 uint32_t fsw_ifp_dlt; 271 272 /* 273 * largest allocated packet size. 274 * used by: mbuf batch allocation logic during netif copy. 275 */ 276 uint32_t fsw_rx_largest_size; 277 278 void (*fsw_ctor)(struct nx_flowswitch *, struct flow_route *); 279 280 /* store stats from na that is going to be deactivated */ 281 struct __nx_stats_fsw *fsw_closed_na_stats; 282 283 /* ip fragments manager */ 284 struct fsw_ip_frag_mgr *fsw_ipfm; 285 286 struct skoid fsw_skoid; 287 288 /* input network emulator */ 289 struct netem *fsw_input_netem; 290 291 struct kern_channel *fsw_dev_ch; 292 struct kern_channel *fsw_host_ch; 293 294 /* 295 * The reaper thread gets scheduled on-demand, whenever there 296 * is any lingering flow entry needing to be freed or becoming 297 * nonviable. Access is protected by fsw_reap_lock. 298 */ 299 decl_lck_mtx_data(, fsw_reap_lock); 300 uint32_t fsw_reap_flags; /* see fsw_REAPF_* */ 301 uint32_t fsw_reap_requests; 302 struct thread *fsw_reap_thread; 303 char fsw_reap_name[MAXTHREADNAMESIZE]; 304 305 uint64_t fsw_reap_last; 306 uint64_t fsw_drain_channel_chk_last; 307 uint64_t fsw_drain_netif_chk_last; 308 uint64_t fsw_rx_stall_chk_last; 309 310 decl_lck_mtx_data(, fsw_linger_lock); 311 struct flow_entry_linger_head fsw_linger_head; 312 uint32_t fsw_linger_cnt; 313 314 fsw_tso_mode_t fsw_tso_mode; 315 uint32_t fsw_tso_sw_mtu; 316 317 #if (DEVELOPMENT || DEBUG) 318 uint32_t fsw_rps_nthreads; 319 struct fsw_rps_thread *__counted_by(fsw_rps_nthreads)fsw_rps_threads; 320 #endif /* !DEVELOPMENT && !DEBUG */ 321 }; 322 323 #define NX_FSW_PRIVATE(_nx) ((struct nx_flowswitch *)(_nx)->nx_arg) 324 325 #define FSW_RWINIT(_fsw) \ 326 lck_rw_init(&(_fsw)->fsw_lock, &nexus_lock_group, &nexus_lock_attr) 327 #define FSW_WLOCK(_fsw) \ 328 lck_rw_lock_exclusive(&(_fsw)->fsw_lock) 329 #define FSW_WUNLOCK(_fsw) \ 330 lck_rw_unlock_exclusive(&(_fsw)->fsw_lock) 331 #define FSW_WLOCKTORLOCK(_fsw) \ 332 lck_rw_lock_exclusive_to_shared(&(_fsw)->fsw_lock) 333 #define FSW_RLOCK(_fsw) \ 334 lck_rw_lock_shared(&(_fsw)->fsw_lock) 335 #define FSW_RLOCKTOWLOCK(_fsw) \ 336 lck_rw_lock_shared_to_exclusive(&(_fsw)->fsw_lock) 337 #define FSW_RTRYLOCK(_fsw) \ 338 lck_rw_try_lock(&(_fsw)->fsw_lock, LCK_RW_TYPE_SHARED) 339 #define FSW_RUNLOCK(_fsw) \ 340 lck_rw_unlock_shared(&(_fsw)->fsw_lock) 341 #define FSW_UNLOCK(_fsw) \ 342 lck_rw_done(&(_fsw)->fsw_lock) 343 #define FSW_RWDESTROY(_fsw) \ 344 lck_rw_destroy(&(_fsw)->fsw_lock, &nexus_lock_group) 345 #define FSW_WLOCK_ASSERT_HELD(_fsw) \ 346 LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_EXCLUSIVE) 347 #define FSW_RLOCK_ASSERT_HELD(_fsw) \ 348 LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_SHARED) 349 #define FSW_LOCK_ASSERT_HELD(_fsw) \ 350 LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_HELD) 351 352 extern struct nxdom nx_flowswitch_dom_s; 353 extern struct kern_nexus_domain_provider nx_fsw_prov_s; 354 355 SYSCTL_DECL(_kern_skywalk_flowswitch); 356 357 /* functions used by external modules to interface with flow switch */ 358 __BEGIN_DECLS 359 extern int nx_fsw_na_find(struct kern_nexus *, struct kern_channel *, 360 struct chreq *, struct nxbind *, struct proc *, struct nexus_adapter **, 361 boolean_t); 362 extern boolean_t nx_fsw_dom_port_is_reserved(struct kern_nexus *nx, 363 nexus_port_t nx_port); 364 extern int nx_fsw_netagent_add(struct kern_nexus *nx); 365 extern int nx_fsw_netagent_remove(struct kern_nexus *nx); 366 extern void nx_fsw_netagent_update(struct kern_nexus *nx); 367 extern void fsw_devna_rx(struct nexus_adapter *, struct __kern_packet *, 368 struct nexus_pkt_stats *); 369 extern struct nx_flowswitch *fsw_ifp_to_fsw(struct ifnet *); 370 extern void fsw_get_tso_capabilities(struct ifnet *, uint32_t *, uint32_t *); 371 __END_DECLS 372 #endif /* CONFIG_NEXUS_FLOWSWITCH */ 373 #endif /* _SKYWALK_NEXUS_FLOWSWITCH_H_ */ 374