1 /*
2 * Copyright (c) 2008-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 * Application of kernel control for interface creation
33 *
34 * Theory of operation:
35 * utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 * This kernel control will register an interface for every client that connects.
37 * ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57 #include <os/log.h>
58
59 #if SKYWALK && CONFIG_NEXUS_KERNEL_PIPE
60 #include <skywalk/os_skywalk_private.h>
61 #include <skywalk/nexus/flowswitch/nx_flowswitch.h>
62 #include <skywalk/nexus/netif/nx_netif.h>
63 #define UTUN_NEXUS 1
64 #else // SKYWALK && CONFIG_NEXUS_KERNEL_PIPE
65 #define UTUN_NEXUS 0
66 #endif // SKYWALK && CONFIG_NEXUS_KERNEL_PIPE
67
68 #if UTUN_NEXUS
69 static nexus_controller_t utun_ncd;
70 static int utun_ncd_refcount;
71 static uuid_t utun_kpipe_uuid;
72 static uuid_t utun_nx_dom_prov;
73
74 typedef struct utun_nx {
75 uuid_t if_provider;
76 uuid_t if_instance;
77 uuid_t fsw_provider;
78 uuid_t fsw_instance;
79 uuid_t fsw_device;
80 uuid_t fsw_agent;
81 } *utun_nx_t;
82
83 #endif // UTUN_NEXUS
84
85 /* Control block allocated for each kernel control connection */
86 struct utun_pcb {
87 TAILQ_ENTRY(utun_pcb) utun_chain;
88 kern_ctl_ref utun_ctlref;
89 ifnet_t utun_ifp;
90 u_int32_t utun_unit;
91 u_int32_t utun_unique_id;
92 u_int32_t utun_flags;
93 int utun_ext_ifdata_stats;
94 u_int32_t utun_max_pending_packets;
95 char utun_if_xname[IFXNAMSIZ];
96 char utun_unique_name[IFXNAMSIZ];
97 // PCB lock protects state fields and rings
98 decl_lck_rw_data(, utun_pcb_lock);
99 struct mbuf * utun_input_chain;
100 struct mbuf * utun_input_chain_last;
101 u_int32_t utun_input_chain_count;
102 // Input chain lock protects the list of input mbufs
103 // The input chain lock must be taken AFTER the PCB lock if both are held
104 lck_mtx_t utun_input_chain_lock;
105
106 #if UTUN_NEXUS
107 // lock to protect utun_pcb_data_move & utun_pcb_drainers
108 decl_lck_mtx_data(, utun_pcb_data_move_lock);
109 u_int32_t utun_pcb_data_move; /* number of data moving contexts */
110 u_int32_t utun_pcb_drainers; /* number of threads waiting to drain */
111 u_int32_t utun_pcb_data_path_state; /* internal state of interface data path */
112
113 struct utun_nx utun_nx;
114 int utun_kpipe_enabled;
115 uuid_t utun_kpipe_uuid;
116 void * utun_kpipe_rxring;
117 void * utun_kpipe_txring;
118 kern_pbufpool_t utun_kpipe_pp;
119 u_int32_t utun_kpipe_tx_ring_size;
120 u_int32_t utun_kpipe_rx_ring_size;
121
122 kern_nexus_t utun_netif_nexus;
123 kern_pbufpool_t utun_netif_pp;
124 void * utun_netif_rxring;
125 void * utun_netif_txring;
126 uint64_t utun_netif_txring_size;
127
128 u_int32_t utun_slot_size;
129 u_int32_t utun_netif_ring_size;
130 u_int32_t utun_tx_fsw_ring_size;
131 u_int32_t utun_rx_fsw_ring_size;
132 // Auto attach flowswitch when netif is enabled. When set to false,
133 // it allows userspace nexus controller to attach and own flowswitch.
134 bool utun_attach_fsw;
135 bool utun_netif_connected;
136 bool utun_use_netif;
137 bool utun_needs_netagent;
138 #endif // UTUN_NEXUS
139 };
140
141 /* Kernel Control functions */
142 static errno_t utun_ctl_setup(u_int32_t *unit, void **unitinfo);
143 static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
144 void **unitinfo);
145 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
146 void **unitinfo);
147 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
148 void *unitinfo);
149 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
150 void *unitinfo, mbuf_t m, int flags);
151 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
152 int opt, void *data, size_t *len);
153 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
154 int opt, void *data, size_t len);
155 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
156 int flags);
157
158 /* Network Interface functions */
159 static void utun_start(ifnet_t interface);
160 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
161 const struct sockaddr *dest, const char *desk_linkaddr,
162 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
163 static errno_t utun_output(ifnet_t interface, mbuf_t data);
164 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
165 protocol_family_t *protocol);
166 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
167 const struct ifnet_demux_desc *demux_array,
168 u_int32_t demux_count);
169 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
170 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
171 static void utun_detached(ifnet_t interface);
172
173 /* Protocol handlers */
174 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
175 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
176 mbuf_t m, char *frame_header);
177 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
178 mbuf_t *packet, const struct sockaddr *dest, void *route,
179 char *frame_type, char *link_layer_dest);
180 static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
181
182 /* data movement refcounting functions */
183 #if UTUN_NEXUS
184 static boolean_t utun_data_move_begin(struct utun_pcb *pcb);
185 static void utun_data_move_end(struct utun_pcb *pcb);
186 static void utun_wait_data_move_drain(struct utun_pcb *pcb);
187
188 /* Data path states */
189 #define UTUN_PCB_DATA_PATH_READY 0x1
190
191 /* Macros to set/clear/test data path states */
192 #define UTUN_SET_DATA_PATH_READY(_pcb) \
193 ((_pcb)->utun_pcb_data_path_state |= UTUN_PCB_DATA_PATH_READY)
194 #define UTUN_CLR_DATA_PATH_READY(_pcb) \
195 ((_pcb)->utun_pcb_data_path_state &= ~UTUN_PCB_DATA_PATH_READY)
196 #define UTUN_IS_DATA_PATH_READY(_pcb) \
197 (((_pcb)->utun_pcb_data_path_state & UTUN_PCB_DATA_PATH_READY) != 0)
198
199 #define UTUN_IF_DEFAULT_SLOT_SIZE 2048
200 #define UTUN_IF_DEFAULT_RING_SIZE 64
201 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
202 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
203 #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
204 #define UTUN_IF_HEADROOM_SIZE 32
205
206 #define UTUN_IF_MIN_RING_SIZE 8
207 #define UTUN_IF_MAX_RING_SIZE 1024
208
209 #define UTUN_IF_MIN_SLOT_SIZE 1024
210 #define UTUN_IF_MAX_SLOT_SIZE 4096
211
212 #define UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT 512
213
214 static int if_utun_max_pending_input = UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT;
215
216 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
217 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
218 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
219
220 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
221 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
222 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
223
224 SYSCTL_DECL(_net_utun);
225 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
226
227 SYSCTL_INT(_net_utun, OID_AUTO, max_pending_input, CTLFLAG_LOCKED | CTLFLAG_RW, &if_utun_max_pending_input, 0, "");
228 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
229 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
230 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
231 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
232 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
233 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
234
235 static errno_t
236 utun_register_nexus(void);
237
238 static errno_t
239 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
240 static errno_t
241 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
242 proc_t p, kern_nexus_t nexus,
243 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
244 static errno_t
245 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
246 kern_channel_t channel);
247 static void
248 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
249 kern_channel_t channel);
250 static void
251 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
252 kern_channel_t channel);
253 static void
254 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
255 kern_channel_t channel);
256 static errno_t
257 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
258 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
259 void **ring_ctx);
260 static void
261 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
262 kern_channel_ring_t ring);
263 static errno_t
264 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
265 kern_channel_ring_t ring, uint32_t flags);
266 static errno_t
267 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
268 kern_channel_ring_t ring, uint32_t flags);
269 #endif // UTUN_NEXUS
270
271 #define UTUN_DEFAULT_MTU 1500
272 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
273
274 static kern_ctl_ref utun_kctlref;
275 static LCK_ATTR_DECLARE(utun_lck_attr, 0, 0);
276 static LCK_GRP_DECLARE(utun_lck_grp, "utun");
277 static LCK_MTX_DECLARE_ATTR(utun_lock, &utun_lck_grp, &utun_lck_attr);
278
279 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
280
281 static ZONE_DECLARE(utun_pcb_zone, "net.if_utun",
282 sizeof(struct utun_pcb), ZC_ZFREE_CLEARMEM);
283
284 #if UTUN_NEXUS
285
286 static int
287 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
288 {
289 #pragma unused(arg1, arg2)
290 int value = if_utun_ring_size;
291
292 int error = sysctl_handle_int(oidp, &value, 0, req);
293 if (error || !req->newptr) {
294 return error;
295 }
296
297 if (value < UTUN_IF_MIN_RING_SIZE ||
298 value > UTUN_IF_MAX_RING_SIZE) {
299 return EINVAL;
300 }
301
302 if_utun_ring_size = value;
303
304 return 0;
305 }
306
307 static int
308 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
309 {
310 #pragma unused(arg1, arg2)
311 int value = if_utun_tx_fsw_ring_size;
312
313 int error = sysctl_handle_int(oidp, &value, 0, req);
314 if (error || !req->newptr) {
315 return error;
316 }
317
318 if (value < UTUN_IF_MIN_RING_SIZE ||
319 value > UTUN_IF_MAX_RING_SIZE) {
320 return EINVAL;
321 }
322
323 if_utun_tx_fsw_ring_size = value;
324
325 return 0;
326 }
327
328 static int
329 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
330 {
331 #pragma unused(arg1, arg2)
332 int value = if_utun_rx_fsw_ring_size;
333
334 int error = sysctl_handle_int(oidp, &value, 0, req);
335 if (error || !req->newptr) {
336 return error;
337 }
338
339 if (value < UTUN_IF_MIN_RING_SIZE ||
340 value > UTUN_IF_MAX_RING_SIZE) {
341 return EINVAL;
342 }
343
344 if_utun_rx_fsw_ring_size = value;
345
346 return 0;
347 }
348
349 static errno_t
utun_netif_ring_init(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel,kern_channel_ring_t ring,boolean_t is_tx_ring,void ** ring_ctx)350 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
351 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
352 void **ring_ctx)
353 {
354 #pragma unused(nxprov)
355 #pragma unused(channel)
356 #pragma unused(ring_ctx)
357 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
358 if (!is_tx_ring) {
359 VERIFY(pcb->utun_netif_rxring == NULL);
360 pcb->utun_netif_rxring = ring;
361 } else {
362 VERIFY(pcb->utun_netif_txring == NULL);
363 pcb->utun_netif_txring = ring;
364 }
365 return 0;
366 }
367
368 static void
utun_netif_ring_fini(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t ring)369 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
370 kern_channel_ring_t ring)
371 {
372 #pragma unused(nxprov)
373 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
374 if (pcb->utun_netif_rxring == ring) {
375 pcb->utun_netif_rxring = NULL;
376 } else if (pcb->utun_netif_txring == ring) {
377 pcb->utun_netif_txring = NULL;
378 }
379 }
380
381 static errno_t
utun_netif_sync_tx(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t tx_ring,uint32_t flags)382 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
383 kern_channel_ring_t tx_ring, uint32_t flags)
384 {
385 #pragma unused(nxprov)
386 #pragma unused(flags)
387 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
388
389 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
390
391 if (!utun_data_move_begin(pcb)) {
392 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n",
393 __func__, if_name(pcb->utun_ifp));
394 return 0;
395 }
396
397 lck_rw_lock_shared(&pcb->utun_pcb_lock);
398
399 struct kern_channel_ring_stat_increment tx_ring_stats;
400 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
401 kern_channel_slot_t tx_pslot = NULL;
402 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
403 kern_packet_t tx_chain_ph = 0;
404
405 STATS_INC(nifs, NETIF_STATS_TX_SYNC);
406
407 if (tx_slot == NULL) {
408 // Nothing to write, don't bother signalling
409 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
410 utun_data_move_end(pcb);
411 return 0;
412 }
413
414 if (pcb->utun_kpipe_enabled) {
415 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
416 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
417
418 // Signal the kernel pipe ring to read
419 if (rx_ring != NULL) {
420 kern_channel_notify(rx_ring, 0);
421 }
422 utun_data_move_end(pcb);
423 return 0;
424 }
425
426 // If we're here, we're injecting into the utun kernel control socket
427 while (tx_slot != NULL) {
428 size_t length = 0;
429 mbuf_t data = NULL;
430
431 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
432
433 if (tx_ph == 0) {
434 // Advance TX ring
435 tx_pslot = tx_slot;
436 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
437 continue;
438 }
439 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
440 if (tx_chain_ph != 0) {
441 kern_packet_append(tx_ph, tx_chain_ph);
442 }
443 tx_chain_ph = tx_ph;
444
445 // Advance TX ring
446 tx_pslot = tx_slot;
447 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
448
449 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
450 VERIFY(tx_buf != NULL);
451
452 /* tx_baddr is the absolute buffer address */
453 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
454 VERIFY(tx_baddr != 0);
455
456 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
457
458 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
459 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
460
461 // The offset must be large enough for the headers
462 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
463
464 // Find family
465 uint32_t af = 0;
466 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
467 u_int ip_version = (vhl >> 4);
468 switch (ip_version) {
469 case 4: {
470 af = AF_INET;
471 break;
472 }
473 case 6: {
474 af = AF_INET6;
475 break;
476 }
477 default: {
478 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
479 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
480 UTUN_HEADER_SIZE(pcb));
481 break;
482 }
483 }
484
485 tx_offset -= UTUN_HEADER_SIZE(pcb);
486 tx_length += UTUN_HEADER_SIZE(pcb);
487 tx_baddr += tx_offset;
488
489 length = MIN(tx_length, pcb->utun_slot_size);
490
491 // Copy in family
492 memcpy(tx_baddr, &af, sizeof(af));
493 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
494 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
495 }
496
497 if (length > 0) {
498 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
499 if (error == 0) {
500 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
501 if (error == 0) {
502 error = utun_output(pcb->utun_ifp, data);
503 if (error != 0) {
504 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
505 }
506 } else {
507 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
508 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
509 STATS_INC(nifs, NETIF_STATS_DROP);
510 mbuf_freem(data);
511 data = NULL;
512 }
513 } else {
514 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
515 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
516 STATS_INC(nifs, NETIF_STATS_DROP);
517 }
518 } else {
519 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
520 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
521 STATS_INC(nifs, NETIF_STATS_DROP);
522 }
523
524 if (data == NULL) {
525 continue;
526 }
527
528 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
529 STATS_INC(nifs, NETIF_STATS_TX_COPY_MBUF);
530
531 tx_ring_stats.kcrsi_slots_transferred++;
532 tx_ring_stats.kcrsi_bytes_transferred += length;
533 }
534 if (tx_chain_ph != 0) {
535 kern_pbufpool_free_chain(tx_ring->ckr_pp, tx_chain_ph);
536 }
537 if (tx_pslot) {
538 kern_channel_advance_slot(tx_ring, tx_pslot);
539 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
540 (void)kern_channel_reclaim(tx_ring);
541 }
542
543 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
544 utun_data_move_end(pcb);
545 return 0;
546 }
547
548 static errno_t
utun_netif_tx_doorbell(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t ring,__unused uint32_t flags)549 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
550 kern_channel_ring_t ring, __unused uint32_t flags)
551 {
552 #pragma unused(nxprov)
553 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
554 boolean_t more = false;
555 errno_t rc = 0;
556
557 if (!utun_data_move_begin(pcb)) {
558 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n",
559 __func__, if_name(pcb->utun_ifp));
560 return 0;
561 }
562
563 /*
564 * Refill and sync the ring; we may be racing against another thread doing
565 * an RX sync that also wants to do kr_enter(), and so use the blocking
566 * variant here.
567 */
568 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
569 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
570 os_log_error(OS_LOG_DEFAULT, "%s, tx refill failed %d\n", __func__, rc);
571 }
572
573 (void) kr_enter(ring, TRUE);
574 lck_rw_lock_shared(&pcb->utun_pcb_lock);
575
576 if (pcb->utun_kpipe_enabled) {
577 uint32_t tx_available = kern_channel_available_slot_count(ring);
578 if (pcb->utun_netif_txring_size > 0 &&
579 tx_available >= pcb->utun_netif_txring_size - 1) {
580 // No room left in tx ring, disable output for now
581 errno_t error = ifnet_disable_output(pcb->utun_ifp);
582 if (error != 0) {
583 os_log_error(OS_LOG_DEFAULT, "utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
584 }
585 }
586 }
587
588 if (pcb->utun_kpipe_enabled) {
589 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
590
591 // Unlock while calling notify
592 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
593 // Signal the kernel pipe ring to read
594 if (rx_ring != NULL) {
595 kern_channel_notify(rx_ring, 0);
596 }
597 } else {
598 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
599 }
600
601 kr_exit(ring);
602 utun_data_move_end(pcb);
603 return 0;
604 }
605
606 static errno_t
utun_netif_sync_rx(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t rx_ring,uint32_t flags)607 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
608 kern_channel_ring_t rx_ring, uint32_t flags)
609 {
610 #pragma unused(nxprov)
611 #pragma unused(flags)
612 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
613 struct kern_channel_ring_stat_increment rx_ring_stats;
614
615 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
616
617 if (!utun_data_move_begin(pcb)) {
618 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n",
619 __func__, if_name(pcb->utun_ifp));
620 return 0;
621 }
622
623 lck_rw_lock_shared(&pcb->utun_pcb_lock);
624
625 // Reclaim user-released slots
626 (void) kern_channel_reclaim(rx_ring);
627
628 STATS_INC(nifs, NETIF_STATS_RX_SYNC);
629
630 uint32_t avail = kern_channel_available_slot_count(rx_ring);
631 if (avail == 0) {
632 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
633 utun_data_move_end(pcb);
634 return 0;
635 }
636
637 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
638 VERIFY(rx_pp != NULL);
639 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
640 kern_channel_slot_t rx_pslot = NULL;
641 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
642
643 while (rx_slot != NULL) {
644 // Check for a waiting packet
645 lck_mtx_lock(&pcb->utun_input_chain_lock);
646 mbuf_t data = pcb->utun_input_chain;
647 if (data == NULL) {
648 lck_mtx_unlock(&pcb->utun_input_chain_lock);
649 break;
650 }
651
652 // Allocate rx packet
653 kern_packet_t rx_ph = 0;
654 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
655 if (__improbable(error != 0)) {
656 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
657 STATS_INC(nifs, NETIF_STATS_DROP);
658 lck_mtx_unlock(&pcb->utun_input_chain_lock);
659 break;
660 }
661
662 // Advance waiting packets
663 if (pcb->utun_input_chain_count > 0) {
664 pcb->utun_input_chain_count--;
665 }
666 pcb->utun_input_chain = data->m_nextpkt;
667 data->m_nextpkt = NULL;
668 if (pcb->utun_input_chain == NULL) {
669 pcb->utun_input_chain_last = NULL;
670 }
671 lck_mtx_unlock(&pcb->utun_input_chain_lock);
672
673 size_t header_offset = UTUN_HEADER_SIZE(pcb);
674 size_t length = mbuf_pkthdr_len(data);
675
676 if (length < header_offset) {
677 // mbuf is too small
678 mbuf_freem(data);
679 kern_pbufpool_free(rx_pp, rx_ph);
680 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
681 STATS_INC(nifs, NETIF_STATS_DROP);
682 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
683 pcb->utun_ifp->if_xname, length, header_offset);
684 continue;
685 }
686
687 length -= header_offset;
688 if (length > rx_pp->pp_buflet_size) {
689 // Flush data
690 mbuf_freem(data);
691 kern_pbufpool_free(rx_pp, rx_ph);
692 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
693 STATS_INC(nifs, NETIF_STATS_DROP);
694 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
695 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
696 continue;
697 }
698
699 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
700
701 // Fillout rx packet
702 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
703 VERIFY(rx_buf != NULL);
704 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
705 VERIFY(rx_baddr != NULL);
706
707 // Copy-in data from mbuf to buflet
708 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
709 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
710
711 // Finalize and attach the packet
712 error = kern_buflet_set_data_offset(rx_buf, 0);
713 VERIFY(error == 0);
714 error = kern_buflet_set_data_length(rx_buf, length);
715 VERIFY(error == 0);
716 error = kern_packet_set_headroom(rx_ph, 0);
717 VERIFY(error == 0);
718 error = kern_packet_finalize(rx_ph);
719 VERIFY(error == 0);
720 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
721 VERIFY(error == 0);
722
723 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
724 STATS_INC(nifs, NETIF_STATS_RX_COPY_MBUF);
725 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
726
727 rx_ring_stats.kcrsi_slots_transferred++;
728 rx_ring_stats.kcrsi_bytes_transferred += length;
729
730 mbuf_freem(data);
731
732 // Advance ring
733 rx_pslot = rx_slot;
734 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
735 }
736
737 struct kern_channel_ring_stat_increment tx_ring_stats;
738 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
739 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
740 kern_channel_slot_t tx_pslot = NULL;
741 kern_channel_slot_t tx_slot = NULL;
742 if (tx_ring == NULL) {
743 // Net-If TX ring not set up yet, nothing to read
744 goto done;
745 }
746 // Unlock utun before entering ring
747 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
748
749 (void)kr_enter(tx_ring, TRUE);
750
751 // Lock again after entering and validate
752 lck_rw_lock_shared(&pcb->utun_pcb_lock);
753 if (tx_ring != pcb->utun_kpipe_txring) {
754 goto done;
755 }
756
757 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
758 if (tx_slot == NULL) {
759 // Nothing to read, don't bother signalling
760 goto done;
761 }
762
763 while (rx_slot != NULL && tx_slot != NULL) {
764 // Allocate rx packet
765 kern_packet_t rx_ph = 0;
766 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
767
768 // Advance TX ring
769 tx_pslot = tx_slot;
770 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
771
772 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
773 if (tx_ph == 0) {
774 continue;
775 }
776
777 /* XXX We could try this alloc before advancing the slot to avoid
778 * dropping the packet on failure to allocate.
779 */
780 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
781 if (__improbable(error != 0)) {
782 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
783 STATS_INC(nifs, NETIF_STATS_DROP);
784 break;
785 }
786
787 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
788 VERIFY(tx_buf != NULL);
789 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
790 VERIFY(tx_baddr != 0);
791 tx_baddr += kern_buflet_get_data_offset(tx_buf);
792
793 // Check packet length
794 size_t header_offset = UTUN_HEADER_SIZE(pcb);
795 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
796 if (tx_length < header_offset) {
797 // Packet is too small
798 kern_pbufpool_free(rx_pp, rx_ph);
799 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
800 STATS_INC(nifs, NETIF_STATS_DROP);
801 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
802 pcb->utun_ifp->if_xname, tx_length, header_offset);
803 continue;
804 }
805
806 size_t length = MIN(tx_length - header_offset,
807 pcb->utun_slot_size);
808
809 tx_ring_stats.kcrsi_slots_transferred++;
810 tx_ring_stats.kcrsi_bytes_transferred += length;
811
812 // Fillout rx packet
813 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
814 VERIFY(rx_buf != NULL);
815 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
816 VERIFY(rx_baddr != NULL);
817
818 // Copy-in data from tx to rx
819 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
820 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
821
822 // Finalize and attach the packet
823 error = kern_buflet_set_data_offset(rx_buf, 0);
824 VERIFY(error == 0);
825 error = kern_buflet_set_data_length(rx_buf, length);
826 VERIFY(error == 0);
827 error = kern_packet_set_headroom(rx_ph, 0);
828 VERIFY(error == 0);
829 error = kern_packet_finalize(rx_ph);
830 VERIFY(error == 0);
831 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
832 VERIFY(error == 0);
833
834 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
835 STATS_INC(nifs, NETIF_STATS_RX_COPY_DIRECT);
836 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
837
838 rx_ring_stats.kcrsi_slots_transferred++;
839 rx_ring_stats.kcrsi_bytes_transferred += length;
840
841 rx_pslot = rx_slot;
842 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
843 }
844
845 done:
846 if (rx_pslot) {
847 kern_channel_advance_slot(rx_ring, rx_pslot);
848 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
849 }
850
851 if (tx_pslot) {
852 kern_channel_advance_slot(tx_ring, tx_pslot);
853 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
854 (void)kern_channel_reclaim(tx_ring);
855 }
856
857 // Unlock first, then exit ring
858 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
859 if (tx_ring != NULL) {
860 if (tx_pslot != NULL) {
861 kern_channel_notify(tx_ring, 0);
862 }
863 kr_exit(tx_ring);
864 }
865
866 utun_data_move_end(pcb);
867 return 0;
868 }
869
870 static errno_t
utun_nexus_ifattach(struct utun_pcb * pcb,struct ifnet_init_eparams * init_params,struct ifnet ** ifp)871 utun_nexus_ifattach(struct utun_pcb *pcb,
872 struct ifnet_init_eparams *init_params,
873 struct ifnet **ifp)
874 {
875 errno_t err;
876 nexus_controller_t controller = kern_nexus_shared_controller();
877 struct kern_nexus_net_init net_init;
878 struct kern_pbufpool_init pp_init;
879
880 nexus_name_t provider_name;
881 snprintf((char *)provider_name, sizeof(provider_name),
882 "com.apple.netif.%s", pcb->utun_if_xname);
883
884 struct kern_nexus_provider_init prov_init = {
885 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
886 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
887 .nxpi_pre_connect = utun_nexus_pre_connect,
888 .nxpi_connected = utun_nexus_connected,
889 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
890 .nxpi_disconnected = utun_nexus_disconnected,
891 .nxpi_ring_init = utun_netif_ring_init,
892 .nxpi_ring_fini = utun_netif_ring_fini,
893 .nxpi_slot_init = NULL,
894 .nxpi_slot_fini = NULL,
895 .nxpi_sync_tx = utun_netif_sync_tx,
896 .nxpi_sync_rx = utun_netif_sync_rx,
897 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
898 };
899
900 nexus_attr_t nxa = NULL;
901 err = kern_nexus_attr_create(&nxa);
902 if (err != 0) {
903 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
904 __func__, err);
905 goto failed;
906 }
907
908 uint64_t slot_buffer_size = pcb->utun_slot_size;
909 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
910 VERIFY(err == 0);
911
912 // Reset ring size for netif nexus to limit memory usage
913 uint64_t ring_size = pcb->utun_netif_ring_size;
914 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
915 VERIFY(err == 0);
916 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
917 VERIFY(err == 0);
918
919 pcb->utun_netif_txring_size = ring_size;
920
921 bzero(&pp_init, sizeof(pp_init));
922 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
923 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
924 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
925 pp_init.kbi_bufsize = pcb->utun_slot_size;
926 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
927 pp_init.kbi_max_frags = 1;
928 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
929 "%s", provider_name);
930 pp_init.kbi_ctx = NULL;
931 pp_init.kbi_ctx_retain = NULL;
932 pp_init.kbi_ctx_release = NULL;
933
934 err = kern_pbufpool_create(&pp_init, &pcb->utun_netif_pp, NULL);
935 if (err != 0) {
936 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, err);
937 goto failed;
938 }
939
940 err = kern_nexus_controller_register_provider(controller,
941 utun_nx_dom_prov,
942 provider_name,
943 &prov_init,
944 sizeof(prov_init),
945 nxa,
946 &pcb->utun_nx.if_provider);
947 if (err != 0) {
948 os_log_error(OS_LOG_DEFAULT, "%s register provider failed, error %d\n",
949 __func__, err);
950 goto failed;
951 }
952
953 bzero(&net_init, sizeof(net_init));
954 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
955 net_init.nxneti_flags = 0;
956 net_init.nxneti_eparams = init_params;
957 net_init.nxneti_lladdr = NULL;
958 net_init.nxneti_prepare = utun_netif_prepare;
959 net_init.nxneti_rx_pbufpool = pcb->utun_netif_pp;
960 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
961 err = kern_nexus_controller_alloc_net_provider_instance(controller,
962 pcb->utun_nx.if_provider,
963 pcb,
964 NULL,
965 &pcb->utun_nx.if_instance,
966 &net_init,
967 ifp);
968 if (err != 0) {
969 os_log_error(OS_LOG_DEFAULT, "%s alloc_net_provider_instance failed, %d\n",
970 __func__, err);
971 kern_nexus_controller_deregister_provider(controller,
972 pcb->utun_nx.if_provider);
973 uuid_clear(pcb->utun_nx.if_provider);
974 goto failed;
975 }
976
977 failed:
978 if (nxa) {
979 kern_nexus_attr_destroy(nxa);
980 }
981 if (err && pcb->utun_netif_pp != NULL) {
982 kern_pbufpool_destroy(pcb->utun_netif_pp);
983 pcb->utun_netif_pp = NULL;
984 }
985 return err;
986 }
987
988 static void
utun_detach_provider_and_instance(uuid_t provider,uuid_t instance)989 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
990 {
991 nexus_controller_t controller = kern_nexus_shared_controller();
992 errno_t err;
993
994 if (!uuid_is_null(instance)) {
995 err = kern_nexus_controller_free_provider_instance(controller,
996 instance);
997 if (err != 0) {
998 os_log_error(OS_LOG_DEFAULT, "%s free_provider_instance failed %d\n",
999 __func__, err);
1000 }
1001 uuid_clear(instance);
1002 }
1003 if (!uuid_is_null(provider)) {
1004 err = kern_nexus_controller_deregister_provider(controller,
1005 provider);
1006 if (err != 0) {
1007 os_log_error(OS_LOG_DEFAULT, "%s deregister_provider %d\n", __func__, err);
1008 }
1009 uuid_clear(provider);
1010 }
1011 return;
1012 }
1013
1014 static void
utun_nexus_detach(struct utun_pcb * pcb)1015 utun_nexus_detach(struct utun_pcb *pcb)
1016 {
1017 utun_nx_t nx = &pcb->utun_nx;
1018 nexus_controller_t controller = kern_nexus_shared_controller();
1019 errno_t err;
1020
1021 if (!uuid_is_null(nx->fsw_device)) {
1022 err = kern_nexus_ifdetach(controller,
1023 nx->fsw_instance,
1024 nx->fsw_device);
1025 if (err != 0) {
1026 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms device failed %d\n",
1027 __func__, err);
1028 }
1029 }
1030
1031 utun_detach_provider_and_instance(nx->fsw_provider,
1032 nx->fsw_instance);
1033 utun_detach_provider_and_instance(nx->if_provider,
1034 nx->if_instance);
1035
1036 if (pcb->utun_netif_pp != NULL) {
1037 kern_pbufpool_destroy(pcb->utun_netif_pp);
1038 pcb->utun_netif_pp = NULL;
1039 }
1040 memset(nx, 0, sizeof(*nx));
1041 }
1042
1043 static errno_t
utun_create_fs_provider_and_instance(struct utun_pcb * pcb,const char * type_name,const char * ifname,uuid_t * provider,uuid_t * instance)1044 utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
1045 const char *type_name,
1046 const char *ifname,
1047 uuid_t *provider, uuid_t *instance)
1048 {
1049 nexus_attr_t attr = NULL;
1050 nexus_controller_t controller = kern_nexus_shared_controller();
1051 uuid_t dom_prov;
1052 errno_t err;
1053 struct kern_nexus_init init;
1054 nexus_name_t provider_name;
1055
1056 err = kern_nexus_get_default_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1057 &dom_prov);
1058 if (err != 0) {
1059 os_log_error(OS_LOG_DEFAULT, "%s can't get %s provider, error %d\n",
1060 __func__, type_name, err);
1061 goto failed;
1062 }
1063
1064 err = kern_nexus_attr_create(&attr);
1065 if (err != 0) {
1066 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1067 __func__, err);
1068 goto failed;
1069 }
1070
1071 uint64_t slot_buffer_size = pcb->utun_slot_size;
1072 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1073 VERIFY(err == 0);
1074
1075 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
1076 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
1077 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1078 VERIFY(err == 0);
1079 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
1080 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1081 VERIFY(err == 0);
1082 /*
1083 * Configure flowswitch to use super-packet (multi-buflet).
1084 * This allows flowswitch to perform intra-stack packet aggregation.
1085 */
1086 err = kern_nexus_attr_set(attr, NEXUS_ATTR_MAX_FRAGS,
1087 sk_fsw_rx_agg_tcp ? NX_PBUF_FRAGS_MAX : 1);
1088 VERIFY(err == 0);
1089
1090 snprintf((char *)provider_name, sizeof(provider_name),
1091 "com.apple.%s.%s", type_name, ifname);
1092 err = kern_nexus_controller_register_provider(controller,
1093 dom_prov,
1094 provider_name,
1095 NULL,
1096 0,
1097 attr,
1098 provider);
1099 kern_nexus_attr_destroy(attr);
1100 attr = NULL;
1101 if (err != 0) {
1102 os_log_error(OS_LOG_DEFAULT, "%s register %s provider failed, error %d\n",
1103 __func__, type_name, err);
1104 goto failed;
1105 }
1106 bzero(&init, sizeof(init));
1107 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1108 err = kern_nexus_controller_alloc_provider_instance(controller,
1109 *provider,
1110 NULL, NULL,
1111 instance, &init);
1112 if (err != 0) {
1113 os_log_error(OS_LOG_DEFAULT, "%s alloc_provider_instance %s failed, %d\n",
1114 __func__, type_name, err);
1115 kern_nexus_controller_deregister_provider(controller,
1116 *provider);
1117 uuid_clear(*provider);
1118 }
1119 failed:
1120 return err;
1121 }
1122
1123 static errno_t
utun_flowswitch_attach(struct utun_pcb * pcb)1124 utun_flowswitch_attach(struct utun_pcb *pcb)
1125 {
1126 nexus_controller_t controller = kern_nexus_shared_controller();
1127 errno_t err = 0;
1128 utun_nx_t nx = &pcb->utun_nx;
1129
1130 // Allocate flowswitch
1131 err = utun_create_fs_provider_and_instance(pcb,
1132 "flowswitch",
1133 pcb->utun_ifp->if_xname,
1134 &nx->fsw_provider,
1135 &nx->fsw_instance);
1136 if (err != 0) {
1137 os_log_error(OS_LOG_DEFAULT, "%s: failed to create bridge provider and instance\n",
1138 __func__);
1139 goto failed;
1140 }
1141
1142 // Attach flowswitch to device port
1143 err = kern_nexus_ifattach(controller, nx->fsw_instance,
1144 NULL, nx->if_instance,
1145 FALSE, &nx->fsw_device);
1146 if (err != 0) {
1147 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms device %d\n", __func__, err);
1148 goto failed;
1149 }
1150
1151 // Extract the agent UUID and save for later
1152 struct kern_nexus *flowswitch_nx = nx_find(nx->fsw_instance, false);
1153 if (flowswitch_nx != NULL) {
1154 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(flowswitch_nx);
1155 if (flowswitch != NULL) {
1156 FSW_RLOCK(flowswitch);
1157 uuid_copy(nx->fsw_agent, flowswitch->fsw_agent_uuid);
1158 FSW_UNLOCK(flowswitch);
1159 } else {
1160 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - flowswitch is NULL\n");
1161 }
1162 nx_release(flowswitch_nx);
1163 } else {
1164 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - unable to find flowswitch nexus\n");
1165 }
1166
1167 return 0;
1168
1169 failed:
1170 utun_nexus_detach(pcb);
1171
1172 errno_t detach_error = 0;
1173 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1174 panic("utun_flowswitch_attach - ifnet_detach failed: %d", detach_error);
1175 /* NOT REACHED */
1176 }
1177
1178 return err;
1179 }
1180
1181 static errno_t
utun_register_kernel_pipe_nexus(struct utun_pcb * pcb)1182 utun_register_kernel_pipe_nexus(struct utun_pcb *pcb)
1183 {
1184 nexus_attr_t nxa = NULL;
1185 errno_t result;
1186
1187 lck_mtx_lock(&utun_lock);
1188 if (utun_ncd_refcount++) {
1189 lck_mtx_unlock(&utun_lock);
1190 return 0;
1191 }
1192
1193 result = kern_nexus_controller_create(&utun_ncd);
1194 if (result) {
1195 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_create failed: %d\n",
1196 __FUNCTION__, result);
1197 goto done;
1198 }
1199
1200 uuid_t dom_prov;
1201 result = kern_nexus_get_default_domain_provider(
1202 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1203 if (result) {
1204 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_get_default_domain_provider failed: %d\n",
1205 __FUNCTION__, result);
1206 goto done;
1207 }
1208
1209 struct kern_nexus_provider_init prov_init = {
1210 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1211 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1212 .nxpi_pre_connect = utun_nexus_pre_connect,
1213 .nxpi_connected = utun_nexus_connected,
1214 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1215 .nxpi_disconnected = utun_nexus_disconnected,
1216 .nxpi_ring_init = utun_kpipe_ring_init,
1217 .nxpi_ring_fini = utun_kpipe_ring_fini,
1218 .nxpi_slot_init = NULL,
1219 .nxpi_slot_fini = NULL,
1220 .nxpi_sync_tx = utun_kpipe_sync_tx,
1221 .nxpi_sync_rx = utun_kpipe_sync_rx,
1222 .nxpi_tx_doorbell = NULL,
1223 };
1224
1225 result = kern_nexus_attr_create(&nxa);
1226 if (result) {
1227 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1228 __FUNCTION__, result);
1229 goto done;
1230 }
1231
1232 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1233 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1234 VERIFY(result == 0);
1235
1236 // Reset ring size for kernel pipe nexus to limit memory usage
1237 uint64_t ring_size =
1238 pcb->utun_kpipe_tx_ring_size != 0 ? pcb->utun_kpipe_tx_ring_size :
1239 if_utun_ring_size;
1240 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1241 VERIFY(result == 0);
1242
1243 ring_size =
1244 pcb->utun_kpipe_rx_ring_size != 0 ? pcb->utun_kpipe_rx_ring_size :
1245 if_utun_ring_size;
1246 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1247 VERIFY(result == 0);
1248
1249 result = kern_nexus_controller_register_provider(utun_ncd,
1250 dom_prov,
1251 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1252 &prov_init,
1253 sizeof(prov_init),
1254 nxa,
1255 &utun_kpipe_uuid);
1256 if (result) {
1257 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_register_provider failed: %d\n",
1258 __FUNCTION__, result);
1259 goto done;
1260 }
1261
1262 done:
1263 if (nxa) {
1264 kern_nexus_attr_destroy(nxa);
1265 }
1266
1267 if (result) {
1268 if (utun_ncd) {
1269 kern_nexus_controller_destroy(utun_ncd);
1270 utun_ncd = NULL;
1271 }
1272 utun_ncd_refcount = 0;
1273 }
1274
1275 lck_mtx_unlock(&utun_lock);
1276
1277 return result;
1278 }
1279
1280 static void
utun_unregister_kernel_pipe_nexus(void)1281 utun_unregister_kernel_pipe_nexus(void)
1282 {
1283 lck_mtx_lock(&utun_lock);
1284
1285 VERIFY(utun_ncd_refcount > 0);
1286
1287 if (--utun_ncd_refcount == 0) {
1288 kern_nexus_controller_destroy(utun_ncd);
1289 utun_ncd = NULL;
1290 }
1291
1292 lck_mtx_unlock(&utun_lock);
1293 }
1294
1295 // For use by socket option, not internally
1296 static errno_t
utun_disable_channel(struct utun_pcb * pcb)1297 utun_disable_channel(struct utun_pcb *pcb)
1298 {
1299 errno_t result;
1300 int enabled;
1301 uuid_t uuid;
1302
1303 /* Wait until all threads in the data paths are done. */
1304 utun_wait_data_move_drain(pcb);
1305
1306 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1307
1308 enabled = pcb->utun_kpipe_enabled;
1309 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1310
1311 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1312
1313 pcb->utun_kpipe_enabled = 0;
1314 uuid_clear(pcb->utun_kpipe_uuid);
1315
1316 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1317
1318 if (enabled) {
1319 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1320 } else {
1321 result = ENXIO;
1322 }
1323
1324 if (!result) {
1325 if (pcb->utun_kpipe_pp != NULL) {
1326 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1327 pcb->utun_kpipe_pp = NULL;
1328 }
1329 utun_unregister_kernel_pipe_nexus();
1330 }
1331
1332 return result;
1333 }
1334
1335 static errno_t
utun_enable_channel(struct utun_pcb * pcb,struct proc * proc)1336 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1337 {
1338 struct kern_nexus_init init;
1339 struct kern_pbufpool_init pp_init;
1340 errno_t result;
1341
1342 kauth_cred_t cred = kauth_cred_get();
1343 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
1344 if (result) {
1345 return result;
1346 }
1347
1348 result = utun_register_kernel_pipe_nexus(pcb);
1349 if (result) {
1350 return result;
1351 }
1352
1353 VERIFY(utun_ncd);
1354
1355 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1356
1357 if (pcb->utun_kpipe_enabled) {
1358 result = EEXIST; // return success instead?
1359 goto done;
1360 }
1361
1362 /*
1363 * Make sure we can fit packets in the channel buffers and
1364 * Allow an extra 4 bytes for the protocol number header in the channel
1365 */
1366 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
1367 result = EOPNOTSUPP;
1368 goto done;
1369 }
1370
1371 bzero(&pp_init, sizeof(pp_init));
1372 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1373 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
1374 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1375 pp_init.kbi_bufsize = pcb->utun_slot_size;
1376 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1377 pp_init.kbi_max_frags = 1;
1378 pp_init.kbi_flags |= KBIF_QUANTUM;
1379 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
1380 "com.apple.kpipe.%s", pcb->utun_if_xname);
1381 pp_init.kbi_ctx = NULL;
1382 pp_init.kbi_ctx_retain = NULL;
1383 pp_init.kbi_ctx_release = NULL;
1384
1385 result = kern_pbufpool_create(&pp_init, &pcb->utun_kpipe_pp,
1386 NULL);
1387 if (result != 0) {
1388 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, result);
1389 goto done;
1390 }
1391
1392 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1393 bzero(&init, sizeof(init));
1394 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1395 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
1396 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1397 utun_kpipe_uuid, pcb, NULL, &pcb->utun_kpipe_uuid, &init);
1398 if (result) {
1399 goto done;
1400 }
1401
1402 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1403 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1404 pcb->utun_kpipe_uuid, &port,
1405 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1406 if (result) {
1407 kern_nexus_controller_free_provider_instance(utun_ncd,
1408 pcb->utun_kpipe_uuid);
1409 uuid_clear(pcb->utun_kpipe_uuid);
1410 goto done;
1411 }
1412
1413 pcb->utun_kpipe_enabled = 1;
1414
1415 done:
1416 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1417
1418 if (result) {
1419 if (pcb->utun_kpipe_pp != NULL) {
1420 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1421 pcb->utun_kpipe_pp = NULL;
1422 }
1423 utun_unregister_kernel_pipe_nexus();
1424 }
1425
1426 return result;
1427 }
1428
1429 #endif // UTUN_NEXUS
1430
1431 errno_t
utun_register_control(void)1432 utun_register_control(void)
1433 {
1434 struct kern_ctl_reg kern_ctl;
1435 errno_t result = 0;
1436
1437 #if UTUN_NEXUS
1438 utun_register_nexus();
1439 #endif // UTUN_NEXUS
1440
1441 TAILQ_INIT(&utun_head);
1442
1443 bzero(&kern_ctl, sizeof(kern_ctl));
1444 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1445 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1446 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_SETUP | CTL_FLAG_REG_EXTENDED; /* Require root */
1447 kern_ctl.ctl_sendsize = 512 * 1024;
1448 kern_ctl.ctl_recvsize = 512 * 1024;
1449 kern_ctl.ctl_setup = utun_ctl_setup;
1450 kern_ctl.ctl_bind = utun_ctl_bind;
1451 kern_ctl.ctl_connect = utun_ctl_connect;
1452 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1453 kern_ctl.ctl_send = utun_ctl_send;
1454 kern_ctl.ctl_setopt = utun_ctl_setopt;
1455 kern_ctl.ctl_getopt = utun_ctl_getopt;
1456 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1457
1458 result = ctl_register(&kern_ctl, &utun_kctlref);
1459 if (result != 0) {
1460 os_log_error(OS_LOG_DEFAULT, "utun_register_control - ctl_register failed: %d\n", result);
1461 return result;
1462 }
1463
1464 /* Register the protocol plumbers */
1465 if ((result = proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN,
1466 utun_attach_proto, NULL)) != 0) {
1467 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN) failed: %d\n",
1468 result);
1469 ctl_deregister(utun_kctlref);
1470 return result;
1471 }
1472
1473 /* Register the protocol plumbers */
1474 if ((result = proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN,
1475 utun_attach_proto, NULL)) != 0) {
1476 proto_unregister_plumber(PF_INET, IFNET_FAMILY_UTUN);
1477 ctl_deregister(utun_kctlref);
1478 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN) failed: %d\n",
1479 result);
1480 return result;
1481 }
1482
1483 return 0;
1484 }
1485
1486 /* Kernel control functions */
1487
1488 static inline int
utun_find_by_unit(u_int32_t unit)1489 utun_find_by_unit(u_int32_t unit)
1490 {
1491 struct utun_pcb *next_pcb = NULL;
1492 int found = 0;
1493
1494 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1495 if (next_pcb->utun_unit == unit) {
1496 found = 1;
1497 break;
1498 }
1499 }
1500
1501 return found;
1502 }
1503
1504 static inline void
utun_free_pcb(struct utun_pcb * pcb,bool locked)1505 utun_free_pcb(struct utun_pcb *pcb, bool locked)
1506 {
1507 #if UTUN_NEXUS
1508 mbuf_freem_list(pcb->utun_input_chain);
1509 pcb->utun_input_chain_count = 0;
1510 lck_mtx_destroy(&pcb->utun_input_chain_lock, &utun_lck_grp);
1511 lck_mtx_destroy(&pcb->utun_pcb_data_move_lock, &utun_lck_grp);
1512 #endif // UTUN_NEXUS
1513 lck_rw_destroy(&pcb->utun_pcb_lock, &utun_lck_grp);
1514 if (!locked) {
1515 lck_mtx_lock(&utun_lock);
1516 }
1517 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1518 if (!locked) {
1519 lck_mtx_unlock(&utun_lock);
1520 }
1521 zfree(utun_pcb_zone, pcb);
1522 }
1523
1524 static errno_t
utun_ctl_setup(u_int32_t * unit,void ** unitinfo)1525 utun_ctl_setup(u_int32_t *unit, void **unitinfo)
1526 {
1527 if (unit == NULL || unitinfo == NULL) {
1528 return EINVAL;
1529 }
1530
1531 lck_mtx_lock(&utun_lock);
1532
1533 /* Find next available unit */
1534 if (*unit == 0) {
1535 *unit = 1;
1536 while (*unit != ctl_maxunit) {
1537 if (utun_find_by_unit(*unit)) {
1538 (*unit)++;
1539 } else {
1540 break;
1541 }
1542 }
1543 if (*unit == ctl_maxunit) {
1544 lck_mtx_unlock(&utun_lock);
1545 return EBUSY;
1546 }
1547 } else if (utun_find_by_unit(*unit)) {
1548 lck_mtx_unlock(&utun_lock);
1549 return EBUSY;
1550 }
1551
1552 /* Find some open interface id */
1553 u_int32_t chosen_unique_id = 1;
1554 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1555 if (next_pcb != NULL) {
1556 /* List was not empty, add one to the last item */
1557 chosen_unique_id = next_pcb->utun_unique_id + 1;
1558 next_pcb = NULL;
1559
1560 /*
1561 * If this wrapped the id number, start looking at
1562 * the front of the list for an unused id.
1563 */
1564 if (chosen_unique_id == 0) {
1565 /* Find the next unused ID */
1566 chosen_unique_id = 1;
1567 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1568 if (next_pcb->utun_unique_id > chosen_unique_id) {
1569 /* We found a gap */
1570 break;
1571 }
1572
1573 chosen_unique_id = next_pcb->utun_unique_id + 1;
1574 }
1575 }
1576 }
1577
1578 struct utun_pcb *pcb = zalloc_flags(utun_pcb_zone, Z_WAITOK | Z_ZERO);
1579
1580 *unitinfo = pcb;
1581 pcb->utun_unit = *unit;
1582 pcb->utun_unique_id = chosen_unique_id;
1583
1584 if (next_pcb != NULL) {
1585 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1586 } else {
1587 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1588 }
1589
1590 lck_mtx_unlock(&utun_lock);
1591
1592 return 0;
1593 }
1594
1595 static errno_t
utun_ctl_bind(kern_ctl_ref kctlref,struct sockaddr_ctl * sac,void ** unitinfo)1596 utun_ctl_bind(kern_ctl_ref kctlref,
1597 struct sockaddr_ctl *sac,
1598 void **unitinfo)
1599 {
1600 if (*unitinfo == NULL) {
1601 u_int32_t unit = 0;
1602 (void)utun_ctl_setup(&unit, unitinfo);
1603 }
1604
1605 struct utun_pcb *pcb = (struct utun_pcb *)*unitinfo;
1606 if (pcb == NULL) {
1607 return EINVAL;
1608 }
1609
1610 pcb->utun_ctlref = kctlref;
1611 pcb->utun_unit = sac->sc_unit;
1612 pcb->utun_max_pending_packets = 1;
1613
1614 #if UTUN_NEXUS
1615 pcb->utun_use_netif = false;
1616 pcb->utun_attach_fsw = true;
1617 pcb->utun_netif_connected = false;
1618 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1619 pcb->utun_netif_ring_size = if_utun_ring_size;
1620 pcb->utun_tx_fsw_ring_size = if_utun_tx_fsw_ring_size;
1621 pcb->utun_rx_fsw_ring_size = if_utun_rx_fsw_ring_size;
1622 pcb->utun_input_chain_count = 0;
1623 lck_mtx_init(&pcb->utun_input_chain_lock, &utun_lck_grp, &utun_lck_attr);
1624 lck_mtx_init(&pcb->utun_pcb_data_move_lock,
1625 &utun_lck_grp, &utun_lck_attr);
1626 #endif // UTUN_NEXUS
1627
1628 lck_rw_init(&pcb->utun_pcb_lock, &utun_lck_grp, &utun_lck_attr);
1629
1630 return 0;
1631 }
1632
1633 static errno_t
utun_ctl_connect(kern_ctl_ref kctlref,struct sockaddr_ctl * sac,void ** unitinfo)1634 utun_ctl_connect(kern_ctl_ref kctlref,
1635 struct sockaddr_ctl *sac,
1636 void **unitinfo)
1637 {
1638 struct ifnet_init_eparams utun_init = {};
1639 errno_t result = 0;
1640
1641 if (*unitinfo == NULL) {
1642 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1643 }
1644
1645 struct utun_pcb *pcb = *unitinfo;
1646 if (pcb == NULL) {
1647 return EINVAL;
1648 }
1649
1650 /* Handle case where utun_ctl_setup() was called, but ipsec_ctl_bind() was not */
1651 if (pcb->utun_ctlref == NULL) {
1652 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1653 }
1654
1655 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1656 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1657
1658 /* Create the interface */
1659 bzero(&utun_init, sizeof(utun_init));
1660 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1661 utun_init.len = sizeof(utun_init);
1662
1663 #if UTUN_NEXUS
1664 if (pcb->utun_use_netif) {
1665 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1666 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1667 } else
1668 #endif // UTUN_NEXUS
1669 {
1670 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1671 utun_init.start = utun_start;
1672 utun_init.framer_extended = utun_framer;
1673 }
1674 utun_init.name = "utun";
1675 utun_init.unit = pcb->utun_unit - 1;
1676 utun_init.uniqueid = pcb->utun_unique_name;
1677 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1678 utun_init.family = IFNET_FAMILY_UTUN;
1679 utun_init.type = IFT_OTHER;
1680 utun_init.demux = utun_demux;
1681 utun_init.add_proto = utun_add_proto;
1682 utun_init.del_proto = utun_del_proto;
1683 utun_init.softc = pcb;
1684 utun_init.ioctl = utun_ioctl;
1685 utun_init.free = utun_detached;
1686
1687 #if UTUN_NEXUS
1688 if (pcb->utun_use_netif) {
1689 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1690 if (result != 0) {
1691 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1692 utun_free_pcb(pcb, false);
1693 *unitinfo = NULL;
1694 return result;
1695 }
1696
1697 if (pcb->utun_attach_fsw) {
1698 result = utun_flowswitch_attach(pcb);
1699 if (result != 0) {
1700 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_flowswitch_attach failed: %d\n", result);
1701 // Do not call utun_free_pcb(). We will be attached already, and will be freed later
1702 // in utun_detached().
1703 *unitinfo = NULL;
1704 return result;
1705 }
1706 }
1707
1708 /* Attach to bpf */
1709 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1710 } else
1711 #endif // UTUN_NEXUS
1712 {
1713 /*
1714 * Upon success, this holds an ifnet reference which we will
1715 * release via ifnet_release() at final detach time.
1716 */
1717 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1718 if (result != 0) {
1719 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1720 utun_free_pcb(pcb, false);
1721 *unitinfo = NULL;
1722 return result;
1723 }
1724
1725 /* Set flags and additional information. */
1726 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1727 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1728
1729 /* The interface must generate its own IPv6 LinkLocal address,
1730 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1731 */
1732 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1733
1734 /* Reset the stats in case as the interface may have been recycled */
1735 struct ifnet_stats_param stats;
1736 bzero(&stats, sizeof(struct ifnet_stats_param));
1737 ifnet_set_stat(pcb->utun_ifp, &stats);
1738
1739 /* Attach the interface */
1740 result = ifnet_attach(pcb->utun_ifp, NULL);
1741 if (result != 0) {
1742 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_attach failed: %d\n", result);
1743 /* Release reference now since attach failed */
1744 ifnet_release(pcb->utun_ifp);
1745 utun_free_pcb(pcb, false);
1746 *unitinfo = NULL;
1747 return result;
1748 }
1749
1750 /* Attach to bpf */
1751 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1752
1753 #if UTUN_NEXUS
1754 lck_mtx_lock(&pcb->utun_pcb_data_move_lock);
1755 UTUN_SET_DATA_PATH_READY(pcb);
1756 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
1757 #endif // UTUN_NEXUS
1758 }
1759
1760 /* The interfaces resoures allocated, mark it as running */
1761 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1762
1763 return result;
1764 }
1765
1766 static errno_t
utun_detach_ip(ifnet_t interface,protocol_family_t protocol,socket_t pf_socket)1767 utun_detach_ip(ifnet_t interface,
1768 protocol_family_t protocol,
1769 socket_t pf_socket)
1770 {
1771 errno_t result = EPROTONOSUPPORT;
1772
1773 /* Attempt a detach */
1774 if (protocol == PF_INET) {
1775 struct ifreq ifr;
1776
1777 bzero(&ifr, sizeof(ifr));
1778 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1779 ifnet_name(interface), ifnet_unit(interface));
1780
1781 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1782 } else if (protocol == PF_INET6) {
1783 struct in6_ifreq ifr6;
1784
1785 bzero(&ifr6, sizeof(ifr6));
1786 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1787 ifnet_name(interface), ifnet_unit(interface));
1788
1789 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1790 }
1791
1792 return result;
1793 }
1794
1795 static void
utun_remove_address(ifnet_t interface,protocol_family_t protocol,ifaddr_t address,socket_t pf_socket)1796 utun_remove_address(ifnet_t interface,
1797 protocol_family_t protocol,
1798 ifaddr_t address,
1799 socket_t pf_socket)
1800 {
1801 errno_t result = 0;
1802
1803 /* Attempt a detach */
1804 if (protocol == PF_INET) {
1805 struct ifreq ifr;
1806
1807 bzero(&ifr, sizeof(ifr));
1808 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1809 ifnet_name(interface), ifnet_unit(interface));
1810 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1811 if (result != 0) {
1812 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed: %d", result);
1813 } else {
1814 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1815 if (result != 0) {
1816 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR failed: %d", result);
1817 }
1818 }
1819 } else if (protocol == PF_INET6) {
1820 struct in6_ifreq ifr6;
1821
1822 bzero(&ifr6, sizeof(ifr6));
1823 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1824 ifnet_name(interface), ifnet_unit(interface));
1825 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1826 sizeof(ifr6.ifr_addr));
1827 if (result != 0) {
1828 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed (v6): %d",
1829 result);
1830 } else {
1831 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1832 if (result != 0) {
1833 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1834 result);
1835 }
1836 }
1837 }
1838 }
1839
1840 static void
utun_cleanup_family(ifnet_t interface,protocol_family_t protocol)1841 utun_cleanup_family(ifnet_t interface,
1842 protocol_family_t protocol)
1843 {
1844 errno_t result = 0;
1845 socket_t pf_socket = NULL;
1846 ifaddr_t *addresses = NULL;
1847 int i;
1848
1849 if (protocol != PF_INET && protocol != PF_INET6) {
1850 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - invalid protocol family %d\n", protocol);
1851 return;
1852 }
1853
1854 /* Create a socket for removing addresses and detaching the protocol */
1855 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1856 if (result != 0) {
1857 if (result != EAFNOSUPPORT) {
1858 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - failed to create %s socket: %d\n",
1859 protocol == PF_INET ? "IP" : "IPv6", result);
1860 }
1861 goto cleanup;
1862 }
1863
1864 /* always set SS_PRIV, we want to close and detach regardless */
1865 sock_setpriv(pf_socket, 1);
1866
1867 result = utun_detach_ip(interface, protocol, pf_socket);
1868 if (result == 0 || result == ENXIO) {
1869 /* We are done! We either detached or weren't attached. */
1870 goto cleanup;
1871 } else if (result != EBUSY) {
1872 /* Uh, not really sure what happened here... */
1873 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1874 goto cleanup;
1875 }
1876
1877 /*
1878 * At this point, we received an EBUSY error. This means there are
1879 * addresses attached. We should detach them and then try again.
1880 */
1881 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1882 if (result != 0) {
1883 os_log_error(OS_LOG_DEFAULT, "fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1884 ifnet_name(interface), ifnet_unit(interface),
1885 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1886 goto cleanup;
1887 }
1888
1889 for (i = 0; addresses[i] != 0; i++) {
1890 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1891 }
1892 ifnet_free_address_list(addresses);
1893 addresses = NULL;
1894
1895 /*
1896 * The addresses should be gone, we should try the remove again.
1897 */
1898 result = utun_detach_ip(interface, protocol, pf_socket);
1899 if (result != 0 && result != ENXIO) {
1900 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1901 }
1902
1903 cleanup:
1904 if (pf_socket != NULL) {
1905 sock_close(pf_socket);
1906 }
1907
1908 if (addresses != NULL) {
1909 ifnet_free_address_list(addresses);
1910 }
1911 }
1912
1913 static errno_t
utun_ctl_disconnect(__unused kern_ctl_ref kctlref,__unused u_int32_t unit,void * unitinfo)1914 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1915 __unused u_int32_t unit,
1916 void *unitinfo)
1917 {
1918 struct utun_pcb *pcb = unitinfo;
1919 ifnet_t ifp = NULL;
1920 errno_t result = 0;
1921
1922 if (pcb == NULL) {
1923 return EINVAL;
1924 }
1925
1926 #if UTUN_NEXUS
1927 /* Wait until all threads in the data paths are done. */
1928 utun_wait_data_move_drain(pcb);
1929 // Tell the nexus to stop all rings
1930 if (pcb->utun_netif_nexus != NULL && pcb->utun_netif_connected) {
1931 kern_nexus_stop(pcb->utun_netif_nexus);
1932 }
1933 #endif // UTUN_NEXUS
1934
1935 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1936
1937 #if UTUN_NEXUS
1938 uuid_t kpipe_uuid;
1939 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1940 uuid_clear(pcb->utun_kpipe_uuid);
1941 pcb->utun_kpipe_enabled = FALSE;
1942 #endif // UTUN_NEXUS
1943
1944 pcb->utun_ctlref = NULL;
1945
1946 ifp = pcb->utun_ifp;
1947 if (ifp != NULL) {
1948 #if UTUN_NEXUS
1949 // Tell the nexus to stop all rings
1950 if (pcb->utun_netif_nexus != NULL) {
1951 /*
1952 * Quiesce the interface and flush any pending outbound packets.
1953 */
1954 if_down(ifp);
1955
1956 /* Increment refcnt, but detach interface */
1957 ifnet_incr_iorefcnt(ifp);
1958 if ((result = ifnet_detach(ifp)) != 0) {
1959 panic("utun_ctl_disconnect - ifnet_detach failed: %d", result);
1960 }
1961
1962 /*
1963 * We want to do everything in our power to ensure that the interface
1964 * really goes away when the socket is closed. We must remove IP/IPv6
1965 * addresses and detach the protocols. Finally, we can remove and
1966 * release the interface.
1967 */
1968 utun_cleanup_family(ifp, AF_INET);
1969 utun_cleanup_family(ifp, AF_INET6);
1970
1971 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1972
1973 if (!uuid_is_null(kpipe_uuid)) {
1974 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1975 if (pcb->utun_kpipe_pp != NULL) {
1976 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1977 pcb->utun_kpipe_pp = NULL;
1978 }
1979 utun_unregister_kernel_pipe_nexus();
1980 }
1981 }
1982 utun_nexus_detach(pcb);
1983
1984 /* Decrement refcnt to finish detaching and freeing */
1985 ifnet_decr_iorefcnt(ifp);
1986 } else
1987 #endif // UTUN_NEXUS
1988 {
1989 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1990
1991 #if UTUN_NEXUS
1992 if (!uuid_is_null(kpipe_uuid)) {
1993 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1994 if (pcb->utun_kpipe_pp != NULL) {
1995 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1996 pcb->utun_kpipe_pp = NULL;
1997 }
1998 utun_unregister_kernel_pipe_nexus();
1999 }
2000 }
2001 #endif // UTUN_NEXUS
2002
2003 /*
2004 * We want to do everything in our power to ensure that the interface
2005 * really goes away when the socket is closed. We must remove IP/IPv6
2006 * addresses and detach the protocols. Finally, we can remove and
2007 * release the interface.
2008 */
2009 utun_cleanup_family(ifp, AF_INET);
2010 utun_cleanup_family(ifp, AF_INET6);
2011
2012 /*
2013 * Detach now; utun_detach() will be called asynchronously once
2014 * the I/O reference count drops to 0. There we will invoke
2015 * ifnet_release().
2016 */
2017 if ((result = ifnet_detach(ifp)) != 0) {
2018 os_log_error(OS_LOG_DEFAULT, "utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
2019 }
2020 }
2021 } else {
2022 // Bound, but not connected
2023 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2024 utun_free_pcb(pcb, false);
2025 }
2026
2027 return 0;
2028 }
2029
2030 static errno_t
utun_ctl_send(__unused kern_ctl_ref kctlref,__unused u_int32_t unit,void * unitinfo,mbuf_t m,__unused int flags)2031 utun_ctl_send(__unused kern_ctl_ref kctlref,
2032 __unused u_int32_t unit,
2033 void *unitinfo,
2034 mbuf_t m,
2035 __unused int flags)
2036 {
2037 /*
2038 * The userland ABI requires the first four bytes have the protocol family
2039 * in network byte order: swap them
2040 */
2041 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
2042 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
2043 } else {
2044 os_log_error(OS_LOG_DEFAULT, "%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m));
2045 }
2046
2047 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
2048 }
2049
2050 static errno_t
utun_ctl_setopt(__unused kern_ctl_ref kctlref,__unused u_int32_t unit,void * unitinfo,int opt,void * data,size_t len)2051 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
2052 __unused u_int32_t unit,
2053 void *unitinfo,
2054 int opt,
2055 void *data,
2056 size_t len)
2057 {
2058 struct utun_pcb *pcb = unitinfo;
2059 errno_t result = 0;
2060 /* check for privileges for privileged options */
2061 switch (opt) {
2062 case UTUN_OPT_FLAGS:
2063 case UTUN_OPT_EXT_IFDATA_STATS:
2064 case UTUN_OPT_SET_DELEGATE_INTERFACE:
2065 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
2066 return EPERM;
2067 }
2068 break;
2069 }
2070
2071 switch (opt) {
2072 case UTUN_OPT_FLAGS:
2073 if (len != sizeof(u_int32_t)) {
2074 result = EMSGSIZE;
2075 break;
2076 }
2077 if (pcb->utun_ifp != NULL) {
2078 // Only can set before connecting
2079 result = EINVAL;
2080 break;
2081 }
2082 pcb->utun_flags = *(u_int32_t *)data;
2083 break;
2084
2085 case UTUN_OPT_EXT_IFDATA_STATS:
2086 if (len != sizeof(int)) {
2087 result = EMSGSIZE;
2088 break;
2089 }
2090 if (pcb->utun_ifp == NULL) {
2091 // Only can set after connecting
2092 result = EINVAL;
2093 break;
2094 }
2095 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
2096 break;
2097
2098 case UTUN_OPT_INC_IFDATA_STATS_IN:
2099 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
2100 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
2101
2102 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
2103 result = EINVAL;
2104 break;
2105 }
2106 if (pcb->utun_ifp == NULL) {
2107 // Only can set after connecting
2108 result = EINVAL;
2109 break;
2110 }
2111 if (!pcb->utun_ext_ifdata_stats) {
2112 result = EINVAL;
2113 break;
2114 }
2115 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) {
2116 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2117 utsp->utsp_bytes, utsp->utsp_errors);
2118 } else {
2119 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2120 utsp->utsp_bytes, utsp->utsp_errors);
2121 }
2122 break;
2123 }
2124 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2125 ifnet_t del_ifp = NULL;
2126 char name[IFNAMSIZ];
2127
2128 if (len > IFNAMSIZ - 1) {
2129 result = EMSGSIZE;
2130 break;
2131 }
2132 if (pcb->utun_ifp == NULL) {
2133 // Only can set after connecting
2134 result = EINVAL;
2135 break;
2136 }
2137 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2138 bcopy(data, name, len);
2139 name[len] = 0;
2140 result = ifnet_find_by_name(name, &del_ifp);
2141 }
2142 if (result == 0) {
2143 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2144 if (del_ifp) {
2145 ifnet_release(del_ifp);
2146 }
2147 }
2148 break;
2149 }
2150 case UTUN_OPT_MAX_PENDING_PACKETS: {
2151 u_int32_t max_pending_packets = 0;
2152 if (len != sizeof(u_int32_t)) {
2153 result = EMSGSIZE;
2154 break;
2155 }
2156 max_pending_packets = *(u_int32_t *)data;
2157 if (max_pending_packets == 0) {
2158 result = EINVAL;
2159 break;
2160 }
2161 pcb->utun_max_pending_packets = max_pending_packets;
2162 break;
2163 }
2164 #if UTUN_NEXUS
2165 case UTUN_OPT_ENABLE_CHANNEL: {
2166 if (len != sizeof(int)) {
2167 result = EMSGSIZE;
2168 break;
2169 }
2170 if (pcb->utun_ifp == NULL) {
2171 // Only can set after connecting
2172 result = EINVAL;
2173 break;
2174 }
2175 if (*(int *)data) {
2176 result = utun_enable_channel(pcb, current_proc());
2177 } else {
2178 result = utun_disable_channel(pcb);
2179 }
2180 break;
2181 }
2182 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2183 if (len != sizeof(int)) {
2184 result = EMSGSIZE;
2185 break;
2186 }
2187 if (pcb->utun_ifp == NULL) {
2188 // Only can set after connecting
2189 result = EINVAL;
2190 break;
2191 }
2192 if (!if_is_fsw_transport_netagent_enabled()) {
2193 result = ENOTSUP;
2194 break;
2195 }
2196 if (uuid_is_null(pcb->utun_nx.fsw_agent)) {
2197 result = ENOENT;
2198 break;
2199 }
2200
2201 uint32_t flags = netagent_get_flags(pcb->utun_nx.fsw_agent);
2202
2203 if (*(int *)data) {
2204 pcb->utun_needs_netagent = true;
2205 flags |= (NETAGENT_FLAG_NEXUS_PROVIDER |
2206 NETAGENT_FLAG_NEXUS_LISTENER);
2207 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2208 } else {
2209 flags &= ~(NETAGENT_FLAG_NEXUS_PROVIDER |
2210 NETAGENT_FLAG_NEXUS_LISTENER);
2211 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2212 pcb->utun_needs_netagent = false;
2213 }
2214 break;
2215 }
2216 case UTUN_OPT_ATTACH_FLOWSWITCH: {
2217 if (len != sizeof(int)) {
2218 result = EMSGSIZE;
2219 break;
2220 }
2221 if (pcb->utun_ifp != NULL) {
2222 // Only can set before connecting
2223 result = EINVAL;
2224 break;
2225 }
2226 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2227 pcb->utun_attach_fsw = !!(*(int *)data);
2228 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2229 break;
2230 }
2231 case UTUN_OPT_ENABLE_NETIF: {
2232 if (len != sizeof(int)) {
2233 result = EMSGSIZE;
2234 break;
2235 }
2236 if (pcb->utun_ifp != NULL) {
2237 // Only can set before connecting
2238 result = EINVAL;
2239 break;
2240 }
2241 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2242 pcb->utun_use_netif = !!(*(int *)data);
2243 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2244 break;
2245 }
2246 case UTUN_OPT_SLOT_SIZE: {
2247 if (len != sizeof(u_int32_t)) {
2248 result = EMSGSIZE;
2249 break;
2250 }
2251 if (pcb->utun_ifp != NULL) {
2252 // Only can set before connecting
2253 result = EINVAL;
2254 break;
2255 }
2256 u_int32_t slot_size = *(u_int32_t *)data;
2257 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2258 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2259 return EINVAL;
2260 }
2261 pcb->utun_slot_size = slot_size;
2262 break;
2263 }
2264 case UTUN_OPT_NETIF_RING_SIZE: {
2265 if (len != sizeof(u_int32_t)) {
2266 result = EMSGSIZE;
2267 break;
2268 }
2269 if (pcb->utun_ifp != NULL) {
2270 // Only can set before connecting
2271 result = EINVAL;
2272 break;
2273 }
2274 u_int32_t ring_size = *(u_int32_t *)data;
2275 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2276 ring_size > UTUN_IF_MAX_RING_SIZE) {
2277 return EINVAL;
2278 }
2279 pcb->utun_netif_ring_size = ring_size;
2280 break;
2281 }
2282 case UTUN_OPT_TX_FSW_RING_SIZE: {
2283 if (len != sizeof(u_int32_t)) {
2284 result = EMSGSIZE;
2285 break;
2286 }
2287 if (pcb->utun_ifp != NULL) {
2288 // Only can set before connecting
2289 result = EINVAL;
2290 break;
2291 }
2292 u_int32_t ring_size = *(u_int32_t *)data;
2293 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2294 ring_size > UTUN_IF_MAX_RING_SIZE) {
2295 return EINVAL;
2296 }
2297 pcb->utun_tx_fsw_ring_size = ring_size;
2298 break;
2299 }
2300 case UTUN_OPT_RX_FSW_RING_SIZE: {
2301 if (len != sizeof(u_int32_t)) {
2302 result = EMSGSIZE;
2303 break;
2304 }
2305 if (pcb->utun_ifp != NULL) {
2306 // Only can set before connecting
2307 result = EINVAL;
2308 break;
2309 }
2310 u_int32_t ring_size = *(u_int32_t *)data;
2311 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2312 ring_size > UTUN_IF_MAX_RING_SIZE) {
2313 return EINVAL;
2314 }
2315 pcb->utun_rx_fsw_ring_size = ring_size;
2316 break;
2317 }
2318 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2319 if (len != sizeof(u_int32_t)) {
2320 result = EMSGSIZE;
2321 break;
2322 }
2323 if (pcb->utun_ifp != NULL) {
2324 // Only can set before connecting
2325 result = EINVAL;
2326 break;
2327 }
2328 u_int32_t ring_size = *(u_int32_t *)data;
2329 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2330 ring_size > UTUN_IF_MAX_RING_SIZE) {
2331 return EINVAL;
2332 }
2333 pcb->utun_kpipe_tx_ring_size = ring_size;
2334 break;
2335 }
2336 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2337 if (len != sizeof(u_int32_t)) {
2338 result = EMSGSIZE;
2339 break;
2340 }
2341 if (pcb->utun_ifp != NULL) {
2342 // Only can set before connecting
2343 result = EINVAL;
2344 break;
2345 }
2346 u_int32_t ring_size = *(u_int32_t *)data;
2347 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2348 ring_size > UTUN_IF_MAX_RING_SIZE) {
2349 return EINVAL;
2350 }
2351 pcb->utun_kpipe_rx_ring_size = ring_size;
2352 break;
2353 }
2354 #endif // UTUN_NEXUS
2355 default: {
2356 result = ENOPROTOOPT;
2357 break;
2358 }
2359 }
2360
2361 return result;
2362 }
2363
2364 static errno_t
utun_ctl_getopt(__unused kern_ctl_ref kctlref,__unused u_int32_t unit,void * unitinfo,int opt,void * data,size_t * len)2365 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2366 __unused u_int32_t unit,
2367 void *unitinfo,
2368 int opt,
2369 void *data,
2370 size_t *len)
2371 {
2372 struct utun_pcb *pcb = unitinfo;
2373 errno_t result = 0;
2374
2375 switch (opt) {
2376 case UTUN_OPT_FLAGS:
2377 if (*len != sizeof(u_int32_t)) {
2378 result = EMSGSIZE;
2379 } else {
2380 *(u_int32_t *)data = pcb->utun_flags;
2381 }
2382 break;
2383
2384 case UTUN_OPT_EXT_IFDATA_STATS:
2385 if (*len != sizeof(int)) {
2386 result = EMSGSIZE;
2387 } else {
2388 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2389 }
2390 break;
2391
2392 case UTUN_OPT_IFNAME:
2393 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2394 result = EMSGSIZE;
2395 } else {
2396 if (pcb->utun_ifp == NULL) {
2397 // Only can get after connecting
2398 result = EINVAL;
2399 break;
2400 }
2401 *len = scnprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2402 }
2403 break;
2404
2405 case UTUN_OPT_MAX_PENDING_PACKETS: {
2406 if (*len != sizeof(u_int32_t)) {
2407 result = EMSGSIZE;
2408 } else {
2409 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2410 }
2411 break;
2412 }
2413
2414 #if UTUN_NEXUS
2415 case UTUN_OPT_ENABLE_CHANNEL: {
2416 if (*len != sizeof(int)) {
2417 result = EMSGSIZE;
2418 } else {
2419 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2420 *(int *)data = pcb->utun_kpipe_enabled;
2421 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2422 }
2423 break;
2424 }
2425
2426 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2427 if (*len != sizeof(int)) {
2428 result = EMSGSIZE;
2429 } else {
2430 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.fsw_agent);
2431 }
2432 break;
2433 }
2434
2435 case UTUN_OPT_ENABLE_NETIF: {
2436 if (*len != sizeof(int)) {
2437 result = EMSGSIZE;
2438 } else {
2439 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2440 *(int *)data = !!pcb->utun_use_netif;
2441 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2442 }
2443 break;
2444 }
2445
2446 case UTUN_OPT_GET_CHANNEL_UUID: {
2447 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2448 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2449 result = ENXIO;
2450 } else if (*len != sizeof(uuid_t)) {
2451 result = EMSGSIZE;
2452 } else {
2453 uuid_copy(data, pcb->utun_kpipe_uuid);
2454 }
2455 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2456 break;
2457 }
2458 case UTUN_OPT_SLOT_SIZE: {
2459 if (*len != sizeof(u_int32_t)) {
2460 result = EMSGSIZE;
2461 } else {
2462 *(u_int32_t *)data = pcb->utun_slot_size;
2463 }
2464 break;
2465 }
2466 case UTUN_OPT_NETIF_RING_SIZE: {
2467 if (*len != sizeof(u_int32_t)) {
2468 result = EMSGSIZE;
2469 } else {
2470 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2471 }
2472 break;
2473 }
2474 case UTUN_OPT_TX_FSW_RING_SIZE: {
2475 if (*len != sizeof(u_int32_t)) {
2476 result = EMSGSIZE;
2477 } else {
2478 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2479 }
2480 break;
2481 }
2482 case UTUN_OPT_RX_FSW_RING_SIZE: {
2483 if (*len != sizeof(u_int32_t)) {
2484 result = EMSGSIZE;
2485 } else {
2486 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2487 }
2488 break;
2489 }
2490 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2491 if (*len != sizeof(u_int32_t)) {
2492 result = EMSGSIZE;
2493 } else {
2494 *(u_int32_t *)data = pcb->utun_kpipe_tx_ring_size;
2495 }
2496 break;
2497 }
2498 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2499 if (*len != sizeof(u_int32_t)) {
2500 result = EMSGSIZE;
2501 } else {
2502 *(u_int32_t *)data = pcb->utun_kpipe_rx_ring_size;
2503 }
2504 break;
2505 }
2506 #endif // UTUN_NEXUS
2507
2508 default:
2509 result = ENOPROTOOPT;
2510 break;
2511 }
2512
2513 return result;
2514 }
2515
2516 static void
utun_ctl_rcvd(kern_ctl_ref kctlref,u_int32_t unit,void * unitinfo,int flags)2517 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2518 {
2519 #pragma unused(flags)
2520 bool reenable_output = false;
2521 struct utun_pcb *pcb = unitinfo;
2522 if (pcb == NULL) {
2523 return;
2524 }
2525 ifnet_lock_exclusive(pcb->utun_ifp);
2526
2527 u_int32_t utun_packet_cnt;
2528 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2529 if (error_pc != 0) {
2530 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2531 utun_packet_cnt = 0;
2532 }
2533
2534 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2535 reenable_output = true;
2536 }
2537
2538 if (reenable_output) {
2539 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2540 if (error != 0) {
2541 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2542 }
2543 }
2544 ifnet_lock_done(pcb->utun_ifp);
2545 }
2546
2547 /* Network Interface functions */
2548 static void
utun_start(ifnet_t interface)2549 utun_start(ifnet_t interface)
2550 {
2551 mbuf_t data;
2552 struct utun_pcb *pcb = ifnet_softc(interface);
2553
2554 VERIFY(pcb != NULL);
2555
2556 #if UTUN_NEXUS
2557 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2558 if (pcb->utun_kpipe_enabled) {
2559 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2560 if (!utun_data_move_begin(pcb)) {
2561 os_log_info(OS_LOG_DEFAULT,
2562 "%s: data path stopped for %s\n",
2563 __func__, if_name(pcb->utun_ifp));
2564 return;
2565 }
2566 /* It's possible to have channels enabled, but not yet have the channel opened,
2567 * in which case the rxring will not be set
2568 */
2569 if (pcb->utun_kpipe_rxring != NULL) {
2570 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2571 }
2572 utun_data_move_end(pcb);
2573 return;
2574 }
2575 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2576 #endif // UTUN_NEXUS
2577
2578 for (;;) {
2579 bool can_accept_packets = true;
2580 ifnet_lock_shared(pcb->utun_ifp);
2581
2582 u_int32_t utun_packet_cnt;
2583 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2584 if (error_pc != 0) {
2585 os_log_error(OS_LOG_DEFAULT, "utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2586 utun_packet_cnt = 0;
2587 }
2588
2589 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2590 if (!can_accept_packets && pcb->utun_ctlref) {
2591 u_int32_t difference = 0;
2592 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2593 if (difference > 0) {
2594 // If the low-water mark has not yet been reached, we still need to enqueue data
2595 // into the buffer
2596 can_accept_packets = true;
2597 }
2598 }
2599 }
2600 if (!can_accept_packets) {
2601 errno_t error = ifnet_disable_output(interface);
2602 if (error != 0) {
2603 os_log_error(OS_LOG_DEFAULT, "utun_start: ifnet_disable_output returned error %d\n", error);
2604 }
2605 ifnet_lock_done(pcb->utun_ifp);
2606 break;
2607 }
2608 ifnet_lock_done(pcb->utun_ifp);
2609 if (ifnet_dequeue(interface, &data) != 0) {
2610 break;
2611 }
2612 if (utun_output(interface, data) != 0) {
2613 break;
2614 }
2615 }
2616 }
2617
2618 static errno_t
utun_output(ifnet_t interface,mbuf_t data)2619 utun_output(ifnet_t interface,
2620 mbuf_t data)
2621 {
2622 struct utun_pcb *pcb = ifnet_softc(interface);
2623 errno_t result;
2624
2625 VERIFY(interface == pcb->utun_ifp);
2626
2627 #if UTUN_NEXUS
2628 if (!pcb->utun_use_netif)
2629 #endif // UTUN_NEXUS
2630 {
2631 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2632 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2633 }
2634 }
2635
2636 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2637 /* flush data */
2638 mbuf_freem(data);
2639 return 0;
2640 }
2641
2642 // otherwise, fall thru to ctl_enqueumbuf
2643 if (pcb->utun_ctlref) {
2644 int length;
2645
2646 /*
2647 * The ABI requires the protocol in network byte order
2648 */
2649 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2650 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2651 }
2652
2653 length = mbuf_pkthdr_len(data);
2654 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2655 if (result != 0) {
2656 mbuf_freem(data);
2657 os_log_error(OS_LOG_DEFAULT, "utun_output - ctl_enqueuembuf failed: %d\n", result);
2658 #if UTUN_NEXUS
2659 if (!pcb->utun_use_netif)
2660 #endif // UTUN_NEXUS
2661 {
2662 ifnet_stat_increment_out(interface, 0, 0, 1);
2663 }
2664 } else {
2665 #if UTUN_NEXUS
2666 if (!pcb->utun_use_netif)
2667 #endif // UTUN_NEXUS
2668 {
2669 if (!pcb->utun_ext_ifdata_stats) {
2670 ifnet_stat_increment_out(interface, 1, length, 0);
2671 }
2672 }
2673 }
2674 } else {
2675 mbuf_freem(data);
2676 }
2677
2678 return 0;
2679 }
2680
2681 static errno_t
utun_demux(__unused ifnet_t interface,mbuf_t data,__unused char * frame_header,protocol_family_t * protocol)2682 utun_demux(__unused ifnet_t interface,
2683 mbuf_t data,
2684 __unused char *frame_header,
2685 protocol_family_t *protocol)
2686 {
2687 #if UTUN_NEXUS
2688 struct utun_pcb *pcb = ifnet_softc(interface);
2689 struct ip *ip;
2690 u_int ip_version;
2691 #endif
2692
2693 while (data != NULL && mbuf_len(data) < 1) {
2694 data = mbuf_next(data);
2695 }
2696
2697 if (data == NULL) {
2698 return ENOENT;
2699 }
2700
2701 #if UTUN_NEXUS
2702 if (pcb->utun_use_netif) {
2703 ip = mtod(data, struct ip *);
2704 ip_version = ip->ip_v;
2705
2706 switch (ip_version) {
2707 case 4:
2708 *protocol = PF_INET;
2709 return 0;
2710 case 6:
2711 *protocol = PF_INET6;
2712 return 0;
2713 default:
2714 *protocol = 0;
2715 break;
2716 }
2717 } else
2718 #endif // UTUN_NEXUS
2719 {
2720 *protocol = *(u_int32_t *)mbuf_data(data);
2721 }
2722
2723 return 0;
2724 }
2725
2726 static errno_t
utun_framer(ifnet_t interface,mbuf_t * packet,__unused const struct sockaddr * dest,__unused const char * desk_linkaddr,const char * frame_type,u_int32_t * prepend_len,u_int32_t * postpend_len)2727 utun_framer(ifnet_t interface,
2728 mbuf_t *packet,
2729 __unused const struct sockaddr *dest,
2730 __unused const char *desk_linkaddr,
2731 const char *frame_type,
2732 u_int32_t *prepend_len,
2733 u_int32_t *postpend_len)
2734 {
2735 struct utun_pcb *pcb = ifnet_softc(interface);
2736 VERIFY(interface == pcb->utun_ifp);
2737
2738 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2739 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2740 os_log_error(OS_LOG_DEFAULT, "utun_framer - ifnet_output prepend failed\n");
2741
2742 ifnet_stat_increment_out(interface, 0, 0, 1);
2743
2744 // just return, because the buffer was freed in mbuf_prepend
2745 return EJUSTRETURN;
2746 }
2747 if (prepend_len != NULL) {
2748 *prepend_len = header_length;
2749 }
2750 if (postpend_len != NULL) {
2751 *postpend_len = 0;
2752 }
2753
2754 // place protocol number at the beginning of the mbuf
2755 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2756
2757 #if NECP
2758 // Add process uuid if applicable
2759 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
2760 if (m_pktlen(*packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2761 u_int8_t *header = (u_int8_t *)mbuf_data(*packet);
2762 int uuid_err = necp_get_app_uuid_from_packet(*packet, (void *)(header + sizeof(u_int32_t)));
2763 if (uuid_err != 0) {
2764 os_log_error(OS_LOG_DEFAULT, "Received app uuid error %d for %s%d\n", uuid_err, ifnet_name(pcb->utun_ifp), ifnet_unit(pcb->utun_ifp));
2765 }
2766 } else {
2767 os_log_error(OS_LOG_DEFAULT, "Cannot set proc uuid for %s%d, size %d < %zu\n", ifnet_name(pcb->utun_ifp), ifnet_unit(pcb->utun_ifp),
2768 m_pktlen(*packet), UTUN_HEADER_SIZE(pcb));
2769 }
2770 }
2771 #endif // NECP
2772
2773 return 0;
2774 }
2775
2776 static errno_t
utun_add_proto(__unused ifnet_t interface,protocol_family_t protocol,__unused const struct ifnet_demux_desc * demux_array,__unused u_int32_t demux_count)2777 utun_add_proto(__unused ifnet_t interface,
2778 protocol_family_t protocol,
2779 __unused const struct ifnet_demux_desc *demux_array,
2780 __unused u_int32_t demux_count)
2781 {
2782 switch (protocol) {
2783 case PF_INET:
2784 return 0;
2785 case PF_INET6:
2786 return 0;
2787 default:
2788 break;
2789 }
2790
2791 return ENOPROTOOPT;
2792 }
2793
2794 static errno_t
utun_del_proto(__unused ifnet_t interface,__unused protocol_family_t protocol)2795 utun_del_proto(__unused ifnet_t interface,
2796 __unused protocol_family_t protocol)
2797 {
2798 return 0;
2799 }
2800
2801 static errno_t
utun_ioctl(ifnet_t interface,u_long command,void * data)2802 utun_ioctl(ifnet_t interface,
2803 u_long command,
2804 void *data)
2805 {
2806 #if UTUN_NEXUS
2807 struct utun_pcb *pcb = ifnet_softc(interface);
2808 #endif
2809 errno_t result = 0;
2810
2811 switch (command) {
2812 case SIOCSIFMTU: {
2813 #if UTUN_NEXUS
2814 if (pcb->utun_use_netif) {
2815 // Make sure we can fit packets in the channel buffers
2816 // Allow for the headroom in the slot
2817 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2818 result = EINVAL;
2819 } else {
2820 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2821 }
2822 } else
2823 #endif // UTUN_NEXUS
2824 {
2825 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2826 }
2827 break;
2828 }
2829
2830 case SIOCSIFFLAGS:
2831 /* ifioctl() takes care of it */
2832 break;
2833
2834 default:
2835 result = EOPNOTSUPP;
2836 }
2837
2838 return result;
2839 }
2840
2841 static void
utun_detached(ifnet_t interface)2842 utun_detached(ifnet_t interface)
2843 {
2844 struct utun_pcb *pcb = ifnet_softc(interface);
2845 (void)ifnet_release(interface);
2846 lck_mtx_lock(&utun_lock);
2847 utun_free_pcb(pcb, true);
2848 (void)ifnet_dispose(interface);
2849 lck_mtx_unlock(&utun_lock);
2850 }
2851
2852 /* Protocol Handlers */
2853
2854 static errno_t
utun_proto_input(__unused ifnet_t interface,protocol_family_t protocol,mbuf_t m,__unused char * frame_header)2855 utun_proto_input(__unused ifnet_t interface,
2856 protocol_family_t protocol,
2857 mbuf_t m,
2858 __unused char *frame_header)
2859 {
2860 struct utun_pcb *pcb = ifnet_softc(interface);
2861 #if UTUN_NEXUS
2862 if (!pcb->utun_use_netif)
2863 #endif // UTUN_NEXUS
2864 {
2865 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2866 }
2867 int32_t pktlen = m->m_pkthdr.len;
2868 if (proto_input(protocol, m) != 0) {
2869 m_freem(m);
2870 #if UTUN_NEXUS
2871 if (!pcb->utun_use_netif)
2872 #endif // UTUN_NEXUS
2873 {
2874 ifnet_stat_increment_in(interface, 0, 0, 1);
2875 }
2876 } else {
2877 #if UTUN_NEXUS
2878 if (!pcb->utun_use_netif)
2879 #endif // UTUN_NEXUS
2880 {
2881 ifnet_stat_increment_in(interface, 1, pktlen, 0);
2882 }
2883 }
2884
2885 return 0;
2886 }
2887
2888 static errno_t
utun_proto_pre_output(__unused ifnet_t interface,protocol_family_t protocol,__unused mbuf_t * packet,__unused const struct sockaddr * dest,__unused void * route,char * frame_type,__unused char * link_layer_dest)2889 utun_proto_pre_output(__unused ifnet_t interface,
2890 protocol_family_t protocol,
2891 __unused mbuf_t *packet,
2892 __unused const struct sockaddr *dest,
2893 __unused void *route,
2894 char *frame_type,
2895 __unused char *link_layer_dest)
2896 {
2897 *(protocol_family_t *)(void *)frame_type = protocol;
2898 return 0;
2899 }
2900
2901 static errno_t
utun_attach_proto(ifnet_t interface,protocol_family_t protocol)2902 utun_attach_proto(ifnet_t interface,
2903 protocol_family_t protocol)
2904 {
2905 struct ifnet_attach_proto_param proto;
2906
2907 bzero(&proto, sizeof(proto));
2908 proto.input = utun_proto_input;
2909 proto.pre_output = utun_proto_pre_output;
2910
2911 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2912 if (result != 0 && result != EEXIST) {
2913 os_log_error(OS_LOG_DEFAULT, "utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2914 protocol, result);
2915 }
2916
2917 return result;
2918 }
2919
2920 static errno_t
utun_pkt_input(struct utun_pcb * pcb,mbuf_t packet)2921 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2922 {
2923 #if UTUN_NEXUS
2924 if (pcb->utun_use_netif) {
2925 if (!utun_data_move_begin(pcb)) {
2926 os_log_info(OS_LOG_DEFAULT,
2927 "%s: data path stopped for %s\n",
2928 __func__, if_name(pcb->utun_ifp));
2929 return ENXIO;
2930 }
2931
2932 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2933
2934 lck_mtx_lock(&pcb->utun_input_chain_lock);
2935
2936 if (pcb->utun_input_chain_count > (u_int32_t)if_utun_max_pending_input) {
2937 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2938 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2939 utun_data_move_end(pcb);
2940 return ENOSPC;
2941 }
2942
2943 if (pcb->utun_input_chain != NULL) {
2944 pcb->utun_input_chain_last->m_nextpkt = packet;
2945 } else {
2946 pcb->utun_input_chain = packet;
2947 }
2948 pcb->utun_input_chain_count++;
2949 while (packet->m_nextpkt) {
2950 VERIFY(packet != packet->m_nextpkt);
2951 packet = packet->m_nextpkt;
2952 pcb->utun_input_chain_count++;
2953 }
2954 pcb->utun_input_chain_last = packet;
2955 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2956
2957 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2958 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2959
2960 if (rx_ring != NULL) {
2961 kern_channel_notify(rx_ring, 0);
2962 }
2963
2964 utun_data_move_end(pcb);
2965 return 0;
2966 } else
2967 #endif // UTUN_NEXUS
2968 {
2969 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
2970
2971 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2972 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2973 }
2974 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2975 /* flush data */
2976 mbuf_freem(packet);
2977 return 0;
2978 }
2979
2980 errno_t result = 0;
2981 if (!pcb->utun_ext_ifdata_stats) {
2982 struct ifnet_stat_increment_param incs = {};
2983 incs.packets_in = 1;
2984 incs.bytes_in = mbuf_pkthdr_len(packet);
2985 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2986 } else {
2987 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2988 }
2989 if (result != 0) {
2990 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2991
2992 os_log_error(OS_LOG_DEFAULT, "%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2993 mbuf_freem(packet);
2994 }
2995
2996 return 0;
2997 }
2998 }
2999
3000 #if UTUN_NEXUS
3001
3002 static errno_t
utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)3003 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
3004 {
3005 return 0;
3006 }
3007
3008 static void
utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)3009 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
3010 {
3011 // Ignore
3012 }
3013
3014 static errno_t
utun_register_nexus(void)3015 utun_register_nexus(void)
3016 {
3017 const struct kern_nexus_domain_provider_init dp_init = {
3018 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
3019 .nxdpi_flags = 0,
3020 .nxdpi_init = utun_nxdp_init,
3021 .nxdpi_fini = utun_nxdp_fini
3022 };
3023 errno_t err = 0;
3024
3025 /* utun_nxdp_init() is called before this function returns */
3026 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
3027 (const uint8_t *) "com.apple.utun",
3028 &dp_init, sizeof(dp_init),
3029 &utun_nx_dom_prov);
3030 if (err != 0) {
3031 os_log_error(OS_LOG_DEFAULT, "%s: failed to register domain provider\n", __func__);
3032 return err;
3033 }
3034 return 0;
3035 }
3036 boolean_t
utun_interface_needs_netagent(ifnet_t interface)3037 utun_interface_needs_netagent(ifnet_t interface)
3038 {
3039 struct utun_pcb *pcb = NULL;
3040
3041 if (interface == NULL) {
3042 return FALSE;
3043 }
3044
3045 pcb = ifnet_softc(interface);
3046
3047 if (pcb == NULL) {
3048 return FALSE;
3049 }
3050
3051 return pcb->utun_needs_netagent == true;
3052 }
3053
3054 static errno_t
utun_ifnet_set_attrs(ifnet_t ifp)3055 utun_ifnet_set_attrs(ifnet_t ifp)
3056 {
3057 /* Set flags and additional information. */
3058 ifnet_set_mtu(ifp, 1500);
3059 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
3060
3061 /* The interface must generate its own IPv6 LinkLocal address,
3062 * if possible following the recommendation of RFC2472 to the 64bit interface ID
3063 */
3064 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
3065
3066 return 0;
3067 }
3068
3069 static errno_t
utun_netif_prepare(kern_nexus_t nexus,ifnet_t ifp)3070 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
3071 {
3072 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3073 pcb->utun_netif_nexus = nexus;
3074 return utun_ifnet_set_attrs(ifp);
3075 }
3076
3077 static errno_t
utun_nexus_pre_connect(kern_nexus_provider_t nxprov,proc_t p,kern_nexus_t nexus,nexus_port_t nexus_port,kern_channel_t channel,void ** ch_ctx)3078 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
3079 proc_t p, kern_nexus_t nexus,
3080 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
3081 {
3082 #pragma unused(nxprov, p)
3083 #pragma unused(nexus, nexus_port, channel, ch_ctx)
3084 return 0;
3085 }
3086
3087 static errno_t
utun_nexus_connected(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel)3088 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3089 kern_channel_t channel)
3090 {
3091 #pragma unused(nxprov, channel)
3092 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3093 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
3094 if (pcb->utun_netif_nexus == nexus) {
3095 pcb->utun_netif_connected = true;
3096 }
3097 if (ok) {
3098 lck_mtx_lock(&pcb->utun_pcb_data_move_lock);
3099 UTUN_SET_DATA_PATH_READY(pcb);
3100 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
3101 }
3102 return ok ? 0 : ENXIO;
3103 }
3104
3105 static void
utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel)3106 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3107 kern_channel_t channel)
3108 {
3109 #pragma unused(nxprov, channel)
3110 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3111 /* Wait until all threads in the data paths are done. */
3112 utun_wait_data_move_drain(pcb);
3113 }
3114
3115 static void
utun_netif_pre_disconnect(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel)3116 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3117 kern_channel_t channel)
3118 {
3119 #pragma unused(nxprov, channel)
3120 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3121 /* Wait until all threads in the data paths are done. */
3122 utun_wait_data_move_drain(pcb);
3123 }
3124
3125 static void
utun_nexus_disconnected(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel)3126 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3127 kern_channel_t channel)
3128 {
3129 #pragma unused(nxprov, channel)
3130 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3131 if (pcb->utun_netif_nexus == nexus) {
3132 pcb->utun_netif_connected = false;
3133 if (pcb->utun_attach_fsw) {
3134 // disconnected by flowswitch that was attached by us
3135 pcb->utun_netif_nexus = NULL;
3136 }
3137 }
3138 ifnet_decr_iorefcnt(pcb->utun_ifp);
3139 }
3140
3141 static errno_t
utun_kpipe_ring_init(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_t channel,kern_channel_ring_t ring,boolean_t is_tx_ring,void ** ring_ctx)3142 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3143 kern_channel_t channel, kern_channel_ring_t ring,
3144 boolean_t is_tx_ring, void **ring_ctx)
3145 {
3146 #pragma unused(nxprov)
3147 #pragma unused(channel)
3148 #pragma unused(ring_ctx)
3149 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3150 if (!is_tx_ring) {
3151 VERIFY(pcb->utun_kpipe_rxring == NULL);
3152 pcb->utun_kpipe_rxring = ring;
3153 } else {
3154 VERIFY(pcb->utun_kpipe_txring == NULL);
3155 pcb->utun_kpipe_txring = ring;
3156 }
3157 return 0;
3158 }
3159
3160 static void
utun_kpipe_ring_fini(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t ring)3161 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3162 kern_channel_ring_t ring)
3163 {
3164 #pragma unused(nxprov)
3165 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3166 if (pcb->utun_kpipe_rxring == ring) {
3167 pcb->utun_kpipe_rxring = NULL;
3168 } else if (pcb->utun_kpipe_txring == ring) {
3169 pcb->utun_kpipe_txring = NULL;
3170 }
3171 }
3172
3173 static errno_t
utun_kpipe_sync_tx(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t tx_ring,uint32_t flags)3174 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3175 kern_channel_ring_t tx_ring, uint32_t flags)
3176 {
3177 #pragma unused(nxprov)
3178 #pragma unused(flags)
3179 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3180
3181 if (!utun_data_move_begin(pcb)) {
3182 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n",
3183 __func__, if_name(pcb->utun_ifp));
3184 return 0;
3185 }
3186
3187 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3188 int channel_enabled = pcb->utun_kpipe_enabled;
3189 if (!channel_enabled) {
3190 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3191 utun_data_move_end(pcb);
3192 return 0;
3193 }
3194
3195 if (pcb->utun_use_netif) {
3196 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3197 if (tx_slot == NULL) {
3198 // Nothing to write, bail
3199 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3200 utun_data_move_end(pcb);
3201 return 0;
3202 }
3203
3204 // Signal the netif ring to read
3205 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
3206 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3207 if (rx_ring != NULL) {
3208 kern_channel_notify(rx_ring, 0);
3209 }
3210 } else {
3211 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3212
3213 struct ifnet_stat_increment_param incs = {};
3214 struct kern_channel_ring_stat_increment tx_ring_stats = {};
3215 MBUFQ_HEAD(mbufq) mbq;
3216 MBUFQ_INIT(&mbq);
3217 kern_channel_slot_t tx_pslot = NULL;
3218 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3219 while (tx_slot != NULL) {
3220 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3221
3222 // Advance TX ring
3223 tx_pslot = tx_slot;
3224 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3225
3226 if (tx_ph == 0) {
3227 continue;
3228 }
3229
3230 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3231 VERIFY(tx_buf != NULL);
3232 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
3233 VERIFY(tx_baddr != 0);
3234 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3235
3236 size_t length = MIN(kern_packet_get_data_length(tx_ph),
3237 pcb->utun_slot_size);
3238
3239 mbuf_t data = NULL;
3240 if (length >= UTUN_HEADER_SIZE(pcb) &&
3241 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
3242 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
3243 VERIFY(0 == error);
3244 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
3245 VERIFY(0 == error);
3246 /*
3247 * The userland ABI requires the first four bytes have
3248 * the protocol family in network byte order: swap them
3249 */
3250 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
3251 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
3252 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3253 incs.packets_in++;
3254 incs.bytes_in += length;
3255 MBUFQ_ENQUEUE(&mbq, data);
3256 }
3257 }
3258 if (tx_pslot) {
3259 kern_channel_advance_slot(tx_ring, tx_pslot);
3260 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
3261 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3262 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3263 (void) kern_channel_reclaim(tx_ring);
3264 }
3265 if (!MBUFQ_EMPTY(&mbq)) {
3266 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
3267 MBUFQ_LAST(&mbq), &incs);
3268 MBUFQ_INIT(&mbq);
3269 }
3270 }
3271
3272 utun_data_move_end(pcb);
3273 return 0;
3274 }
3275
3276 static errno_t
utun_kpipe_sync_rx(kern_nexus_provider_t nxprov,kern_nexus_t nexus,kern_channel_ring_t rx_ring,uint32_t flags)3277 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3278 kern_channel_ring_t rx_ring, uint32_t flags)
3279 {
3280 #pragma unused(nxprov)
3281 #pragma unused(flags)
3282 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3283 struct kern_channel_ring_stat_increment rx_ring_stats = {};
3284
3285 if (!utun_data_move_begin(pcb)) {
3286 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n",
3287 __func__, if_name(pcb->utun_ifp));
3288 return 0;
3289 }
3290
3291 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3292
3293 int channel_enabled = pcb->utun_kpipe_enabled;
3294 if (!channel_enabled) {
3295 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3296 utun_data_move_end(pcb);
3297 return 0;
3298 }
3299
3300 /* reclaim user-released slots */
3301 (void) kern_channel_reclaim(rx_ring);
3302
3303 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3304 if (avail == 0) {
3305 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3306 utun_data_move_end(pcb);
3307 return 0;
3308 }
3309
3310 if (pcb->utun_use_netif) {
3311 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3312 if (tx_ring == NULL ||
3313 pcb->utun_netif_nexus == NULL) {
3314 // Net-If TX ring not set up yet, nothing to read
3315 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3316 utun_data_move_end(pcb);
3317 return 0;
3318 }
3319
3320 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
3321
3322 // Unlock utun before entering ring
3323 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3324
3325 (void)kr_enter(tx_ring, TRUE);
3326
3327 // Lock again after entering and validate
3328 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3329 if (tx_ring != pcb->utun_netif_txring) {
3330 // Ring no longer valid
3331 // Unlock first, then exit ring
3332 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3333 kr_exit(tx_ring);
3334 utun_data_move_end(pcb);
3335 return 0;
3336 }
3337
3338 struct kern_channel_ring_stat_increment tx_ring_stats;
3339 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3340 kern_channel_slot_t tx_pslot = NULL;
3341 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3342 if (tx_slot == NULL) {
3343 // Nothing to read, don't bother signalling
3344 // Unlock first, then exit ring
3345 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3346 kr_exit(tx_ring);
3347 utun_data_move_end(pcb);
3348 return 0;
3349 }
3350
3351 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3352 VERIFY(rx_pp != NULL);
3353 struct kern_pbufpool *tx_pp = tx_ring->ckr_pp;
3354 VERIFY(tx_pp != NULL);
3355 kern_channel_slot_t rx_pslot = NULL;
3356 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3357 kern_packet_t tx_chain_ph = 0;
3358
3359 while (rx_slot != NULL && tx_slot != NULL) {
3360 size_t length;
3361 kern_buflet_t rx_buf;
3362 void *rx_baddr;
3363
3364 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3365
3366 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3367 if (tx_ph == 0) {
3368 // Advance TX ring
3369 tx_pslot = tx_slot;
3370 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3371 continue;
3372 }
3373 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
3374 if (tx_chain_ph != 0) {
3375 kern_packet_append(tx_ph, tx_chain_ph);
3376 }
3377 tx_chain_ph = tx_ph;
3378
3379 // Advance TX ring
3380 tx_pslot = tx_slot;
3381 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3382
3383 // Allocate rx packet
3384 kern_packet_t rx_ph = 0;
3385 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3386 if (__improbable(error != 0)) {
3387 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3388 pcb->utun_ifp->if_xname);
3389 break;
3390 }
3391
3392 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3393 VERIFY(tx_buf != NULL);
3394 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
3395 VERIFY(tx_baddr != NULL);
3396 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3397
3398 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3399
3400 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
3401 pcb->utun_slot_size);
3402
3403 tx_ring_stats.kcrsi_slots_transferred++;
3404 tx_ring_stats.kcrsi_bytes_transferred += length;
3405
3406 if (length < UTUN_HEADER_SIZE(pcb) ||
3407 length > pcb->utun_slot_size ||
3408 length > rx_pp->pp_buflet_size ||
3409 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3410 /* flush data */
3411 kern_pbufpool_free(rx_pp, rx_ph);
3412 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3413 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3414 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
3415 STATS_INC(nifs, NETIF_STATS_DROP);
3416 continue;
3417 }
3418
3419 /* fillout packet */
3420 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3421 VERIFY(rx_buf != NULL);
3422 rx_baddr = kern_buflet_get_data_address(rx_buf);
3423 VERIFY(rx_baddr != NULL);
3424
3425 // Find family
3426 uint32_t af = 0;
3427 uint8_t vhl = *(uint8_t *)(tx_baddr);
3428 u_int ip_version = (vhl >> 4);
3429 switch (ip_version) {
3430 case 4: {
3431 af = AF_INET;
3432 break;
3433 }
3434 case 6: {
3435 af = AF_INET6;
3436 break;
3437 }
3438 default: {
3439 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3440 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3441 break;
3442 }
3443 }
3444
3445 // Copy header
3446 af = htonl(af);
3447 memcpy((void *)rx_baddr, &af, sizeof(af));
3448 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3449 kern_packet_get_euuid(tx_ph, (void *)((uintptr_t)rx_baddr + sizeof(af)));
3450 }
3451
3452 // Copy data from tx to rx
3453 memcpy((void *)((uintptr_t)rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3454 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
3455
3456 /* finalize and attach the packet */
3457 error = kern_buflet_set_data_offset(rx_buf, 0);
3458 VERIFY(error == 0);
3459 error = kern_buflet_set_data_length(rx_buf, length);
3460 VERIFY(error == 0);
3461 error = kern_packet_finalize(rx_ph);
3462 VERIFY(error == 0);
3463 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3464 VERIFY(error == 0);
3465
3466 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
3467 STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
3468
3469 rx_ring_stats.kcrsi_slots_transferred++;
3470 rx_ring_stats.kcrsi_bytes_transferred += length;
3471
3472 rx_pslot = rx_slot;
3473 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3474 }
3475
3476 if (rx_pslot) {
3477 kern_channel_advance_slot(rx_ring, rx_pslot);
3478 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3479 }
3480
3481 if (tx_chain_ph != 0) {
3482 kern_pbufpool_free_chain(tx_pp, tx_chain_ph);
3483 }
3484
3485 if (tx_pslot) {
3486 kern_channel_advance_slot(tx_ring, tx_pslot);
3487 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3488 (void)kern_channel_reclaim(tx_ring);
3489 }
3490
3491 /* just like utun_ctl_rcvd(), always reenable output */
3492 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3493 if (error != 0) {
3494 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
3495 }
3496
3497 // Unlock first, then exit ring
3498 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3499
3500 if (tx_pslot != NULL) {
3501 kern_channel_notify(tx_ring, 0);
3502 }
3503 kr_exit(tx_ring);
3504 } else {
3505 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3506
3507 uint32_t mb_cnt = 0;
3508 uint32_t mb_len = 0;
3509 struct mbuf *mb_head = NULL;
3510 struct mbuf *mb_tail = NULL;
3511
3512 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3513 &mb_tail, &mb_cnt, &mb_len) != 0) {
3514 utun_data_move_end(pcb);
3515 return 0;
3516 }
3517 VERIFY(mb_cnt <= avail);
3518
3519 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3520 VERIFY(rx_pp != NULL);
3521 kern_channel_slot_t rx_pslot = NULL;
3522 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3523 while (rx_slot) {
3524 size_t length = 0;
3525 mbuf_t data = NULL;
3526 if ((data = mb_head) == NULL) {
3527 VERIFY(mb_cnt == 0);
3528 break;
3529 }
3530 mb_head = mbuf_nextpkt(mb_head);
3531 mbuf_setnextpkt(data, NULL);
3532 VERIFY(mb_cnt != 0);
3533 --mb_cnt;
3534 length = mbuf_pkthdr_len(data);
3535 if (length < UTUN_HEADER_SIZE(pcb) ||
3536 length > pcb->utun_slot_size ||
3537 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3538 /* flush data */
3539 mbuf_freem(data);
3540 continue;
3541 }
3542 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3543
3544 // Allocate rx packet
3545 kern_packet_t rx_ph = 0;
3546 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3547 if (__improbable(error != 0)) {
3548 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3549 pcb->utun_ifp->if_xname);
3550 break;
3551 }
3552
3553 /*
3554 * The ABI requires the protocol in network byte order
3555 */
3556 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3557
3558 // Fillout rx packet
3559 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3560 VERIFY(rx_buf != NULL);
3561 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
3562 VERIFY(rx_baddr != NULL);
3563
3564 // Copy-in data from mbuf to buflet
3565 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3566 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3567
3568 // Finalize and attach the packet
3569 error = kern_buflet_set_data_offset(rx_buf, 0);
3570 VERIFY(error == 0);
3571 error = kern_buflet_set_data_length(rx_buf, length);
3572 VERIFY(error == 0);
3573 error = kern_packet_finalize(rx_ph);
3574 VERIFY(error == 0);
3575 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3576 VERIFY(error == 0);
3577
3578 rx_ring_stats.kcrsi_slots_transferred++;
3579 rx_ring_stats.kcrsi_bytes_transferred += length;
3580
3581 if (!pcb->utun_ext_ifdata_stats) {
3582 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3583 }
3584
3585 mbuf_freem(data);
3586
3587 rx_pslot = rx_slot;
3588 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3589 }
3590 if (rx_pslot) {
3591 kern_channel_advance_slot(rx_ring, rx_pslot);
3592 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3593 }
3594 if (mb_head != NULL) {
3595 VERIFY(mb_cnt != 0);
3596 mbuf_freem_list(mb_head);
3597 }
3598 }
3599
3600 utun_data_move_end(pcb);
3601 return 0;
3602 }
3603
3604 #endif // UTUN_NEXUS
3605
3606
3607 /*
3608 * These are place holders until coreTLS kext stops calling them
3609 */
3610 errno_t utun_ctl_register_dtls(void *reg);
3611 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3612 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3613
3614 errno_t
utun_ctl_register_dtls(void * reg)3615 utun_ctl_register_dtls(void *reg)
3616 {
3617 #pragma unused(reg)
3618 return 0;
3619 }
3620
3621 int
utun_pkt_dtls_input(struct utun_pcb * pcb,mbuf_t * pkt,protocol_family_t family)3622 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3623 {
3624 #pragma unused(pcb)
3625 #pragma unused(pkt)
3626 #pragma unused(family)
3627 return 0;
3628 }
3629
3630 void
utun_ctl_disable_crypto_dtls(struct utun_pcb * pcb)3631 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3632 {
3633 #pragma unused(pcb)
3634 }
3635
3636 #if UTUN_NEXUS
3637 static boolean_t
utun_data_move_begin(struct utun_pcb * pcb)3638 utun_data_move_begin(struct utun_pcb *pcb)
3639 {
3640 bool data_path_ready = false;
3641
3642 lck_mtx_lock_spin(&pcb->utun_pcb_data_move_lock);
3643 if ((data_path_ready = UTUN_IS_DATA_PATH_READY(pcb))) {
3644 pcb->utun_pcb_data_move++;
3645 }
3646 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
3647
3648 return data_path_ready;
3649 }
3650
3651 static void
utun_data_move_end(struct utun_pcb * pcb)3652 utun_data_move_end(struct utun_pcb *pcb)
3653 {
3654 lck_mtx_lock_spin(&pcb->utun_pcb_data_move_lock);
3655 VERIFY(pcb->utun_pcb_data_move > 0);
3656 /*
3657 * if there's no more thread moving data, wakeup any
3658 * drainers that are blocked waiting for this.
3659 */
3660 if (--pcb->utun_pcb_data_move == 0 && pcb->utun_pcb_drainers > 0) {
3661 wakeup(&(pcb->utun_pcb_data_move));
3662 }
3663 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
3664 }
3665
3666 static void
utun_data_move_drain(struct utun_pcb * pcb)3667 utun_data_move_drain(struct utun_pcb *pcb)
3668 {
3669 lck_mtx_lock(&pcb->utun_pcb_data_move_lock);
3670 /* data path must already be marked as not ready */
3671 VERIFY(!UTUN_IS_DATA_PATH_READY(pcb));
3672 pcb->utun_pcb_drainers++;
3673 while (pcb->utun_pcb_data_move != 0) {
3674 (void) msleep(&(pcb->utun_pcb_data_move),
3675 &pcb->utun_pcb_data_move_lock, (PZERO - 1), __func__, NULL);
3676 }
3677 VERIFY(!UTUN_IS_DATA_PATH_READY(pcb));
3678 VERIFY(pcb->utun_pcb_drainers > 0);
3679 pcb->utun_pcb_drainers--;
3680 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
3681 }
3682
3683 static void
utun_wait_data_move_drain(struct utun_pcb * pcb)3684 utun_wait_data_move_drain(struct utun_pcb *pcb)
3685 {
3686 /*
3687 * Mark the data path as not usable.
3688 */
3689 lck_mtx_lock(&pcb->utun_pcb_data_move_lock);
3690 UTUN_CLR_DATA_PATH_READY(pcb);
3691 lck_mtx_unlock(&pcb->utun_pcb_data_move_lock);
3692
3693 /* Wait until all threads in the data path are done. */
3694 utun_data_move_drain(pcb);
3695 }
3696 #endif // UTUN_NEXUS
3697