1 /*
2 * Copyright (c) 2012-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <kern/task.h>
49 #include <mach/task_info.h>
50 #include <net/if_var.h>
51 #include <net/route.h>
52 #include <net/flowhash.h>
53 #include <net/ntstat.h>
54 #include <net/content_filter.h>
55 #include <net/necp.h>
56 #include <netinet/in.h>
57 #include <netinet/in_var.h>
58 #include <netinet/tcp.h>
59 #include <netinet/tcp_var.h>
60 #include <netinet/tcp_fsm.h>
61 #include <netinet/flow_divert.h>
62 #include <netinet/flow_divert_proto.h>
63 #include <netinet6/in6_pcb.h>
64 #include <netinet6/ip6protosw.h>
65 #include <dev/random/randomdev.h>
66 #include <libkern/crypto/sha1.h>
67 #include <libkern/crypto/crypto_internal.h>
68 #include <os/log.h>
69 #include <corecrypto/cc.h>
70 #if CONTENT_FILTER
71 #include <net/content_filter.h>
72 #endif /* CONTENT_FILTER */
73
74 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75 #define FLOW_DIVERT_READ_CLOSED 0x00000002
76 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
79 #define FLOW_DIVERT_HAS_HMAC 0x00000040
80 #define FLOW_DIVERT_NOTIFY_ON_RECEIVED 0x00000080
81 #define FLOW_DIVERT_IMPLICIT_CONNECT 0x00000100
82 #define FLOW_DIVERT_DID_SET_LOCAL_ADDR 0x00000200
83 #define FLOW_DIVERT_HAS_TOKEN 0x00000400
84 #define FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR 0x00000800
85 #define FLOW_DIVERT_FLOW_IS_TRANSPARENT 0x00001000
86
87 #define FDLOG(level, pcb, format, ...) \
88 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
89
90 #define FDLOG0(level, pcb, msg) \
91 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
92
93 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
94 #define FDRELEASE(pcb) \
95 do { \
96 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
97 flow_divert_pcb_destroy(pcb); \
98 } \
99 } while (0)
100
101 #define FDGRP_RETAIN(grp) if ((grp) != NULL) OSIncrementAtomic(&(grp)->ref_count)
102 #define FDGRP_RELEASE(grp) if ((grp) != NULL && 1 == OSDecrementAtomic(&(grp)->ref_count)) flow_divert_group_destroy(grp)
103
104 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
105 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
106
107 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
108
109 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
110
111 #define GROUP_COUNT_MAX 31
112 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
113 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
114 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
115
116 struct flow_divert_trie_node {
117 uint16_t start;
118 uint16_t length;
119 uint16_t child_map;
120 };
121
122 #define CHILD_MAP_SIZE 256
123 #define NULL_TRIE_IDX 0xffff
124 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
125 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
126 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
127
128 #define SO_IS_DIVERTED(s) (((s)->so_flags & SOF_FLOW_DIVERT) && (s)->so_fd_pcb != NULL)
129
130 static struct flow_divert_pcb nil_pcb;
131
132 static LCK_ATTR_DECLARE(flow_divert_mtx_attr, 0, 0);
133 static LCK_GRP_DECLARE(flow_divert_mtx_grp, FLOW_DIVERT_CONTROL_NAME);
134 static LCK_RW_DECLARE_ATTR(g_flow_divert_group_lck, &flow_divert_mtx_grp,
135 &flow_divert_mtx_attr);
136
137 static struct flow_divert_group **g_flow_divert_groups = NULL;
138 static uint32_t g_active_group_count = 0;
139
140 static errno_t g_init_result = 0;
141
142 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
143
144 static struct protosw g_flow_divert_in_protosw;
145 static struct pr_usrreqs g_flow_divert_in_usrreqs;
146 static struct protosw g_flow_divert_in_udp_protosw;
147 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
148 static struct ip6protosw g_flow_divert_in6_protosw;
149 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
150 static struct ip6protosw g_flow_divert_in6_udp_protosw;
151 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
152
153 static struct protosw *g_tcp_protosw = NULL;
154 static struct ip6protosw *g_tcp6_protosw = NULL;
155 static struct protosw *g_udp_protosw = NULL;
156 static struct ip6protosw *g_udp6_protosw = NULL;
157
158 ZONE_DEFINE(flow_divert_group_zone, "flow_divert_group",
159 sizeof(struct flow_divert_group), ZC_ZFREE_CLEARMEM);
160 ZONE_DEFINE(flow_divert_pcb_zone, "flow_divert_pcb",
161 sizeof(struct flow_divert_pcb), ZC_ZFREE_CLEARMEM);
162
163 static errno_t
164 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
165
166 static boolean_t
167 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
168
169 static int
170 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
171
172 struct sockaddr *
173 flow_divert_get_buffered_target_address(mbuf_t buffer);
174
175 static void
176 flow_divert_disconnect_socket(struct socket *so, bool is_connected);
177
178 static void flow_divert_group_destroy(struct flow_divert_group *group);
179
180 static inline uint8_t
flow_divert_syslog_type_to_oslog_type(int syslog_type)181 flow_divert_syslog_type_to_oslog_type(int syslog_type)
182 {
183 switch (syslog_type) {
184 case LOG_ERR: return OS_LOG_TYPE_ERROR;
185 case LOG_INFO: return OS_LOG_TYPE_INFO;
186 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
187 default: return OS_LOG_TYPE_DEFAULT;
188 }
189 }
190
191 static inline int
flow_divert_pcb_cmp(const struct flow_divert_pcb * pcb_a,const struct flow_divert_pcb * pcb_b)192 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
193 {
194 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
195 }
196
197 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
198 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
199
200 static const char *
flow_divert_packet_type2str(uint8_t packet_type)201 flow_divert_packet_type2str(uint8_t packet_type)
202 {
203 switch (packet_type) {
204 case FLOW_DIVERT_PKT_CONNECT:
205 return "connect";
206 case FLOW_DIVERT_PKT_CONNECT_RESULT:
207 return "connect result";
208 case FLOW_DIVERT_PKT_DATA:
209 return "data";
210 case FLOW_DIVERT_PKT_CLOSE:
211 return "close";
212 case FLOW_DIVERT_PKT_READ_NOTIFY:
213 return "read notification";
214 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
215 return "properties update";
216 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
217 return "app map create";
218 default:
219 return "unknown";
220 }
221 }
222
223 static struct flow_divert_pcb *
flow_divert_pcb_lookup(uint32_t hash,struct flow_divert_group * group)224 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
225 {
226 struct flow_divert_pcb key_item;
227 struct flow_divert_pcb *fd_cb = NULL;
228
229 key_item.hash = hash;
230
231 lck_rw_lock_shared(&group->lck);
232 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
233 FDRETAIN(fd_cb);
234 lck_rw_done(&group->lck);
235
236 return fd_cb;
237 }
238
239 static struct flow_divert_group *
flow_divert_group_lookup(uint32_t ctl_unit,struct flow_divert_pcb * fd_cb)240 flow_divert_group_lookup(uint32_t ctl_unit, struct flow_divert_pcb *fd_cb)
241 {
242 struct flow_divert_group *group = NULL;
243 lck_rw_lock_shared(&g_flow_divert_group_lck);
244 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
245 if (fd_cb != NULL) {
246 FDLOG0(LOG_ERR, fd_cb, "No active groups, flow divert cannot be used for this socket");
247 }
248 } else if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
249 FDLOG(LOG_ERR, fd_cb, "Cannot lookup group with invalid control unit (%u)", ctl_unit);
250 } else {
251 group = g_flow_divert_groups[ctl_unit];
252 if (group == NULL) {
253 if (fd_cb != NULL) {
254 FDLOG(LOG_ERR, fd_cb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
255 }
256 } else {
257 FDGRP_RETAIN(group);
258 }
259 }
260 lck_rw_done(&g_flow_divert_group_lck);
261 return group;
262 }
263
264 static errno_t
flow_divert_pcb_insert(struct flow_divert_pcb * fd_cb,struct flow_divert_group * group)265 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, struct flow_divert_group *group)
266 {
267 int error = 0;
268 lck_rw_lock_exclusive(&group->lck);
269 if (!(group->flags & FLOW_DIVERT_GROUP_FLAG_DEFUNCT)) {
270 if (NULL == RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb)) {
271 fd_cb->group = group;
272 fd_cb->control_group_unit = group->ctl_unit;
273 FDRETAIN(fd_cb); /* The group now has a reference */
274 } else {
275 FDLOG(LOG_ERR, fd_cb, "Group %u already contains a PCB with hash %u", group->ctl_unit, fd_cb->hash);
276 error = EEXIST;
277 }
278 } else {
279 FDLOG(LOG_ERR, fd_cb, "Group %u is defunct, cannot insert", group->ctl_unit);
280 error = ENOENT;
281 }
282 lck_rw_done(&group->lck);
283 return error;
284 }
285
286 static errno_t
flow_divert_add_to_group(struct flow_divert_pcb * fd_cb,uint32_t ctl_unit)287 flow_divert_add_to_group(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
288 {
289 errno_t error = 0;
290 struct flow_divert_group *group = NULL;
291 static uint32_t g_nextkey = 1;
292 static uint32_t g_hash_seed = 0;
293 int try_count = 0;
294
295 group = flow_divert_group_lookup(ctl_unit, fd_cb);
296 if (group == NULL) {
297 return ENOENT;
298 }
299
300 do {
301 uint32_t key[2];
302 uint32_t idx;
303
304 key[0] = g_nextkey++;
305 key[1] = RandomULong();
306
307 if (g_hash_seed == 0) {
308 g_hash_seed = RandomULong();
309 }
310
311 error = 0;
312 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
313
314 for (idx = 1; idx < GROUP_COUNT_MAX && error == 0; idx++) {
315 if (idx == ctl_unit) {
316 continue;
317 }
318 struct flow_divert_group *curr_group = flow_divert_group_lookup(idx, NULL);
319 if (curr_group != NULL) {
320 lck_rw_lock_shared(&curr_group->lck);
321 if (NULL != RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb)) {
322 error = EEXIST;
323 }
324 lck_rw_done(&curr_group->lck);
325 FDGRP_RELEASE(curr_group);
326 }
327 }
328
329 if (error == 0) {
330 error = flow_divert_pcb_insert(fd_cb, group);
331 }
332 } while (error == EEXIST && try_count++ < 3);
333
334 if (error == EEXIST) {
335 FDLOG0(LOG_ERR, fd_cb, "Failed to create a unique hash");
336 fd_cb->hash = 0;
337 }
338
339 FDGRP_RELEASE(group);
340 return error;
341 }
342
343 static struct flow_divert_pcb *
flow_divert_pcb_create(socket_t so)344 flow_divert_pcb_create(socket_t so)
345 {
346 struct flow_divert_pcb *new_pcb = NULL;
347
348 new_pcb = zalloc_flags(flow_divert_pcb_zone, Z_WAITOK | Z_ZERO);
349 lck_mtx_init(&new_pcb->mtx, &flow_divert_mtx_grp, &flow_divert_mtx_attr);
350 new_pcb->so = so;
351 new_pcb->log_level = nil_pcb.log_level;
352
353 FDRETAIN(new_pcb); /* Represents the socket's reference */
354
355 return new_pcb;
356 }
357
358 static void
flow_divert_pcb_destroy(struct flow_divert_pcb * fd_cb)359 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
360 {
361 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u",
362 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
363
364 if (fd_cb->connect_token != NULL) {
365 mbuf_freem(fd_cb->connect_token);
366 }
367 if (fd_cb->connect_packet != NULL) {
368 mbuf_freem(fd_cb->connect_packet);
369 }
370 if (fd_cb->app_data != NULL) {
371 kfree_data(fd_cb->app_data, fd_cb->app_data_length);
372 }
373 if (fd_cb->original_remote_endpoint != NULL) {
374 free_sockaddr(fd_cb->original_remote_endpoint);
375 }
376 zfree(flow_divert_pcb_zone, fd_cb);
377 }
378
379 static void
flow_divert_pcb_remove(struct flow_divert_pcb * fd_cb)380 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
381 {
382 if (fd_cb->group != NULL) {
383 struct flow_divert_group *group = fd_cb->group;
384 lck_rw_lock_exclusive(&group->lck);
385 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
386 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
387 fd_cb->group = NULL;
388 FDRELEASE(fd_cb); /* Release the group's reference */
389 lck_rw_done(&group->lck);
390 }
391 }
392
393 static int
flow_divert_packet_init(struct flow_divert_pcb * fd_cb,uint8_t packet_type,mbuf_t * packet)394 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
395 {
396 struct flow_divert_packet_header hdr;
397 int error = 0;
398
399 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
400 if (error) {
401 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
402 return error;
403 }
404
405 hdr.packet_type = packet_type;
406 hdr.conn_id = htonl(fd_cb->hash);
407
408 /* Lay down the header */
409 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
410 if (error) {
411 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
412 mbuf_freem(*packet);
413 *packet = NULL;
414 return error;
415 }
416
417 return 0;
418 }
419
420 static int
flow_divert_packet_append_tlv(mbuf_t packet,uint8_t type,uint32_t length,const void * value)421 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
422 {
423 uint32_t net_length = htonl(length);
424 int error = 0;
425
426 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
427 if (error) {
428 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
429 return error;
430 }
431
432 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
433 if (error) {
434 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
435 return error;
436 }
437
438 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
439 if (error) {
440 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
441 return error;
442 }
443
444 return error;
445 }
446
447 static int
flow_divert_packet_find_tlv(mbuf_t packet,int offset,uint8_t type,int * err,int next)448 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
449 {
450 size_t cursor = offset;
451 int error = 0;
452 uint32_t curr_length;
453 uint8_t curr_type;
454
455 *err = 0;
456
457 do {
458 if (!next) {
459 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
460 if (error) {
461 *err = ENOENT;
462 return -1;
463 }
464 } else {
465 next = 0;
466 curr_type = FLOW_DIVERT_TLV_NIL;
467 }
468
469 if (curr_type != type) {
470 cursor += sizeof(curr_type);
471 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
472 if (error) {
473 *err = error;
474 return -1;
475 }
476
477 cursor += (sizeof(curr_length) + ntohl(curr_length));
478 }
479 } while (curr_type != type);
480
481 return (int)cursor;
482 }
483
484 static int
flow_divert_packet_get_tlv(mbuf_t packet,int offset,uint8_t type,size_t buff_len,void * buff,uint32_t * val_size)485 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
486 {
487 int error = 0;
488 uint32_t length;
489 int tlv_offset;
490
491 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
492 if (tlv_offset < 0) {
493 return error;
494 }
495
496 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
497 if (error) {
498 return error;
499 }
500
501 length = ntohl(length);
502
503 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
504
505 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
506 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
507 return EINVAL;
508 }
509
510 if (val_size != NULL) {
511 *val_size = length;
512 }
513
514 if (buff != NULL && buff_len > 0) {
515 memset(buff, 0, buff_len);
516 size_t to_copy = (length < buff_len) ? length : buff_len;
517 error = mbuf_copydata(packet, data_offset, to_copy, buff);
518 if (error) {
519 return error;
520 }
521 }
522
523 return 0;
524 }
525
526 static int
flow_divert_packet_compute_hmac(mbuf_t packet,struct flow_divert_group * group,uint8_t * hmac)527 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
528 {
529 mbuf_t curr_mbuf = packet;
530
531 if (g_crypto_funcs == NULL || group->token_key == NULL) {
532 return ENOPROTOOPT;
533 }
534
535 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
536 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
537
538 while (curr_mbuf != NULL) {
539 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
540 curr_mbuf = mbuf_next(curr_mbuf);
541 }
542
543 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
544
545 return 0;
546 }
547
548 static int
flow_divert_packet_verify_hmac(mbuf_t packet,uint32_t ctl_unit)549 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
550 {
551 int error = 0;
552 struct flow_divert_group *group = NULL;
553 int hmac_offset;
554 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
555 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
556 mbuf_t tail;
557
558 group = flow_divert_group_lookup(ctl_unit, NULL);
559 if (group == NULL) {
560 FDLOG(LOG_ERR, &nil_pcb, "Failed to lookup group for control unit %u", ctl_unit);
561 return ENOPROTOOPT;
562 }
563
564 lck_rw_lock_shared(&group->lck);
565
566 if (group->token_key == NULL) {
567 error = ENOPROTOOPT;
568 goto done;
569 }
570
571 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
572 if (hmac_offset < 0) {
573 goto done;
574 }
575
576 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
577 if (error) {
578 goto done;
579 }
580
581 /* Chop off the HMAC TLV */
582 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
583 if (error) {
584 goto done;
585 }
586
587 mbuf_free(tail);
588
589 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
590 if (error) {
591 goto done;
592 }
593
594 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
595 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
596 error = EINVAL;
597 goto done;
598 }
599
600 done:
601 if (group != NULL) {
602 lck_rw_done(&group->lck);
603 FDGRP_RELEASE(group);
604 }
605 return error;
606 }
607
608 static void
flow_divert_add_data_statistics(struct flow_divert_pcb * fd_cb,size_t data_len,Boolean send)609 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send)
610 {
611 struct inpcb *inp = NULL;
612 struct ifnet *ifp = NULL;
613 Boolean cell = FALSE;
614 Boolean wifi = FALSE;
615 Boolean wired = FALSE;
616
617 inp = sotoinpcb(fd_cb->so);
618 if (inp == NULL) {
619 return;
620 }
621
622 if (inp->inp_vflag & INP_IPV4) {
623 ifp = inp->inp_last_outifp;
624 } else if (inp->inp_vflag & INP_IPV6) {
625 ifp = inp->in6p_last_outifp;
626 }
627 if (ifp != NULL) {
628 cell = IFNET_IS_CELLULAR(ifp);
629 wifi = (!cell && IFNET_IS_WIFI(ifp));
630 wired = (!wifi && IFNET_IS_WIRED(ifp));
631 }
632
633 if (send) {
634 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
635 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
636 } else {
637 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
638 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
639 }
640 inp_set_activity_bitmap(inp);
641 }
642
643 static errno_t
flow_divert_check_no_cellular(struct flow_divert_pcb * fd_cb)644 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
645 {
646 struct inpcb *inp = sotoinpcb(fd_cb->so);
647 if (INP_NO_CELLULAR(inp)) {
648 struct ifnet *ifp = NULL;
649 if (inp->inp_vflag & INP_IPV4) {
650 ifp = inp->inp_last_outifp;
651 } else if (inp->inp_vflag & INP_IPV6) {
652 ifp = inp->in6p_last_outifp;
653 }
654 if (ifp != NULL && IFNET_IS_CELLULAR(ifp)) {
655 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied");
656 return EHOSTUNREACH;
657 }
658 }
659 return 0;
660 }
661
662 static errno_t
flow_divert_check_no_expensive(struct flow_divert_pcb * fd_cb)663 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
664 {
665 struct inpcb *inp = sotoinpcb(fd_cb->so);
666 if (INP_NO_EXPENSIVE(inp)) {
667 struct ifnet *ifp = NULL;
668 if (inp->inp_vflag & INP_IPV4) {
669 ifp = inp->inp_last_outifp;
670 } else if (inp->inp_vflag & INP_IPV6) {
671 ifp = inp->in6p_last_outifp;
672 }
673 if (ifp != NULL && IFNET_IS_EXPENSIVE(ifp)) {
674 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied");
675 return EHOSTUNREACH;
676 }
677 }
678 return 0;
679 }
680
681 static errno_t
flow_divert_check_no_constrained(struct flow_divert_pcb * fd_cb)682 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
683 {
684 struct inpcb *inp = sotoinpcb(fd_cb->so);
685 if (INP_NO_CONSTRAINED(inp)) {
686 struct ifnet *ifp = NULL;
687 if (inp->inp_vflag & INP_IPV4) {
688 ifp = inp->inp_last_outifp;
689 } else if (inp->inp_vflag & INP_IPV6) {
690 ifp = inp->in6p_last_outifp;
691 }
692 if (ifp != NULL && IFNET_IS_CONSTRAINED(ifp)) {
693 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied");
694 return EHOSTUNREACH;
695 }
696 }
697 return 0;
698 }
699
700 static void
flow_divert_update_closed_state(struct flow_divert_pcb * fd_cb,int how,Boolean tunnel)701 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
702 {
703 if (how != SHUT_RD) {
704 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
705 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
706 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
707 /* If the tunnel is not accepting writes any more, then flush the send buffer */
708 sbflush(&fd_cb->so->so_snd);
709 }
710 }
711 if (how != SHUT_WR) {
712 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
713 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
714 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
715 }
716 }
717 }
718
719 static uint16_t
trie_node_alloc(struct flow_divert_trie * trie)720 trie_node_alloc(struct flow_divert_trie *trie)
721 {
722 if (trie->nodes_free_next < trie->nodes_count) {
723 uint16_t node_idx = trie->nodes_free_next++;
724 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
725 return node_idx;
726 } else {
727 return NULL_TRIE_IDX;
728 }
729 }
730
731 static uint16_t
trie_child_map_alloc(struct flow_divert_trie * trie)732 trie_child_map_alloc(struct flow_divert_trie *trie)
733 {
734 if (trie->child_maps_free_next < trie->child_maps_count) {
735 return trie->child_maps_free_next++;
736 } else {
737 return NULL_TRIE_IDX;
738 }
739 }
740
741 static uint16_t
trie_bytes_move(struct flow_divert_trie * trie,uint16_t bytes_idx,size_t bytes_size)742 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
743 {
744 uint16_t start = trie->bytes_free_next;
745 if (start + bytes_size <= trie->bytes_count) {
746 if (start != bytes_idx) {
747 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
748 }
749 trie->bytes_free_next += bytes_size;
750 return start;
751 } else {
752 return NULL_TRIE_IDX;
753 }
754 }
755
756 static uint16_t
flow_divert_trie_insert(struct flow_divert_trie * trie,uint16_t string_start,size_t string_len)757 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
758 {
759 uint16_t current = trie->root;
760 uint16_t child = trie->root;
761 uint16_t string_end = string_start + (uint16_t)string_len;
762 uint16_t string_idx = string_start;
763 uint16_t string_remainder = (uint16_t)string_len;
764
765 while (child != NULL_TRIE_IDX) {
766 uint16_t parent = current;
767 uint16_t node_idx;
768 uint16_t current_end;
769
770 current = child;
771 child = NULL_TRIE_IDX;
772
773 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
774
775 for (node_idx = TRIE_NODE(trie, current).start;
776 node_idx < current_end &&
777 string_idx < string_end &&
778 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
779 node_idx++, string_idx++) {
780 ;
781 }
782
783 string_remainder = string_end - string_idx;
784
785 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
786 /*
787 * We did not reach the end of the current node's string.
788 * We need to split the current node into two:
789 * 1. A new node that contains the prefix of the node that matches
790 * the prefix of the string being inserted.
791 * 2. The current node modified to point to the remainder
792 * of the current node's string.
793 */
794 uint16_t prefix = trie_node_alloc(trie);
795 if (prefix == NULL_TRIE_IDX) {
796 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
797 return NULL_TRIE_IDX;
798 }
799
800 /*
801 * Prefix points to the portion of the current nodes's string that has matched
802 * the input string thus far.
803 */
804 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
805 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
806
807 /*
808 * Prefix has the current node as the child corresponding to the first byte
809 * after the split.
810 */
811 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
812 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
813 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
814 return NULL_TRIE_IDX;
815 }
816 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
817
818 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
819 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
820
821 /* Current node is adjusted to point to the remainder */
822 TRIE_NODE(trie, current).start = node_idx;
823 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
824
825 /* We want to insert the new leaf (if any) as a child of the prefix */
826 current = prefix;
827 }
828
829 if (string_remainder > 0) {
830 /*
831 * We still have bytes in the string that have not been matched yet.
832 * If the current node has children, iterate to the child corresponding
833 * to the next byte in the string.
834 */
835 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
836 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
837 }
838 }
839 } /* while (child != NULL_TRIE_IDX) */
840
841 if (string_remainder > 0) {
842 /* Add a new leaf containing the remainder of the string */
843 uint16_t leaf = trie_node_alloc(trie);
844 if (leaf == NULL_TRIE_IDX) {
845 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
846 return NULL_TRIE_IDX;
847 }
848
849 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
850 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
851 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
852 return NULL_TRIE_IDX;
853 }
854 TRIE_NODE(trie, leaf).length = string_remainder;
855
856 /* Set the new leaf as the child of the current node */
857 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
858 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
859 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
860 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
861 return NULL_TRIE_IDX;
862 }
863 }
864 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
865 current = leaf;
866 } /* else duplicate or this string is a prefix of one of the existing strings */
867
868 return current;
869 }
870
871 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
872 static uint16_t
flow_divert_trie_search(struct flow_divert_trie * trie,const uint8_t * string_bytes)873 flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
874 {
875 uint16_t current = trie->root;
876 uint16_t string_idx = 0;
877
878 while (current != NULL_TRIE_IDX) {
879 uint16_t next = NULL_TRIE_IDX;
880 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
881 uint16_t node_idx;
882
883 for (node_idx = TRIE_NODE(trie, current).start;
884 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
885 node_idx++, string_idx++) {
886 ;
887 }
888
889 if (node_idx == node_end) {
890 if (string_bytes[string_idx] == '\0') {
891 return current; /* Got an exact match */
892 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
893 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
894 return current; /* Got an apple webclip id prefix match */
895 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
896 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
897 }
898 }
899 current = next;
900 }
901
902 return NULL_TRIE_IDX;
903 }
904
905 struct uuid_search_info {
906 uuid_t target_uuid;
907 char *found_signing_id;
908 boolean_t found_multiple_signing_ids;
909 proc_t found_proc;
910 };
911
912 static int
flow_divert_find_proc_by_uuid_callout(proc_t p,void * arg)913 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
914 {
915 struct uuid_search_info *info = (struct uuid_search_info *)arg;
916 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
917
918 if (info->found_signing_id != NULL) {
919 if (!info->found_multiple_signing_ids) {
920 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
921 info->found_proc = p;
922 result = PROC_CLAIMED_DONE;
923 } else {
924 uuid_string_t uuid_str;
925 uuid_unparse(info->target_uuid, uuid_str);
926 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
927 }
928 kfree_data(info->found_signing_id, strlen(info->found_signing_id) + 1);
929 info->found_signing_id = NULL;
930 }
931
932 if (result == PROC_RETURNED_DONE) {
933 uuid_string_t uuid_str;
934 uuid_unparse(info->target_uuid, uuid_str);
935 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
936 }
937
938 return result;
939 }
940
941 static int
flow_divert_find_proc_by_uuid_filter(proc_t p,void * arg)942 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
943 {
944 struct uuid_search_info *info = (struct uuid_search_info *)arg;
945 int include = 0;
946
947 if (info->found_multiple_signing_ids) {
948 return include;
949 }
950
951 include = (uuid_compare(proc_executableuuid_addr(p), info->target_uuid) == 0);
952 if (include) {
953 const char *signing_id = cs_identity_get(p);
954 if (signing_id != NULL) {
955 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", proc_getpid(p), signing_id);
956 size_t signing_id_size = strlen(signing_id) + 1;
957 if (info->found_signing_id == NULL) {
958 info->found_signing_id = kalloc_data(signing_id_size, Z_WAITOK);
959 memcpy(info->found_signing_id, signing_id, signing_id_size);
960 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
961 info->found_multiple_signing_ids = TRUE;
962 }
963 } else {
964 info->found_multiple_signing_ids = TRUE;
965 }
966 include = !info->found_multiple_signing_ids;
967 }
968
969 return include;
970 }
971
972 static proc_t
flow_divert_find_proc_by_uuid(uuid_t uuid)973 flow_divert_find_proc_by_uuid(uuid_t uuid)
974 {
975 struct uuid_search_info info;
976
977 if (LOG_INFO <= nil_pcb.log_level) {
978 uuid_string_t uuid_str;
979 uuid_unparse(uuid, uuid_str);
980 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
981 }
982
983 memset(&info, 0, sizeof(info));
984 info.found_proc = PROC_NULL;
985 uuid_copy(info.target_uuid, uuid);
986
987 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
988
989 return info.found_proc;
990 }
991
992 static int
flow_divert_add_proc_info(struct flow_divert_pcb * fd_cb,proc_t proc,const char * signing_id,mbuf_t connect_packet,bool is_effective)993 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
994 {
995 int error = 0;
996 uint8_t *cdhash = NULL;
997 audit_token_t audit_token = {};
998 const char *proc_cs_id = signing_id;
999
1000 proc_lock(proc);
1001
1002 if (proc_cs_id == NULL) {
1003 if (proc_getcsflags(proc) & (CS_VALID | CS_DEBUGGED)) {
1004 proc_cs_id = cs_identity_get(proc);
1005 } else {
1006 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
1007 }
1008 }
1009
1010 if (is_effective) {
1011 lck_rw_lock_shared(&fd_cb->group->lck);
1012 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1013 if (proc_cs_id != NULL) {
1014 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
1015 if (result == NULL_TRIE_IDX) {
1016 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
1017 error = EPERM;
1018 } else {
1019 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
1020 }
1021 } else {
1022 error = EPERM;
1023 }
1024 }
1025 lck_rw_done(&fd_cb->group->lck);
1026 }
1027
1028 if (error != 0) {
1029 goto done;
1030 }
1031
1032 /*
1033 * If signing_id is not NULL then it came from the flow divert token and will be added
1034 * as part of the token, so there is no need to add it here.
1035 */
1036 if (signing_id == NULL && proc_cs_id != NULL) {
1037 error = flow_divert_packet_append_tlv(connect_packet,
1038 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
1039 (uint32_t)strlen(proc_cs_id),
1040 proc_cs_id);
1041 if (error != 0) {
1042 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1043 goto done;
1044 }
1045 }
1046
1047 cdhash = cs_get_cdhash(proc);
1048 if (cdhash != NULL) {
1049 error = flow_divert_packet_append_tlv(connect_packet,
1050 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
1051 SHA1_RESULTLEN,
1052 cdhash);
1053 if (error) {
1054 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1055 goto done;
1056 }
1057 } else {
1058 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash");
1059 }
1060
1061 task_t task = proc_task(proc);
1062 if (task != TASK_NULL) {
1063 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1064 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1065 if (rc == KERN_SUCCESS) {
1066 int append_error = flow_divert_packet_append_tlv(connect_packet,
1067 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1068 sizeof(audit_token_t),
1069 &audit_token);
1070 if (append_error) {
1071 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1072 }
1073 }
1074 }
1075
1076 done:
1077 proc_unlock(proc);
1078
1079 return error;
1080 }
1081
1082 static int
flow_divert_add_all_proc_info(struct flow_divert_pcb * fd_cb,struct socket * so,proc_t proc,const char * signing_id,mbuf_t connect_packet)1083 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1084 {
1085 int error = 0;
1086 proc_t effective_proc = PROC_NULL;
1087 proc_t responsible_proc = PROC_NULL;
1088 proc_t real_proc = proc_find(so->last_pid);
1089 bool release_real_proc = true;
1090
1091 proc_t src_proc = PROC_NULL;
1092 proc_t real_src_proc = PROC_NULL;
1093
1094 if (real_proc == PROC_NULL) {
1095 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1096 release_real_proc = false;
1097 real_proc = proc;
1098 if (real_proc == PROC_NULL) {
1099 real_proc = current_proc();
1100 }
1101 }
1102
1103 if (so->so_flags & SOF_DELEGATED) {
1104 if (proc_getpid(real_proc) != so->e_pid) {
1105 effective_proc = proc_find(so->e_pid);
1106 } else if (uuid_compare(proc_executableuuid_addr(real_proc), so->e_uuid)) {
1107 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
1108 }
1109 }
1110
1111 #if defined(XNU_TARGET_OS_OSX)
1112 lck_rw_lock_shared(&fd_cb->group->lck);
1113 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1114 if (so->so_rpid > 0) {
1115 responsible_proc = proc_find(so->so_rpid);
1116 }
1117 }
1118 lck_rw_done(&fd_cb->group->lck);
1119 #endif
1120
1121 real_src_proc = real_proc;
1122
1123 if (responsible_proc != PROC_NULL) {
1124 src_proc = responsible_proc;
1125 if (effective_proc != NULL) {
1126 real_src_proc = effective_proc;
1127 }
1128 } else if (effective_proc != PROC_NULL) {
1129 src_proc = effective_proc;
1130 } else {
1131 src_proc = real_proc;
1132 }
1133
1134 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1135 if (error != 0) {
1136 goto done;
1137 }
1138
1139 if (real_src_proc != NULL && real_src_proc != src_proc) {
1140 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1141 if (error != 0) {
1142 goto done;
1143 }
1144 }
1145
1146 done:
1147 if (responsible_proc != PROC_NULL) {
1148 proc_rele(responsible_proc);
1149 }
1150
1151 if (effective_proc != PROC_NULL) {
1152 proc_rele(effective_proc);
1153 }
1154
1155 if (real_proc != PROC_NULL && release_real_proc) {
1156 proc_rele(real_proc);
1157 }
1158
1159 return error;
1160 }
1161
1162 static int
flow_divert_send_packet(struct flow_divert_pcb * fd_cb,mbuf_t packet,Boolean enqueue)1163 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1164 {
1165 int error;
1166
1167 if (fd_cb->group == NULL) {
1168 FDLOG0(LOG_INFO, fd_cb, "no provider, cannot send packet");
1169 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1170 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT));
1171 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1172 error = ECONNABORTED;
1173 } else {
1174 error = EHOSTUNREACH;
1175 }
1176 fd_cb->so->so_error = (uint16_t)error;
1177 return error;
1178 }
1179
1180 lck_rw_lock_shared(&fd_cb->group->lck);
1181
1182 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1183 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1184 } else {
1185 error = ENOBUFS;
1186 }
1187
1188 if (error == ENOBUFS) {
1189 if (enqueue) {
1190 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1191 lck_rw_lock_exclusive(&fd_cb->group->lck);
1192 }
1193 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1194 error = 0;
1195 }
1196 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1197 }
1198
1199 lck_rw_done(&fd_cb->group->lck);
1200
1201 return error;
1202 }
1203
1204 static void
flow_divert_append_domain_name(char * domain_name,void * ctx)1205 flow_divert_append_domain_name(char *domain_name, void *ctx)
1206 {
1207 mbuf_t packet = (mbuf_t)ctx;
1208 size_t domain_name_length = 0;
1209
1210 if (packet == NULL || domain_name == NULL) {
1211 return;
1212 }
1213
1214 domain_name_length = strlen(domain_name);
1215 if (domain_name_length > 0 && domain_name_length < FLOW_DIVERT_MAX_NAME_SIZE) {
1216 int error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TARGET_HOSTNAME, (uint32_t)domain_name_length, domain_name);
1217 if (error) {
1218 FDLOG(LOG_ERR, &nil_pcb, "Failed to append %s: %d", domain_name, error);
1219 }
1220 }
1221 }
1222
1223 static int
flow_divert_create_connect_packet(struct flow_divert_pcb * fd_cb,struct sockaddr * to,struct socket * so,proc_t p,mbuf_t * out_connect_packet)1224 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
1225 {
1226 int error = 0;
1227 int flow_type = 0;
1228 char *signing_id = NULL;
1229 uint32_t sid_size = 0;
1230 mbuf_t connect_packet = NULL;
1231 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1232 const void *cfil_id = NULL;
1233 size_t cfil_id_size = 0;
1234 struct inpcb *inp = sotoinpcb(so);
1235 struct ifnet *ifp = NULL;
1236 uint32_t flags = 0;
1237
1238 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1239 if (error) {
1240 goto done;
1241 }
1242
1243 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1244 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1245 if (find_error == 0 && sid_size > 0) {
1246 signing_id = kalloc_data(sid_size + 1, Z_WAITOK | Z_ZERO);
1247 if (signing_id != NULL) {
1248 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1249 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1250 }
1251 }
1252 }
1253
1254 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
1255
1256 if (signing_id != NULL) {
1257 kfree_data(signing_id, sid_size + 1);
1258 }
1259
1260 if (error) {
1261 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
1262 goto done;
1263 }
1264
1265 error = flow_divert_packet_append_tlv(connect_packet,
1266 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1267 sizeof(fd_cb->so->so_traffic_class),
1268 &fd_cb->so->so_traffic_class);
1269 if (error) {
1270 goto done;
1271 }
1272
1273 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1274 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1275 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1276 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1277 } else {
1278 error = EINVAL;
1279 goto done;
1280 }
1281 error = flow_divert_packet_append_tlv(connect_packet,
1282 FLOW_DIVERT_TLV_FLOW_TYPE,
1283 sizeof(flow_type),
1284 &flow_type);
1285
1286 if (error) {
1287 goto done;
1288 }
1289
1290 if (fd_cb->connect_token != NULL) {
1291 unsigned int token_len = m_length(fd_cb->connect_token);
1292 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1293 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1294 fd_cb->connect_token = NULL;
1295 } else {
1296 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1297 if (error) {
1298 goto done;
1299 }
1300
1301 necp_with_inp_domain_name(so, connect_packet, flow_divert_append_domain_name);
1302 }
1303
1304 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1305 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1306 if (error) {
1307 goto done;
1308 }
1309 }
1310
1311 if (inp->inp_vflag & INP_IPV4) {
1312 ifp = inp->inp_last_outifp;
1313 } else if (inp->inp_vflag & INP_IPV6) {
1314 ifp = inp->in6p_last_outifp;
1315 }
1316 if (ifp != NULL) {
1317 uint32_t flow_if_index = ifp->if_index;
1318 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1319 sizeof(flow_if_index), &flow_if_index);
1320 if (error) {
1321 goto done;
1322 }
1323 }
1324
1325 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1326 flags |= FLOW_DIVERT_TOKEN_FLAG_TFO;
1327 }
1328
1329 if ((inp->inp_flags & INP_BOUND_IF) ||
1330 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) ||
1331 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
1332 flags |= FLOW_DIVERT_TOKEN_FLAG_BOUND;
1333 }
1334
1335 if (flags != 0) {
1336 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1337 if (error) {
1338 goto done;
1339 }
1340 }
1341
1342 if (SOCK_TYPE(so) == SOCK_DGRAM) {
1343 cfil_sock_id = cfil_sock_id_from_datagram_socket(so, NULL, to);
1344 } else {
1345 cfil_sock_id = cfil_sock_id_from_socket(so);
1346 }
1347
1348 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1349 cfil_id = &cfil_sock_id;
1350 cfil_id_size = sizeof(cfil_sock_id);
1351 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1352 cfil_id = &inp->necp_client_uuid;
1353 cfil_id_size = sizeof(inp->necp_client_uuid);
1354 }
1355
1356 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
1357 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, (uint32_t)cfil_id_size, cfil_id);
1358 if (error) {
1359 goto done;
1360 }
1361 }
1362
1363 done:
1364 if (!error) {
1365 *out_connect_packet = connect_packet;
1366 } else if (connect_packet != NULL) {
1367 mbuf_freem(connect_packet);
1368 }
1369
1370 return error;
1371 }
1372
1373 static int
flow_divert_send_connect_packet(struct flow_divert_pcb * fd_cb)1374 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb)
1375 {
1376 int error = 0;
1377 mbuf_t connect_packet = fd_cb->connect_packet;
1378 mbuf_t saved_connect_packet = NULL;
1379
1380 if (connect_packet != NULL) {
1381 error = mbuf_copym(connect_packet, 0, mbuf_pkthdr_len(connect_packet), MBUF_DONTWAIT, &saved_connect_packet);
1382 if (error) {
1383 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet");
1384 goto done;
1385 }
1386
1387 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
1388 if (error) {
1389 goto done;
1390 }
1391
1392 fd_cb->connect_packet = saved_connect_packet;
1393 saved_connect_packet = NULL;
1394 } else {
1395 error = ENOENT;
1396 }
1397 done:
1398 if (saved_connect_packet != NULL) {
1399 mbuf_freem(saved_connect_packet);
1400 }
1401
1402 return error;
1403 }
1404
1405 static int
flow_divert_send_connect_result(struct flow_divert_pcb * fd_cb)1406 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1407 {
1408 int error = 0;
1409 mbuf_t packet = NULL;
1410 int rbuff_space = 0;
1411
1412 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1413 if (error) {
1414 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1415 goto done;
1416 }
1417
1418 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1419 if (rbuff_space < 0) {
1420 rbuff_space = 0;
1421 }
1422 rbuff_space = htonl(rbuff_space);
1423 error = flow_divert_packet_append_tlv(packet,
1424 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1425 sizeof(rbuff_space),
1426 &rbuff_space);
1427 if (error) {
1428 goto done;
1429 }
1430
1431 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1432 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1433 if (error) {
1434 goto done;
1435 }
1436 }
1437
1438 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1439 if (error) {
1440 goto done;
1441 }
1442
1443 done:
1444 if (error && packet != NULL) {
1445 mbuf_freem(packet);
1446 }
1447
1448 return error;
1449 }
1450
1451 static int
flow_divert_send_close(struct flow_divert_pcb * fd_cb,int how)1452 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1453 {
1454 int error = 0;
1455 mbuf_t packet = NULL;
1456 uint32_t zero = 0;
1457
1458 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1459 if (error) {
1460 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1461 goto done;
1462 }
1463
1464 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1465 if (error) {
1466 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1467 goto done;
1468 }
1469
1470 how = htonl(how);
1471 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1472 if (error) {
1473 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1474 goto done;
1475 }
1476
1477 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1478 if (error) {
1479 goto done;
1480 }
1481
1482 done:
1483 if (error && packet != NULL) {
1484 mbuf_free(packet);
1485 }
1486
1487 return error;
1488 }
1489
1490 static int
flow_divert_tunnel_how_closed(struct flow_divert_pcb * fd_cb)1491 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1492 {
1493 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1494 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1495 return SHUT_RDWR;
1496 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1497 return SHUT_RD;
1498 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1499 return SHUT_WR;
1500 }
1501
1502 return -1;
1503 }
1504
1505 /*
1506 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1507 * writes. Returns FALSE otherwise.
1508 */
1509 static void
flow_divert_send_close_if_needed(struct flow_divert_pcb * fd_cb)1510 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1511 {
1512 int how = -1;
1513
1514 /* Do not send any close messages if there is still data in the send buffer */
1515 if (fd_cb->so->so_snd.sb_cc == 0) {
1516 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1517 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1518 how = SHUT_RD;
1519 }
1520 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1521 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1522 if (how == SHUT_RD) {
1523 how = SHUT_RDWR;
1524 } else {
1525 how = SHUT_WR;
1526 }
1527 }
1528 }
1529
1530 if (how != -1) {
1531 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1532 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1533 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1534 if (how != SHUT_RD) {
1535 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1536 }
1537 if (how != SHUT_WR) {
1538 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1539 }
1540 }
1541 }
1542
1543 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1544 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT));
1545 }
1546 }
1547
1548 static errno_t
flow_divert_send_data_packet(struct flow_divert_pcb * fd_cb,mbuf_t data,size_t data_len,Boolean force)1549 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, Boolean force)
1550 {
1551 mbuf_t packet = NULL;
1552 mbuf_t last = NULL;
1553 int error = 0;
1554
1555 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1556 if (error || packet == NULL) {
1557 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1558 goto done;
1559 }
1560
1561 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
1562 last = m_last(packet);
1563 mbuf_setnext(last, data);
1564 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1565 } else {
1566 data_len = 0;
1567 }
1568 error = flow_divert_send_packet(fd_cb, packet, force);
1569 if (error == 0 && data_len > 0) {
1570 fd_cb->bytes_sent += data_len;
1571 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1572 }
1573
1574 done:
1575 if (error) {
1576 if (last != NULL) {
1577 mbuf_setnext(last, NULL);
1578 }
1579 if (packet != NULL) {
1580 mbuf_freem(packet);
1581 }
1582 }
1583
1584 return error;
1585 }
1586
1587 static errno_t
flow_divert_send_datagram_packet(struct flow_divert_pcb * fd_cb,mbuf_t data,size_t data_len,struct sockaddr * toaddr,Boolean force,Boolean is_fragment,size_t datagram_size)1588 flow_divert_send_datagram_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force, Boolean is_fragment, size_t datagram_size)
1589 {
1590 mbuf_t packet = NULL;
1591 mbuf_t last = NULL;
1592 int error = 0;
1593
1594 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1595 if (error || packet == NULL) {
1596 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1597 goto done;
1598 }
1599
1600 if (toaddr != NULL) {
1601 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1602 if (error) {
1603 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1604 goto done;
1605 }
1606 }
1607 if (is_fragment) {
1608 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_IS_FRAGMENT, sizeof(is_fragment), &is_fragment);
1609 if (error) {
1610 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_IS_FRAGMENT) failed: %d", error);
1611 goto done;
1612 }
1613 }
1614
1615 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_DATAGRAM_SIZE, sizeof(datagram_size), &datagram_size);
1616 if (error) {
1617 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_DATAGRAM_SIZE) failed: %d", error);
1618 goto done;
1619 }
1620
1621 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
1622 last = m_last(packet);
1623 mbuf_setnext(last, data);
1624 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1625 } else {
1626 data_len = 0;
1627 }
1628 error = flow_divert_send_packet(fd_cb, packet, force);
1629 if (error == 0 && data_len > 0) {
1630 fd_cb->bytes_sent += data_len;
1631 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1632 }
1633
1634 done:
1635 if (error) {
1636 if (last != NULL) {
1637 mbuf_setnext(last, NULL);
1638 }
1639 if (packet != NULL) {
1640 mbuf_freem(packet);
1641 }
1642 }
1643
1644 return error;
1645 }
1646
1647 static errno_t
flow_divert_send_fragmented_datagram(struct flow_divert_pcb * fd_cb,mbuf_t datagram,size_t datagram_len,struct sockaddr * toaddr,Boolean force)1648 flow_divert_send_fragmented_datagram(struct flow_divert_pcb *fd_cb, mbuf_t datagram, size_t datagram_len, struct sockaddr *toaddr, Boolean force)
1649 {
1650 mbuf_t next_data = datagram;
1651 size_t remaining_len = datagram_len;
1652 mbuf_t remaining_data = NULL;
1653 int error = 0;
1654 bool first = true;
1655
1656 while (remaining_len > 0 && next_data != NULL) {
1657 size_t to_send = remaining_len;
1658 remaining_data = NULL;
1659
1660 if (to_send > FLOW_DIVERT_CHUNK_SIZE) {
1661 to_send = FLOW_DIVERT_CHUNK_SIZE;
1662 error = mbuf_split(next_data, to_send, MBUF_DONTWAIT, &remaining_data);
1663 if (error) {
1664 break;
1665 }
1666 }
1667
1668 error = flow_divert_send_datagram_packet(fd_cb, next_data, to_send, (first ? toaddr : NULL), force, TRUE, (first ? datagram_len : 0));
1669 if (error) {
1670 break;
1671 }
1672
1673 first = false;
1674 remaining_len -= to_send;
1675 next_data = remaining_data;
1676 }
1677
1678 if (error) {
1679 if (next_data != NULL) {
1680 mbuf_freem(next_data);
1681 }
1682 if (remaining_data != NULL) {
1683 mbuf_freem(remaining_data);
1684 }
1685 }
1686 return error;
1687 }
1688
1689 static void
flow_divert_send_buffered_data(struct flow_divert_pcb * fd_cb,Boolean force)1690 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1691 {
1692 size_t to_send;
1693 size_t sent = 0;
1694 int error = 0;
1695 mbuf_t buffer;
1696
1697 to_send = fd_cb->so->so_snd.sb_cc;
1698 buffer = fd_cb->so->so_snd.sb_mb;
1699
1700 if (buffer == NULL && to_send > 0) {
1701 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1702 return;
1703 }
1704
1705 /* Ignore the send window if force is enabled */
1706 if (!force && (to_send > fd_cb->send_window)) {
1707 to_send = fd_cb->send_window;
1708 }
1709
1710 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1711 while (sent < to_send) {
1712 mbuf_t data;
1713 size_t data_len;
1714
1715 data_len = to_send - sent;
1716 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1717 data_len = FLOW_DIVERT_CHUNK_SIZE;
1718 }
1719
1720 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1721 if (error) {
1722 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1723 break;
1724 }
1725
1726 error = flow_divert_send_data_packet(fd_cb, data, data_len, force);
1727 if (error) {
1728 if (data != NULL) {
1729 mbuf_freem(data);
1730 }
1731 break;
1732 }
1733
1734 sent += data_len;
1735 }
1736 sbdrop(&fd_cb->so->so_snd, (int)sent);
1737 sowwakeup(fd_cb->so);
1738 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1739 mbuf_t data;
1740 mbuf_t m;
1741 size_t data_len;
1742
1743 while (buffer) {
1744 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1745
1746 m = buffer;
1747 if (toaddr != NULL) {
1748 /* look for data in the chain */
1749 do {
1750 m = m->m_next;
1751 if (m != NULL && m->m_type == MT_DATA) {
1752 break;
1753 }
1754 } while (m);
1755 if (m == NULL) {
1756 /* unexpected */
1757 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1758 goto move_on;
1759 }
1760 }
1761 data_len = mbuf_pkthdr_len(m);
1762 if (data_len > 0) {
1763 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1764 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1765 if (error) {
1766 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1767 break;
1768 }
1769 } else {
1770 data = NULL;
1771 }
1772 if (data_len <= FLOW_DIVERT_CHUNK_SIZE) {
1773 error = flow_divert_send_datagram_packet(fd_cb, data, data_len, toaddr, force, FALSE, 0);
1774 } else {
1775 error = flow_divert_send_fragmented_datagram(fd_cb, data, data_len, toaddr, force);
1776 data = NULL;
1777 }
1778 if (error) {
1779 if (data != NULL) {
1780 mbuf_freem(data);
1781 }
1782 break;
1783 }
1784 sent += data_len;
1785 move_on:
1786 buffer = buffer->m_nextpkt;
1787 (void) sbdroprecord(&(fd_cb->so->so_snd));
1788 }
1789 }
1790
1791 if (sent > 0) {
1792 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1793 if (fd_cb->send_window >= sent) {
1794 fd_cb->send_window -= sent;
1795 } else {
1796 fd_cb->send_window = 0;
1797 }
1798 }
1799 }
1800
1801 static int
flow_divert_send_app_data(struct flow_divert_pcb * fd_cb,mbuf_t data,struct sockaddr * toaddr)1802 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1803 {
1804 size_t to_send = mbuf_pkthdr_len(data);
1805 int error = 0;
1806
1807 if (to_send > fd_cb->send_window) {
1808 to_send = fd_cb->send_window;
1809 }
1810
1811 if (fd_cb->so->so_snd.sb_cc > 0) {
1812 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1813 }
1814
1815 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1816 size_t sent = 0;
1817 mbuf_t remaining_data = data;
1818 mbuf_t pkt_data = NULL;
1819 while (sent < to_send && remaining_data != NULL) {
1820 size_t pkt_data_len;
1821
1822 pkt_data = remaining_data;
1823
1824 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1825 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1826 } else {
1827 pkt_data_len = to_send - sent;
1828 }
1829
1830 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1831 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1832 if (error) {
1833 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1834 pkt_data = NULL;
1835 break;
1836 }
1837 } else {
1838 remaining_data = NULL;
1839 }
1840
1841 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, FALSE);
1842
1843 if (error) {
1844 break;
1845 }
1846
1847 pkt_data = NULL;
1848 sent += pkt_data_len;
1849 }
1850
1851 fd_cb->send_window -= sent;
1852
1853 error = 0;
1854
1855 if (pkt_data != NULL) {
1856 if (sbspace(&fd_cb->so->so_snd) > 0) {
1857 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1858 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1859 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1860 }
1861 } else {
1862 mbuf_freem(pkt_data);
1863 error = ENOBUFS;
1864 }
1865 }
1866
1867 if (remaining_data != NULL) {
1868 if (sbspace(&fd_cb->so->so_snd) > 0) {
1869 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1870 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1871 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1872 }
1873 } else {
1874 mbuf_freem(remaining_data);
1875 error = ENOBUFS;
1876 }
1877 }
1878 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1879 if (to_send || mbuf_pkthdr_len(data) == 0) {
1880 if (to_send <= FLOW_DIVERT_CHUNK_SIZE) {
1881 error = flow_divert_send_datagram_packet(fd_cb, data, to_send, toaddr, FALSE, FALSE, 0);
1882 } else {
1883 error = flow_divert_send_fragmented_datagram(fd_cb, data, to_send, toaddr, FALSE);
1884 data = NULL;
1885 }
1886 if (error) {
1887 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_datagram_packet failed. send data size = %lu", to_send);
1888 if (data != NULL) {
1889 mbuf_freem(data);
1890 }
1891 } else {
1892 fd_cb->send_window -= to_send;
1893 }
1894 } else {
1895 /* buffer it */
1896 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1897 if (toaddr != NULL) {
1898 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1899 FDLOG(LOG_ERR, fd_cb,
1900 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1901 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1902 }
1903 error = 0;
1904 } else {
1905 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1906 FDLOG(LOG_ERR, fd_cb,
1907 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1908 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1909 }
1910 }
1911 } else {
1912 if (data != NULL) {
1913 mbuf_freem(data);
1914 }
1915 error = ENOBUFS;
1916 }
1917 }
1918 }
1919
1920 return error;
1921 }
1922
1923 static int
flow_divert_send_read_notification(struct flow_divert_pcb * fd_cb)1924 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb)
1925 {
1926 int error = 0;
1927 mbuf_t packet = NULL;
1928
1929 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1930 if (error) {
1931 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1932 goto done;
1933 }
1934
1935 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1936 if (error) {
1937 goto done;
1938 }
1939
1940 done:
1941 if (error && packet != NULL) {
1942 mbuf_free(packet);
1943 }
1944
1945 return error;
1946 }
1947
1948 static int
flow_divert_send_traffic_class_update(struct flow_divert_pcb * fd_cb,int traffic_class)1949 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1950 {
1951 int error = 0;
1952 mbuf_t packet = NULL;
1953
1954 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1955 if (error) {
1956 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1957 goto done;
1958 }
1959
1960 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1961 if (error) {
1962 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1963 goto done;
1964 }
1965
1966 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1967 if (error) {
1968 goto done;
1969 }
1970
1971 done:
1972 if (error && packet != NULL) {
1973 mbuf_free(packet);
1974 }
1975
1976 return error;
1977 }
1978
1979 static void
flow_divert_set_local_endpoint(struct flow_divert_pcb * fd_cb,struct sockaddr * local_endpoint)1980 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint)
1981 {
1982 struct inpcb *inp = sotoinpcb(fd_cb->so);
1983
1984 if (local_endpoint->sa_family == AF_INET6) {
1985 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
1986 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1987 inp->in6p_laddr = (satosin6(local_endpoint))->sin6_addr;
1988 inp->inp_lifscope = (satosin6(local_endpoint))->sin6_scope_id;
1989 in6_verify_ifscope(&inp->in6p_laddr, inp->inp_lifscope);
1990 }
1991 if (inp->inp_lport == 0) {
1992 inp->inp_lport = (satosin6(local_endpoint))->sin6_port;
1993 }
1994 } else if (local_endpoint->sa_family == AF_INET) {
1995 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
1996 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1997 inp->inp_laddr = (satosin(local_endpoint))->sin_addr;
1998 }
1999 if (inp->inp_lport == 0) {
2000 inp->inp_lport = (satosin(local_endpoint))->sin_port;
2001 }
2002 }
2003 }
2004
2005 static void
flow_divert_set_remote_endpoint(struct flow_divert_pcb * fd_cb,struct sockaddr * remote_endpoint)2006 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint)
2007 {
2008 struct inpcb *inp = sotoinpcb(fd_cb->so);
2009
2010 if (remote_endpoint->sa_family == AF_INET6) {
2011 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
2012 inp->in6p_faddr = (satosin6(remote_endpoint))->sin6_addr;
2013 inp->inp_fifscope = (satosin6(remote_endpoint))->sin6_scope_id;
2014 in6_verify_ifscope(&inp->in6p_faddr, inp->inp_fifscope);
2015 }
2016 if (inp->inp_fport == 0) {
2017 inp->inp_fport = (satosin6(remote_endpoint))->sin6_port;
2018 }
2019 } else if (remote_endpoint->sa_family == AF_INET) {
2020 if (inp->inp_faddr.s_addr == INADDR_ANY) {
2021 inp->inp_faddr = (satosin(remote_endpoint))->sin_addr;
2022 }
2023 if (inp->inp_fport == 0) {
2024 inp->inp_fport = (satosin(remote_endpoint))->sin_port;
2025 }
2026 }
2027 }
2028
2029 static uint32_t
flow_divert_derive_kernel_control_unit(uint32_t * ctl_unit,uint32_t * aggregate_unit,bool * is_aggregate)2030 flow_divert_derive_kernel_control_unit(uint32_t *ctl_unit, uint32_t *aggregate_unit, bool *is_aggregate)
2031 {
2032 uint32_t result = *ctl_unit;
2033
2034 *is_aggregate = false;
2035 if (aggregate_unit != NULL && *aggregate_unit != 0) {
2036 uint32_t counter;
2037 for (counter = 0; counter < (GROUP_COUNT_MAX - 1); counter++) {
2038 if ((*aggregate_unit) & (1 << counter)) {
2039 break;
2040 }
2041 }
2042 if (counter < (GROUP_COUNT_MAX - 1)) {
2043 *aggregate_unit &= ~(1 << counter);
2044 *is_aggregate = true;
2045 return counter + 1;
2046 } else {
2047 *ctl_unit = 0;
2048 return result;
2049 }
2050 } else {
2051 *ctl_unit = 0;
2052 return result;
2053 }
2054 }
2055
2056 static int
flow_divert_try_next_group(struct flow_divert_pcb * fd_cb)2057 flow_divert_try_next_group(struct flow_divert_pcb *fd_cb)
2058 {
2059 int error = 0;
2060 uint32_t policy_control_unit = fd_cb->policy_control_unit;
2061
2062 flow_divert_pcb_remove(fd_cb);
2063
2064 do {
2065 struct flow_divert_group *next_group = NULL;
2066 bool is_aggregate = false;
2067 uint32_t next_ctl_unit = flow_divert_derive_kernel_control_unit(&policy_control_unit, &(fd_cb->aggregate_unit), &is_aggregate);
2068
2069 if (fd_cb->control_group_unit == next_ctl_unit) {
2070 FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling flow divert");
2071 error = EALREADY;
2072 break;
2073 }
2074
2075 if (next_ctl_unit == 0 || next_ctl_unit >= GROUP_COUNT_MAX) {
2076 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert");
2077 error = ENOENT;
2078 break;
2079 }
2080
2081 next_group = flow_divert_group_lookup(next_ctl_unit, fd_cb);
2082 if (next_group == NULL) {
2083 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit);
2084 continue;
2085 }
2086
2087 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", fd_cb->control_group_unit, next_ctl_unit);
2088
2089 error = flow_divert_pcb_insert(fd_cb, next_group);
2090 if (error == 0) {
2091 if (is_aggregate) {
2092 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT;
2093 } else {
2094 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT;
2095 }
2096 }
2097 FDGRP_RELEASE(next_group);
2098 } while (fd_cb->group == NULL);
2099
2100 if (fd_cb->group == NULL) {
2101 return error ? error : ENOENT;
2102 }
2103
2104 error = flow_divert_send_connect_packet(fd_cb);
2105 if (error) {
2106 FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", fd_cb->control_group_unit);
2107 flow_divert_pcb_remove(fd_cb);
2108 error = ENOENT;
2109 }
2110
2111 return error;
2112 }
2113
2114 static void
flow_divert_disable(struct flow_divert_pcb * fd_cb)2115 flow_divert_disable(struct flow_divert_pcb *fd_cb)
2116 {
2117 struct socket *so = NULL;
2118 mbuf_t buffer;
2119 int error = 0;
2120 proc_t last_proc = NULL;
2121 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint;
2122 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT);
2123 struct inpcb *inp = NULL;
2124
2125 so = fd_cb->so;
2126 if (so == NULL) {
2127 goto done;
2128 }
2129
2130 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert");
2131
2132 /* Restore the IP state */
2133 inp = sotoinpcb(so);
2134 inp->inp_vflag = fd_cb->original_vflag;
2135 inp->inp_faddr.s_addr = INADDR_ANY;
2136 inp->inp_fport = 0;
2137 memset(&(inp->in6p_faddr), 0, sizeof(inp->in6p_faddr));
2138 inp->inp_fifscope = IFSCOPE_NONE;
2139 inp->in6p_fport = 0;
2140 /* If flow divert set the local address, clear it out */
2141 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) {
2142 inp->inp_laddr.s_addr = INADDR_ANY;
2143 memset(&(inp->in6p_laddr), 0, sizeof(inp->in6p_laddr));
2144 inp->inp_lifscope = IFSCOPE_NONE;
2145 }
2146 inp->inp_last_outifp = fd_cb->original_last_outifp;
2147 inp->in6p_last_outifp = fd_cb->original_last_outifp6;
2148
2149 /* Dis-associate the socket */
2150 so->so_flags &= ~SOF_FLOW_DIVERT;
2151 so->so_flags1 |= SOF1_FLOW_DIVERT_SKIP;
2152 so->so_fd_pcb = NULL;
2153 fd_cb->so = NULL;
2154
2155 FDRELEASE(fd_cb); /* Release the socket's reference */
2156
2157 /* Revert back to the original protocol */
2158 so->so_proto = pffindproto(SOCK_DOM(so), SOCK_PROTO(so), SOCK_TYPE(so));
2159
2160 /* Reset the socket state to avoid confusing NECP */
2161 so->so_state &= ~(SS_ISCONNECTING | SS_ISCONNECTED);
2162
2163 last_proc = proc_find(so->last_pid);
2164
2165 if (do_connect) {
2166 /* Connect using the original protocol */
2167 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, remote_endpoint, (last_proc != NULL ? last_proc : current_proc()));
2168 if (error) {
2169 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error);
2170 goto done;
2171 }
2172 }
2173
2174 buffer = so->so_snd.sb_mb;
2175 if (buffer == NULL) {
2176 /* No buffered data, done */
2177 goto done;
2178 }
2179
2180 /* Send any buffered data using the original protocol */
2181 if (SOCK_TYPE(so) == SOCK_STREAM) {
2182 mbuf_t data_to_send = NULL;
2183 size_t data_len = so->so_snd.sb_cc;
2184
2185 error = mbuf_copym(buffer, 0, data_len, MBUF_DONTWAIT, &data_to_send);
2186 if (error) {
2187 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer");
2188 goto done;
2189 }
2190
2191 sbflush(&so->so_snd);
2192
2193 if (data_to_send->m_flags & M_PKTHDR) {
2194 mbuf_pkthdr_setlen(data_to_send, data_len);
2195 }
2196
2197 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2198 0,
2199 data_to_send,
2200 NULL,
2201 NULL,
2202 (last_proc != NULL ? last_proc : current_proc()));
2203
2204 if (error && error != EWOULDBLOCK) {
2205 FDLOG(LOG_ERR, fd_cb, "Failed to send queued TCP data using the socket's original protocol: %d", error);
2206 } else {
2207 error = 0;
2208 }
2209 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
2210 struct sockbuf *sb = &so->so_snd;
2211 MBUFQ_HEAD(send_queue_head) send_queue;
2212 MBUFQ_INIT(&send_queue);
2213
2214 /* Flush the send buffer, moving all records to a temporary queue */
2215 while (sb->sb_mb != NULL) {
2216 mbuf_t record = sb->sb_mb;
2217 mbuf_t m = record;
2218 sb->sb_mb = sb->sb_mb->m_nextpkt;
2219 while (m != NULL) {
2220 sbfree(sb, m);
2221 m = m->m_next;
2222 }
2223 record->m_nextpkt = NULL;
2224 MBUFQ_ENQUEUE(&send_queue, record);
2225 }
2226 SB_EMPTY_FIXUP(sb);
2227
2228 while (!MBUFQ_EMPTY(&send_queue)) {
2229 mbuf_t next_record = MBUFQ_FIRST(&send_queue);
2230 mbuf_t addr = NULL;
2231 mbuf_t control = NULL;
2232 mbuf_t last_control = NULL;
2233 mbuf_t data = NULL;
2234 mbuf_t m = next_record;
2235 struct sockaddr *to_endpoint = NULL;
2236
2237 MBUFQ_DEQUEUE(&send_queue, next_record);
2238
2239 while (m != NULL) {
2240 if (m->m_type == MT_SONAME) {
2241 addr = m;
2242 } else if (m->m_type == MT_CONTROL) {
2243 if (control == NULL) {
2244 control = m;
2245 }
2246 last_control = m;
2247 } else if (m->m_type == MT_DATA) {
2248 data = m;
2249 break;
2250 }
2251 m = m->m_next;
2252 }
2253
2254 if (addr != NULL && !do_connect) {
2255 to_endpoint = flow_divert_get_buffered_target_address(addr);
2256 if (to_endpoint == NULL) {
2257 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer");
2258 }
2259 }
2260
2261 if (data == NULL) {
2262 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data");
2263 mbuf_freem(next_record);
2264 continue;
2265 }
2266
2267 if (!(data->m_flags & M_PKTHDR)) {
2268 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header");
2269 mbuf_freem(next_record);
2270 continue;
2271 }
2272
2273 if (addr != NULL) {
2274 addr->m_next = NULL;
2275 }
2276
2277 if (last_control != NULL) {
2278 last_control->m_next = NULL;
2279 }
2280
2281 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2282 0,
2283 data,
2284 to_endpoint,
2285 control,
2286 (last_proc != NULL ? last_proc : current_proc()));
2287
2288 if (addr != NULL) {
2289 mbuf_freem(addr);
2290 }
2291
2292 if (error) {
2293 FDLOG(LOG_ERR, fd_cb, "Failed to send queued UDP data using the socket's original protocol: %d", error);
2294 }
2295 }
2296 }
2297 done:
2298 if (last_proc != NULL) {
2299 proc_rele(last_proc);
2300 }
2301
2302 if (error && so != NULL) {
2303 so->so_error = (uint16_t)error;
2304 flow_divert_disconnect_socket(so, do_connect);
2305 }
2306 }
2307
2308 static void
flow_divert_scope(struct flow_divert_pcb * fd_cb,int out_if_index,bool derive_new_address)2309 flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address)
2310 {
2311 struct socket *so = NULL;
2312 struct inpcb *inp = NULL;
2313 struct ifnet *current_ifp = NULL;
2314 struct ifnet *new_ifp = NULL;
2315 int error = 0;
2316
2317 so = fd_cb->so;
2318 if (so == NULL) {
2319 return;
2320 }
2321
2322 inp = sotoinpcb(so);
2323
2324 if (out_if_index <= 0) {
2325 return;
2326 }
2327
2328 if (inp->inp_vflag & INP_IPV6) {
2329 current_ifp = inp->in6p_last_outifp;
2330 } else {
2331 current_ifp = inp->inp_last_outifp;
2332 }
2333
2334 if (current_ifp != NULL) {
2335 if (current_ifp->if_index == out_if_index) {
2336 /* No change */
2337 return;
2338 }
2339
2340 /* Scope the socket to the given interface */
2341 error = inp_bindif(inp, out_if_index, &new_ifp);
2342 if (error != 0) {
2343 FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error);
2344 return;
2345 }
2346
2347 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) {
2348 /* Get the appropriate address for the given interface */
2349 if (inp->inp_vflag & INP_IPV6) {
2350 inp->in6p_laddr = sa6_any.sin6_addr;
2351 error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr), NULL);
2352 } else {
2353 inp->inp_laddr.s_addr = INADDR_ANY;
2354 error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, NULL, 0);
2355 }
2356
2357 if (error != 0) {
2358 FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr returned %d", out_if_index, error);
2359 }
2360 }
2361 } else {
2362 ifnet_head_lock_shared();
2363 if (out_if_index <= if_index) {
2364 new_ifp = ifindex2ifnet[out_if_index];
2365 }
2366 ifnet_head_done();
2367 }
2368
2369 /* Update the "last interface" of the socket */
2370 if (new_ifp != NULL) {
2371 if (inp->inp_vflag & INP_IPV6) {
2372 inp->in6p_last_outifp = new_ifp;
2373 } else {
2374 inp->inp_last_outifp = new_ifp;
2375 }
2376
2377 #if SKYWALK
2378 if (NETNS_TOKEN_VALID(&inp->inp_netns_token)) {
2379 netns_set_ifnet(&inp->inp_netns_token, new_ifp);
2380 }
2381 #endif /* SKYWALK */
2382 }
2383 }
2384
2385 static void
flow_divert_handle_connect_result(struct flow_divert_pcb * fd_cb,mbuf_t packet,int offset)2386 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2387 {
2388 uint32_t connect_error = 0;
2389 uint32_t ctl_unit = 0;
2390 int error = 0;
2391 union sockaddr_in_4_6 local_endpoint = {};
2392 union sockaddr_in_4_6 remote_endpoint = {};
2393 int out_if_index = 0;
2394 uint32_t send_window;
2395 uint32_t app_data_length = 0;
2396
2397 memset(&local_endpoint, 0, sizeof(local_endpoint));
2398 memset(&remote_endpoint, 0, sizeof(remote_endpoint));
2399
2400 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
2401 if (error) {
2402 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
2403 return;
2404 }
2405
2406 connect_error = ntohl(connect_error);
2407 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
2408
2409 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
2410 if (error) {
2411 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
2412 return;
2413 }
2414
2415 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
2416 if (error) {
2417 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
2418 }
2419
2420 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_endpoint), &(local_endpoint.sa), NULL);
2421 if (error) {
2422 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
2423 }
2424
2425 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_endpoint), &(remote_endpoint.sa), NULL);
2426 if (error) {
2427 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2428 }
2429
2430 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2431 if (error) {
2432 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
2433 }
2434
2435 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2436 if (error) {
2437 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
2438 }
2439
2440 error = 0;
2441
2442 FDLOCK(fd_cb);
2443 if (fd_cb->so != NULL) {
2444 struct inpcb *inp = NULL;
2445 struct socket *so = fd_cb->so;
2446 bool local_address_is_valid = false;
2447
2448 socket_lock(so, 0);
2449
2450 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
2451 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result");
2452 goto done;
2453 }
2454
2455 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_state & SS_ISCONNECTING)) {
2456 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result");
2457 goto done;
2458 }
2459
2460 inp = sotoinpcb(so);
2461
2462 if (connect_error || error) {
2463 goto set_socket_state;
2464 }
2465
2466 if (flow_divert_is_sockaddr_valid(SA(&local_endpoint))) {
2467 if (local_endpoint.sa.sa_family == AF_INET) {
2468 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2469 if ((inp->inp_vflag & INP_IPV4) && local_endpoint.sin.sin_addr.s_addr != INADDR_ANY) {
2470 local_address_is_valid = true;
2471 fd_cb->local_endpoint = local_endpoint;
2472 inp->inp_laddr.s_addr = INADDR_ANY;
2473 } else {
2474 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port;
2475 }
2476 } else if (local_endpoint.sa.sa_family == AF_INET6) {
2477 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2478 if ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&local_endpoint.sin6.sin6_addr)) {
2479 local_address_is_valid = true;
2480 fd_cb->local_endpoint = local_endpoint;
2481 inp->in6p_laddr = sa6_any.sin6_addr;
2482 } else {
2483 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port;
2484 }
2485 }
2486 }
2487
2488 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid);
2489 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint));
2490
2491 if (flow_divert_is_sockaddr_valid(SA(&remote_endpoint)) && SOCK_TYPE(so) == SOCK_STREAM) {
2492 if (remote_endpoint.sa.sa_family == AF_INET) {
2493 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2494 } else if (remote_endpoint.sa.sa_family == AF_INET6) {
2495 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2496 }
2497 flow_divert_set_remote_endpoint(fd_cb, SA(&remote_endpoint));
2498 }
2499
2500 if (app_data_length > 0) {
2501 uint8_t *app_data = NULL;
2502 app_data = kalloc_data(app_data_length, Z_WAITOK);
2503 if (app_data != NULL) {
2504 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2505 if (error == 0) {
2506 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
2507 if (fd_cb->app_data != NULL) {
2508 kfree_data(fd_cb->app_data, fd_cb->app_data_length);
2509 }
2510 fd_cb->app_data = app_data;
2511 fd_cb->app_data_length = app_data_length;
2512 } else {
2513 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
2514 kfree_data(app_data, app_data_length);
2515 }
2516 } else {
2517 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
2518 }
2519 }
2520
2521 if (error) {
2522 goto set_socket_state;
2523 }
2524
2525 if (fd_cb->group == NULL) {
2526 error = EINVAL;
2527 goto set_socket_state;
2528 }
2529
2530 ctl_unit = ntohl(ctl_unit);
2531 if (ctl_unit > 0) {
2532 int insert_error = 0;
2533 struct flow_divert_group *grp = NULL;
2534
2535 if (ctl_unit >= GROUP_COUNT_MAX) {
2536 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
2537 error = EINVAL;
2538 goto set_socket_state;
2539 }
2540
2541 grp = flow_divert_group_lookup(ctl_unit, fd_cb);
2542 if (grp == NULL) {
2543 error = ECONNRESET;
2544 goto set_socket_state;
2545 }
2546
2547 flow_divert_pcb_remove(fd_cb);
2548 insert_error = flow_divert_pcb_insert(fd_cb, grp);
2549 FDGRP_RELEASE(grp);
2550
2551 if (insert_error != 0) {
2552 error = ECONNRESET;
2553 goto set_socket_state;
2554 }
2555 }
2556
2557 fd_cb->send_window = ntohl(send_window);
2558
2559 set_socket_state:
2560 if (!connect_error && !error) {
2561 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
2562 error = flow_divert_send_connect_result(fd_cb);
2563 }
2564
2565 if (connect_error || error) {
2566 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) {
2567 error = flow_divert_try_next_group(fd_cb);
2568 if (error && fd_cb->policy_control_unit == 0) {
2569 flow_divert_disable(fd_cb);
2570 goto done;
2571 } else if (error == 0) {
2572 goto done;
2573 }
2574 }
2575
2576 if (!connect_error) {
2577 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2578 so->so_error = (uint16_t)error;
2579 flow_divert_send_close_if_needed(fd_cb);
2580 } else {
2581 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2582 so->so_error = (uint16_t)connect_error;
2583 }
2584 flow_divert_disconnect_socket(so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT));
2585 } else {
2586 #if NECP
2587 /* Update NECP client with connected five-tuple */
2588 if (!uuid_is_null(inp->necp_client_uuid)) {
2589 socket_unlock(so, 0);
2590 necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp);
2591 socket_lock(so, 0);
2592 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
2593 /* The socket was closed while it was unlocked */
2594 goto done;
2595 }
2596 }
2597 #endif /* NECP */
2598
2599 flow_divert_send_buffered_data(fd_cb, FALSE);
2600 soisconnected(so);
2601 }
2602
2603 /* We don't need the connect packet any more */
2604 if (fd_cb->connect_packet != NULL) {
2605 mbuf_freem(fd_cb->connect_packet);
2606 fd_cb->connect_packet = NULL;
2607 }
2608
2609 /* We don't need the original remote endpoint any more */
2610 free_sockaddr(fd_cb->original_remote_endpoint);
2611 done:
2612 socket_unlock(so, 0);
2613 }
2614 FDUNLOCK(fd_cb);
2615 }
2616
2617 static void
flow_divert_handle_close(struct flow_divert_pcb * fd_cb,mbuf_t packet,int offset)2618 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2619 {
2620 uint32_t close_error = 0;
2621 int error = 0;
2622 int how = 0;
2623
2624 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2625 if (error) {
2626 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2627 return;
2628 }
2629
2630 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2631 if (error) {
2632 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2633 return;
2634 }
2635
2636 how = ntohl(how);
2637
2638 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2639
2640 FDLOCK(fd_cb);
2641 if (fd_cb->so != NULL) {
2642 bool is_connected = (SOCK_TYPE(fd_cb->so) == SOCK_STREAM || !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT));
2643 socket_lock(fd_cb->so, 0);
2644
2645 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2646 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider");
2647 goto done;
2648 }
2649
2650 fd_cb->so->so_error = (uint16_t)ntohl(close_error);
2651
2652 flow_divert_update_closed_state(fd_cb, how, TRUE);
2653
2654 /* Only do this for stream flows because "shutdown by peer" doesn't make sense for datagram flows */
2655 how = flow_divert_tunnel_how_closed(fd_cb);
2656 if (how == SHUT_RDWR) {
2657 flow_divert_disconnect_socket(fd_cb->so, is_connected);
2658 } else if (how == SHUT_RD && is_connected) {
2659 socantrcvmore(fd_cb->so);
2660 } else if (how == SHUT_WR && is_connected) {
2661 socantsendmore(fd_cb->so);
2662 }
2663 done:
2664 socket_unlock(fd_cb->so, 0);
2665 }
2666 FDUNLOCK(fd_cb);
2667 }
2668
2669 static mbuf_t
flow_divert_create_control_mbuf(struct flow_divert_pcb * fd_cb)2670 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb)
2671 {
2672 struct inpcb *inp = sotoinpcb(fd_cb->so);
2673 bool need_recvdstaddr = false;
2674 /* Socket flow tracking needs to see the local address */
2675 need_recvdstaddr = SOFLOW_ENABLED(inp->inp_socket);
2676 if ((inp->inp_vflag & INP_IPV4) &&
2677 fd_cb->local_endpoint.sa.sa_family == AF_INET &&
2678 ((inp->inp_flags & INP_RECVDSTADDR) || need_recvdstaddr)) {
2679 return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2680 } else if ((inp->inp_vflag & INP_IPV6) &&
2681 fd_cb->local_endpoint.sa.sa_family == AF_INET6 &&
2682 ((inp->inp_flags & IN6P_PKTINFO) || need_recvdstaddr)) {
2683 struct in6_pktinfo pi6;
2684 memset(&pi6, 0, sizeof(pi6));
2685 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr;
2686
2687 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
2688 }
2689 return NULL;
2690 }
2691
2692 static int
flow_divert_handle_data(struct flow_divert_pcb * fd_cb,mbuf_t packet,size_t offset)2693 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2694 {
2695 int error = 0;
2696
2697 FDLOCK(fd_cb);
2698 if (fd_cb->so != NULL) {
2699 mbuf_t data = NULL;
2700 size_t data_size;
2701 struct sockaddr_storage remote_address;
2702 boolean_t got_remote_sa = FALSE;
2703 boolean_t appended = FALSE;
2704 boolean_t append_success = FALSE;
2705
2706 socket_lock(fd_cb->so, 0);
2707
2708 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2709 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data");
2710 goto done;
2711 }
2712
2713 if (sbspace(&fd_cb->so->so_rcv) == 0) {
2714 error = ENOBUFS;
2715 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED;
2716 FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some data");
2717 goto done;
2718 }
2719
2720 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2721 uint32_t val_size = 0;
2722
2723 /* check if we got remote address with data */
2724 memset(&remote_address, 0, sizeof(remote_address));
2725 error = flow_divert_packet_get_tlv(packet, (int)offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2726 if (error || val_size > sizeof(remote_address)) {
2727 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2728 error = 0;
2729 } else {
2730 if (remote_address.ss_len > sizeof(remote_address)) {
2731 remote_address.ss_len = sizeof(remote_address);
2732 }
2733 /* validate the address */
2734 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2735 got_remote_sa = TRUE;
2736 } else {
2737 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid");
2738 }
2739 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2740 }
2741 }
2742
2743 data_size = (mbuf_pkthdr_len(packet) - offset);
2744
2745 if (fd_cb->so->so_state & SS_CANTRCVMORE) {
2746 FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size);
2747 goto done;
2748 }
2749
2750 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) {
2751 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so));
2752 goto done;
2753 }
2754
2755 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2756
2757 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2758 if (error || data == NULL) {
2759 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2760 goto done;
2761 }
2762
2763 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2764 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0);
2765 append_success = TRUE;
2766 } else {
2767 struct sockaddr *append_sa = NULL;
2768 mbuf_t mctl;
2769
2770 if (got_remote_sa == TRUE) {
2771 error = flow_divert_dup_addr(remote_address.ss_family, (struct sockaddr *)&remote_address, &append_sa);
2772 } else {
2773 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) {
2774 error = in6_mapped_peeraddr(fd_cb->so, &append_sa);
2775 } else {
2776 error = in_getpeeraddr(fd_cb->so, &append_sa);
2777 }
2778 }
2779 if (error) {
2780 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2781 }
2782
2783 mctl = flow_divert_create_control_mbuf(fd_cb);
2784 int append_error = 0;
2785 appended = sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error);
2786 if (appended || append_error == 0) {
2787 append_success = TRUE;
2788 } else {
2789 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error);
2790 }
2791
2792 free_sockaddr(append_sa);
2793 }
2794
2795 if (append_success) {
2796 fd_cb->bytes_received += data_size;
2797 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2798 }
2799
2800 if (appended) {
2801 sorwakeup(fd_cb->so);
2802 }
2803 done:
2804 socket_unlock(fd_cb->so, 0);
2805 }
2806 FDUNLOCK(fd_cb);
2807
2808 return error;
2809 }
2810
2811 static void
flow_divert_handle_read_notification(struct flow_divert_pcb * fd_cb,mbuf_t packet,int offset)2812 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2813 {
2814 uint32_t read_count = 0;
2815 int error = 0;
2816
2817 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2818 if (error) {
2819 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2820 return;
2821 }
2822
2823 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2824
2825 FDLOCK(fd_cb);
2826 if (fd_cb->so != NULL) {
2827 socket_lock(fd_cb->so, 0);
2828
2829 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2830 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification");
2831 goto done;
2832 }
2833
2834 fd_cb->send_window += ntohl(read_count);
2835 flow_divert_send_buffered_data(fd_cb, FALSE);
2836 done:
2837 socket_unlock(fd_cb->so, 0);
2838 }
2839 FDUNLOCK(fd_cb);
2840 }
2841
2842 static void
flow_divert_handle_group_init(struct flow_divert_group * group,mbuf_t packet,int offset)2843 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2844 {
2845 int error = 0;
2846 uint32_t key_size = 0;
2847 int log_level = 0;
2848 uint32_t flags = 0;
2849
2850 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2851 if (error) {
2852 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2853 return;
2854 }
2855
2856 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2857 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2858 return;
2859 }
2860
2861 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2862 if (!error) {
2863 nil_pcb.log_level = (uint8_t)log_level;
2864 }
2865
2866 lck_rw_lock_exclusive(&group->lck);
2867
2868 if (group->flags & FLOW_DIVERT_GROUP_FLAG_DEFUNCT) {
2869 FDLOG(LOG_ERR, &nil_pcb, "Skipping (re)initialization of defunct group %u", group->ctl_unit);
2870 lck_rw_done(&group->lck);
2871 return;
2872 }
2873
2874 if (group->token_key != NULL) {
2875 kfree_data(group->token_key, group->token_key_size);
2876 group->token_key = NULL;
2877 }
2878
2879 group->token_key = kalloc_data(key_size, Z_WAITOK);
2880 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2881 if (error) {
2882 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2883 kfree_data(group->token_key, key_size);
2884 group->token_key = NULL;
2885 lck_rw_done(&group->lck);
2886 return;
2887 }
2888
2889 group->token_key_size = key_size;
2890
2891 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2892 if (!error) {
2893 group->flags = flags;
2894 }
2895
2896 lck_rw_done(&group->lck);
2897 }
2898
2899 static void
flow_divert_handle_properties_update(struct flow_divert_pcb * fd_cb,mbuf_t packet,int offset)2900 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2901 {
2902 int error = 0;
2903 int out_if_index = 0;
2904 uint32_t app_data_length = 0;
2905
2906 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2907
2908 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2909 if (error) {
2910 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2911 }
2912
2913 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2914 if (error) {
2915 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2916 }
2917
2918 FDLOCK(fd_cb);
2919 if (fd_cb->so != NULL) {
2920 socket_lock(fd_cb->so, 0);
2921
2922 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2923 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update");
2924 goto done;
2925 }
2926
2927 if (out_if_index > 0) {
2928 flow_divert_scope(fd_cb, out_if_index, true);
2929 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint));
2930 }
2931
2932 if (app_data_length > 0) {
2933 uint8_t *app_data = NULL;
2934 app_data = kalloc_data(app_data_length, Z_WAITOK);
2935 if (app_data != NULL) {
2936 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2937 if (error == 0) {
2938 if (fd_cb->app_data != NULL) {
2939 kfree_data(fd_cb->app_data, fd_cb->app_data_length);
2940 }
2941 fd_cb->app_data = app_data;
2942 fd_cb->app_data_length = app_data_length;
2943 } else {
2944 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2945 kfree_data(app_data, app_data_length);
2946 }
2947 } else {
2948 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2949 }
2950 }
2951 done:
2952 socket_unlock(fd_cb->so, 0);
2953 }
2954 FDUNLOCK(fd_cb);
2955 }
2956
2957 static void
flow_divert_handle_app_map_create(struct flow_divert_group * group,mbuf_t packet,int offset)2958 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2959 {
2960 size_t bytes_mem_size;
2961 size_t child_maps_mem_size;
2962 size_t nodes_mem_size;
2963 size_t trie_memory_size = 0;
2964 int cursor;
2965 int error = 0;
2966 struct flow_divert_trie new_trie;
2967 int insert_error = 0;
2968 int prefix_count = -1;
2969 int signing_id_count = 0;
2970 size_t bytes_count = 0;
2971 size_t nodes_count = 0;
2972 size_t maps_count = 0;
2973
2974 lck_rw_lock_exclusive(&group->lck);
2975
2976 /* Re-set the current trie */
2977 if (group->signing_id_trie.memory != NULL) {
2978 kfree_data_addr(group->signing_id_trie.memory);
2979 }
2980 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2981 group->signing_id_trie.root = NULL_TRIE_IDX;
2982
2983 memset(&new_trie, 0, sizeof(new_trie));
2984
2985 /* Get the number of shared prefixes in the new set of signing ID strings */
2986 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2987
2988 if (prefix_count < 0 || error) {
2989 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
2990 lck_rw_done(&group->lck);
2991 return;
2992 }
2993
2994 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2995 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2996 cursor >= 0;
2997 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2998 uint32_t sid_size = 0;
2999 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
3000 if (error || sid_size == 0) {
3001 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
3002 signing_id_count = 0;
3003 break;
3004 }
3005 if (os_add_overflow(bytes_count, sid_size, &bytes_count)) {
3006 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while incrementing number of bytes");
3007 signing_id_count = 0;
3008 break;
3009 }
3010 signing_id_count++;
3011 }
3012
3013 if (signing_id_count == 0) {
3014 lck_rw_done(&group->lck);
3015 FDLOG0(LOG_NOTICE, &nil_pcb, "No signing identifiers");
3016 return;
3017 }
3018
3019 if (os_add3_overflow(prefix_count, signing_id_count, 1, &nodes_count)) { /* + 1 for the root node */
3020 lck_rw_done(&group->lck);
3021 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of nodes");
3022 return;
3023 }
3024
3025 if (os_add_overflow(prefix_count, 1, &maps_count)) { /* + 1 for the root node */
3026 lck_rw_done(&group->lck);
3027 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of maps");
3028 return;
3029 }
3030
3031 if (bytes_count > UINT16_MAX || nodes_count > UINT16_MAX || maps_count > UINT16_MAX) {
3032 lck_rw_done(&group->lck);
3033 FDLOG(LOG_NOTICE, &nil_pcb, "Invalid bytes count (%lu), nodes count (%lu) or maps count (%lu)", bytes_count, nodes_count, maps_count);
3034 return;
3035 }
3036
3037 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
3038 nodes_count, maps_count, bytes_count);
3039
3040 if (os_mul_overflow(sizeof(*new_trie.nodes), (size_t)nodes_count, &nodes_mem_size) ||
3041 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, (size_t)maps_count, &child_maps_mem_size) ||
3042 os_mul_overflow(sizeof(*new_trie.bytes), (size_t)bytes_count, &bytes_mem_size) ||
3043 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
3044 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
3045 lck_rw_done(&group->lck);
3046 return;
3047 }
3048
3049 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
3050 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
3051 lck_rw_done(&group->lck);
3052 return;
3053 }
3054
3055 new_trie.memory = kalloc_data(trie_memory_size, Z_WAITOK);
3056 if (new_trie.memory == NULL) {
3057 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
3058 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
3059 lck_rw_done(&group->lck);
3060 return;
3061 }
3062
3063 new_trie.bytes_count = (uint16_t)bytes_count;
3064 new_trie.nodes_count = (uint16_t)nodes_count;
3065 new_trie.child_maps_count = (uint16_t)maps_count;
3066
3067 /* Initialize the free lists */
3068 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
3069 new_trie.nodes_free_next = 0;
3070 memset(new_trie.nodes, 0, nodes_mem_size);
3071
3072 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
3073 new_trie.child_maps_free_next = 0;
3074 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
3075
3076 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
3077 new_trie.bytes_free_next = 0;
3078 memset(new_trie.bytes, 0, bytes_mem_size);
3079
3080 /* The root is an empty node */
3081 new_trie.root = trie_node_alloc(&new_trie);
3082
3083 /* Add each signing ID to the trie */
3084 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
3085 cursor >= 0;
3086 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3087 uint32_t sid_size = 0;
3088 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
3089 if (error || sid_size == 0) {
3090 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
3091 insert_error = EINVAL;
3092 break;
3093 }
3094 if (sid_size <= UINT16_MAX && new_trie.bytes_free_next + (uint16_t)sid_size <= new_trie.bytes_count) {
3095 uint16_t new_node_idx;
3096 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
3097 if (error) {
3098 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
3099 insert_error = EINVAL;
3100 break;
3101 }
3102 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
3103 if (new_node_idx == NULL_TRIE_IDX) {
3104 insert_error = EINVAL;
3105 break;
3106 }
3107 } else {
3108 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
3109 insert_error = ENOBUFS;
3110 break;
3111 }
3112 }
3113
3114 if (!insert_error) {
3115 group->signing_id_trie = new_trie;
3116 } else {
3117 kfree_data(new_trie.memory, trie_memory_size);
3118 }
3119
3120 lck_rw_done(&group->lck);
3121 }
3122
3123 static int
flow_divert_input(mbuf_t packet,struct flow_divert_group * group)3124 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
3125 {
3126 struct flow_divert_packet_header hdr;
3127 int error = 0;
3128 struct flow_divert_pcb *fd_cb;
3129
3130 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
3131 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
3132 error = EINVAL;
3133 goto done;
3134 }
3135
3136 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
3137 if (error) {
3138 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
3139 error = ENOBUFS;
3140 goto done;
3141 }
3142
3143 hdr.conn_id = ntohl(hdr.conn_id);
3144
3145 if (hdr.conn_id == 0) {
3146 switch (hdr.packet_type) {
3147 case FLOW_DIVERT_PKT_GROUP_INIT:
3148 flow_divert_handle_group_init(group, packet, sizeof(hdr));
3149 break;
3150 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
3151 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
3152 break;
3153 default:
3154 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
3155 break;
3156 }
3157 goto done;
3158 }
3159
3160 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
3161 if (fd_cb == NULL) {
3162 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
3163 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
3164 }
3165 goto done;
3166 }
3167
3168 switch (hdr.packet_type) {
3169 case FLOW_DIVERT_PKT_CONNECT_RESULT:
3170 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
3171 break;
3172 case FLOW_DIVERT_PKT_CLOSE:
3173 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
3174 break;
3175 case FLOW_DIVERT_PKT_DATA:
3176 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
3177 break;
3178 case FLOW_DIVERT_PKT_READ_NOTIFY:
3179 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
3180 break;
3181 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
3182 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
3183 break;
3184 default:
3185 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
3186 break;
3187 }
3188
3189 FDRELEASE(fd_cb);
3190
3191 done:
3192 mbuf_freem(packet);
3193 return error;
3194 }
3195
3196 static void
flow_divert_close_all(struct flow_divert_group * group)3197 flow_divert_close_all(struct flow_divert_group *group)
3198 {
3199 struct flow_divert_pcb *fd_cb;
3200 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3201
3202 SLIST_INIT(&tmp_list);
3203
3204 lck_rw_lock_exclusive(&group->lck);
3205
3206 MBUFQ_DRAIN(&group->send_queue);
3207
3208 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3209 FDRETAIN(fd_cb);
3210 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3211 }
3212
3213 group->flags |= FLOW_DIVERT_GROUP_FLAG_DEFUNCT;
3214
3215 lck_rw_done(&group->lck);
3216
3217 while (!SLIST_EMPTY(&tmp_list)) {
3218 fd_cb = SLIST_FIRST(&tmp_list);
3219 FDLOCK(fd_cb);
3220 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
3221 if (fd_cb->so != NULL) {
3222 socket_lock(fd_cb->so, 0);
3223 flow_divert_pcb_remove(fd_cb);
3224 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
3225 fd_cb->so->so_error = ECONNABORTED;
3226 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT));
3227 socket_unlock(fd_cb->so, 0);
3228 }
3229 FDUNLOCK(fd_cb);
3230 FDRELEASE(fd_cb);
3231 }
3232 }
3233
3234 void
flow_divert_detach(struct socket * so)3235 flow_divert_detach(struct socket *so)
3236 {
3237 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3238
3239 if (!SO_IS_DIVERTED(so)) {
3240 return;
3241 }
3242
3243 so->so_flags &= ~SOF_FLOW_DIVERT;
3244 so->so_fd_pcb = NULL;
3245
3246 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
3247
3248 if (fd_cb->group != NULL) {
3249 /* Last-ditch effort to send any buffered data */
3250 flow_divert_send_buffered_data(fd_cb, TRUE);
3251
3252 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3253 flow_divert_send_close_if_needed(fd_cb);
3254 /* Remove from the group */
3255 flow_divert_pcb_remove(fd_cb);
3256 }
3257
3258 socket_unlock(so, 0);
3259 FDLOCK(fd_cb);
3260 fd_cb->so = NULL;
3261 FDUNLOCK(fd_cb);
3262 socket_lock(so, 0);
3263
3264 FDRELEASE(fd_cb); /* Release the socket's reference */
3265 }
3266
3267 static int
flow_divert_close(struct socket * so)3268 flow_divert_close(struct socket *so)
3269 {
3270 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3271
3272 if (!SO_IS_DIVERTED(so)) {
3273 return EINVAL;
3274 }
3275
3276 FDLOG0(LOG_INFO, fd_cb, "Closing");
3277
3278 if (SOCK_TYPE(so) == SOCK_STREAM) {
3279 soisdisconnecting(so);
3280 sbflush(&so->so_rcv);
3281 }
3282
3283 flow_divert_send_buffered_data(fd_cb, TRUE);
3284 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3285 flow_divert_send_close_if_needed(fd_cb);
3286
3287 /* Remove from the group */
3288 flow_divert_pcb_remove(fd_cb);
3289
3290 return 0;
3291 }
3292
3293 static int
flow_divert_disconnectx(struct socket * so,sae_associd_t aid,sae_connid_t cid __unused)3294 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
3295 sae_connid_t cid __unused)
3296 {
3297 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
3298 return EINVAL;
3299 }
3300
3301 return flow_divert_close(so);
3302 }
3303
3304 static int
flow_divert_shutdown(struct socket * so)3305 flow_divert_shutdown(struct socket *so)
3306 {
3307 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3308
3309 if (!SO_IS_DIVERTED(so)) {
3310 return EINVAL;
3311 }
3312
3313 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
3314
3315 socantsendmore(so);
3316
3317 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
3318 flow_divert_send_close_if_needed(fd_cb);
3319
3320 return 0;
3321 }
3322
3323 static int
flow_divert_rcvd(struct socket * so,int flags __unused)3324 flow_divert_rcvd(struct socket *so, int flags __unused)
3325 {
3326 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3327 int space = 0;
3328
3329 if (!SO_IS_DIVERTED(so)) {
3330 return EINVAL;
3331 }
3332
3333 space = sbspace(&so->so_rcv);
3334 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space);
3335 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) &&
3336 (space > 0) &&
3337 flow_divert_send_read_notification(fd_cb) == 0) {
3338 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification");
3339 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED;
3340 }
3341
3342 return 0;
3343 }
3344
3345 static int
flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet,struct sockaddr * toaddr)3346 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
3347 {
3348 int error = 0;
3349 int port = 0;
3350
3351 if (!flow_divert_is_sockaddr_valid(toaddr)) {
3352 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
3353 error = EINVAL;
3354 goto done;
3355 }
3356
3357 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
3358 if (error) {
3359 goto done;
3360 }
3361
3362 if (toaddr->sa_family == AF_INET) {
3363 port = ntohs((satosin(toaddr))->sin_port);
3364 } else {
3365 port = ntohs((satosin6(toaddr))->sin6_port);
3366 }
3367
3368 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
3369 if (error) {
3370 goto done;
3371 }
3372
3373 done:
3374 return error;
3375 }
3376
3377 struct sockaddr *
flow_divert_get_buffered_target_address(mbuf_t buffer)3378 flow_divert_get_buffered_target_address(mbuf_t buffer)
3379 {
3380 if (buffer != NULL && buffer->m_type == MT_SONAME) {
3381 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
3382 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
3383 return toaddr;
3384 }
3385 }
3386 return NULL;
3387 }
3388
3389 static boolean_t
flow_divert_is_sockaddr_valid(struct sockaddr * addr)3390 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
3391 {
3392 switch (addr->sa_family) {
3393 case AF_INET:
3394 if (addr->sa_len < sizeof(struct sockaddr_in)) {
3395 return FALSE;
3396 }
3397 break;
3398 case AF_INET6:
3399 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3400 return FALSE;
3401 }
3402 break;
3403 default:
3404 return FALSE;
3405 }
3406 return TRUE;
3407 }
3408
3409 static errno_t
flow_divert_dup_addr(sa_family_t family,struct sockaddr * addr,struct sockaddr ** dup)3410 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
3411 struct sockaddr **dup)
3412 {
3413 int error = 0;
3414 struct sockaddr *result;
3415 struct sockaddr_storage ss;
3416
3417 if (addr != NULL) {
3418 result = addr;
3419 } else {
3420 memset(&ss, 0, sizeof(ss));
3421 ss.ss_family = family;
3422 if (ss.ss_family == AF_INET) {
3423 ss.ss_len = sizeof(struct sockaddr_in);
3424 } else if (ss.ss_family == AF_INET6) {
3425 ss.ss_len = sizeof(struct sockaddr_in6);
3426 } else {
3427 error = EINVAL;
3428 }
3429 result = (struct sockaddr *)&ss;
3430 }
3431
3432 if (!error) {
3433 *dup = dup_sockaddr(result, 1);
3434 if (*dup == NULL) {
3435 error = ENOBUFS;
3436 }
3437 }
3438
3439 return error;
3440 }
3441
3442 static void
flow_divert_disconnect_socket(struct socket * so,bool is_connected)3443 flow_divert_disconnect_socket(struct socket *so, bool is_connected)
3444 {
3445 if (SOCK_TYPE(so) == SOCK_STREAM || is_connected) {
3446 soisdisconnected(so);
3447 }
3448 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3449 struct inpcb *inp = sotoinpcb(so);
3450 if (inp != NULL && !(so->so_flags & SOF_PCBCLEARING)) {
3451 /*
3452 * Let NetworkStatistics know this PCB is going away
3453 * before we detach it.
3454 */
3455 if (nstat_collect && (SOCK_PROTO(so) == IPPROTO_TCP || SOCK_PROTO(so) == IPPROTO_UDP)) {
3456 nstat_pcb_detach(inp);
3457 }
3458
3459 if (SOCK_DOM(so) == PF_INET6) {
3460 ROUTE_RELEASE(&inp->in6p_route);
3461 } else {
3462 ROUTE_RELEASE(&inp->inp_route);
3463 }
3464 inp->inp_state = INPCB_STATE_DEAD;
3465 /* makes sure we're not called twice from so_close */
3466 so->so_flags |= SOF_PCBCLEARING;
3467 inpcb_gc_sched(inp->inp_pcbinfo, INPCB_TIMER_FAST);
3468 }
3469 }
3470 }
3471
3472 static errno_t
flow_divert_ctloutput(struct socket * so,struct sockopt * sopt)3473 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
3474 {
3475 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3476
3477 if (!SO_IS_DIVERTED(so)) {
3478 return EINVAL;
3479 }
3480
3481 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
3482 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3483 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
3484 }
3485 }
3486
3487 if (SOCK_DOM(so) == PF_INET) {
3488 return g_tcp_protosw->pr_ctloutput(so, sopt);
3489 } else if (SOCK_DOM(so) == PF_INET6) {
3490 return g_tcp6_protosw->pr_ctloutput(so, sopt);
3491 }
3492 return 0;
3493 }
3494
3495 static errno_t
flow_divert_connect_out_internal(struct socket * so,struct sockaddr * to,proc_t p,bool implicit)3496 flow_divert_connect_out_internal(struct socket *so, struct sockaddr *to, proc_t p, bool implicit)
3497 {
3498 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3499 int error = 0;
3500 struct inpcb *inp = sotoinpcb(so);
3501 struct sockaddr_in *sinp;
3502 mbuf_t connect_packet = NULL;
3503 int do_send = 1;
3504
3505 if (!SO_IS_DIVERTED(so)) {
3506 return EINVAL;
3507 }
3508
3509 if (fd_cb->group == NULL) {
3510 error = ENETUNREACH;
3511 goto done;
3512 }
3513
3514 if (inp == NULL) {
3515 error = EINVAL;
3516 goto done;
3517 } else if (inp->inp_state == INPCB_STATE_DEAD) {
3518 if (so->so_error) {
3519 error = so->so_error;
3520 so->so_error = 0;
3521 } else {
3522 error = EINVAL;
3523 }
3524 goto done;
3525 }
3526
3527 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3528 error = EALREADY;
3529 goto done;
3530 }
3531
3532 FDLOG0(LOG_INFO, fd_cb, "Connecting");
3533
3534 if (fd_cb->connect_packet == NULL) {
3535 struct sockaddr_in sin = {};
3536 struct ifnet *ifp = NULL;
3537
3538 if (to == NULL) {
3539 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
3540 error = EINVAL;
3541 goto done;
3542 }
3543
3544 if (!flow_divert_is_sockaddr_valid(to)) {
3545 FDLOG0(LOG_ERR, fd_cb, "Destination address is not valid when creating connect packet");
3546 error = EINVAL;
3547 goto done;
3548 }
3549
3550 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0);
3551 if (fd_cb->original_remote_endpoint == NULL) {
3552 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint");
3553 error = ENOMEM;
3554 goto done;
3555 }
3556 fd_cb->original_vflag = inp->inp_vflag;
3557 fd_cb->original_last_outifp = inp->inp_last_outifp;
3558 fd_cb->original_last_outifp6 = inp->in6p_last_outifp;
3559
3560 sinp = (struct sockaddr_in *)(void *)to;
3561 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
3562 error = EAFNOSUPPORT;
3563 goto done;
3564 }
3565
3566 if (to->sa_family == AF_INET6 && !(inp->inp_flags & IN6P_IPV6_V6ONLY)) {
3567 struct sockaddr_in6 sin6 = {};
3568 sin6.sin6_family = AF_INET6;
3569 sin6.sin6_len = sizeof(struct sockaddr_in6);
3570 sin6.sin6_port = satosin6(to)->sin6_port;
3571 sin6.sin6_addr = satosin6(to)->sin6_addr;
3572 if (IN6_IS_ADDR_V4MAPPED(&(sin6.sin6_addr))) {
3573 in6_sin6_2_sin(&sin, &sin6);
3574 to = (struct sockaddr *)&sin;
3575 }
3576 }
3577
3578 if (to->sa_family == AF_INET6) {
3579 struct sockaddr_in6 *to6 = satosin6(to);
3580
3581 inp->inp_vflag &= ~INP_IPV4;
3582 inp->inp_vflag |= INP_IPV6;
3583 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6);
3584 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6;
3585 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport;
3586 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp);
3587 if (error) {
3588 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error);
3589 if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || IN6_IS_ADDR_UNSPECIFIED(&(satosin6(to)->sin6_addr))) {
3590 error = 0;
3591 } else {
3592 goto done;
3593 }
3594 }
3595 if (ifp != NULL) {
3596 inp->in6p_last_outifp = ifp;
3597 ifnet_release(ifp);
3598 }
3599
3600 if (IN6_IS_SCOPE_EMBED(&(fd_cb->local_endpoint.sin6.sin6_addr)) &&
3601 in6_embedded_scope &&
3602 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] != 0) {
3603 fd_cb->local_endpoint.sin6.sin6_scope_id = ntohs(fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1]);
3604 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] = 0;
3605 }
3606
3607 if (IN6_IS_SCOPE_EMBED(&(to6->sin6_addr)) &&
3608 in6_embedded_scope &&
3609 to6->sin6_addr.s6_addr16[1] != 0) {
3610 to6->sin6_scope_id = ntohs(to6->sin6_addr.s6_addr16[1]);
3611 to6->sin6_addr.s6_addr16[1] = 0;
3612 }
3613 } else if (to->sa_family == AF_INET) {
3614 inp->inp_vflag |= INP_IPV4;
3615 inp->inp_vflag &= ~INP_IPV6;
3616 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in);
3617 fd_cb->local_endpoint.sin.sin_family = AF_INET;
3618 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport;
3619 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0);
3620 if (error) {
3621 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error);
3622 if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || satosin(to)->sin_addr.s_addr == INADDR_ANY) {
3623 error = 0;
3624 } else {
3625 goto done;
3626 }
3627 }
3628 if (ifp != NULL) {
3629 inp->inp_last_outifp = ifp;
3630 ifnet_release(ifp);
3631 }
3632 } else {
3633 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family);
3634 }
3635
3636 error = flow_divert_check_no_cellular(fd_cb) ||
3637 flow_divert_check_no_expensive(fd_cb) ||
3638 flow_divert_check_no_constrained(fd_cb);
3639 if (error) {
3640 goto done;
3641 }
3642
3643 if (SOCK_TYPE(so) == SOCK_STREAM || /* TCP or */
3644 !implicit || /* connect() was called or */
3645 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) || /* local address is not un-specified */
3646 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
3647 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR;
3648 }
3649
3650 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
3651 if (error) {
3652 goto done;
3653 }
3654
3655 if (!implicit || SOCK_TYPE(so) == SOCK_STREAM) {
3656 flow_divert_set_remote_endpoint(fd_cb, to);
3657 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint));
3658 }
3659
3660 if (implicit) {
3661 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT;
3662 }
3663
3664 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
3665 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
3666 do_send = 0;
3667 }
3668
3669 fd_cb->connect_packet = connect_packet;
3670 connect_packet = NULL;
3671 } else {
3672 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
3673 }
3674
3675 if (do_send) {
3676 error = flow_divert_send_connect_packet(fd_cb);
3677 if (error) {
3678 goto done;
3679 }
3680
3681 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3682 }
3683
3684 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) {
3685 soisconnected(so);
3686 } else {
3687 soisconnecting(so);
3688 }
3689
3690 done:
3691 return error;
3692 }
3693
3694 errno_t
flow_divert_connect_out(struct socket * so,struct sockaddr * to,proc_t p)3695 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
3696 {
3697 #if CONTENT_FILTER
3698 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_flags & SOF_CONTENT_FILTER)) {
3699 int error = cfil_sock_attach(so, NULL, to, CFS_CONNECTION_DIR_OUT);
3700 if (error != 0) {
3701 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3702 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error);
3703 return error;
3704 }
3705 }
3706 #endif /* CONTENT_FILTER */
3707
3708 return flow_divert_connect_out_internal(so, to, p, false);
3709 }
3710
3711 static int
flow_divert_connectx_out_common(struct socket * so,struct sockaddr * dst,struct proc * p,sae_connid_t * pcid,struct uio * auio,user_ssize_t * bytes_written)3712 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3713 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
3714 {
3715 struct inpcb *inp = sotoinpcb(so);
3716 int error;
3717
3718 if (inp == NULL) {
3719 return EINVAL;
3720 }
3721
3722 VERIFY(dst != NULL);
3723
3724 #if CONTENT_FILTER && NECP
3725 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3726 if (fd_cb != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_TOKEN) &&
3727 SOCK_TYPE(so) == SOCK_STREAM && !(so->so_flags & SOF_CONTENT_FILTER)) {
3728 inp_update_necp_policy(sotoinpcb(so), NULL, dst, 0);
3729 }
3730 #endif /* CONTENT_FILTER */
3731
3732 error = flow_divert_connect_out(so, dst, p);
3733
3734 if (error != 0) {
3735 return error;
3736 }
3737
3738 /* if there is data, send it */
3739 if (auio != NULL) {
3740 user_ssize_t datalen = 0;
3741
3742 socket_unlock(so, 0);
3743
3744 VERIFY(bytes_written != NULL);
3745
3746 datalen = uio_resid(auio);
3747 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3748 socket_lock(so, 0);
3749
3750 if (error == 0 || error == EWOULDBLOCK) {
3751 *bytes_written = datalen - uio_resid(auio);
3752 }
3753
3754 /*
3755 * sosend returns EWOULDBLOCK if it's a non-blocking
3756 * socket or a timeout occured (this allows to return
3757 * the amount of queued data through sendit()).
3758 *
3759 * However, connectx() returns EINPROGRESS in case of a
3760 * blocking socket. So we change the return value here.
3761 */
3762 if (error == EWOULDBLOCK) {
3763 error = EINPROGRESS;
3764 }
3765 }
3766
3767 if (error == 0 && pcid != NULL) {
3768 *pcid = 1; /* there is only 1 connection for a TCP */
3769 }
3770
3771 return error;
3772 }
3773
3774 static int
flow_divert_connectx_out(struct socket * so,struct sockaddr * src __unused,struct sockaddr * dst,struct proc * p,uint32_t ifscope __unused,sae_associd_t aid __unused,sae_connid_t * pcid,uint32_t flags __unused,void * arg __unused,uint32_t arglen __unused,struct uio * uio,user_ssize_t * bytes_written)3775 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3776 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3777 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3778 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3779 {
3780 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3781 }
3782
3783 static int
flow_divert_connectx6_out(struct socket * so,struct sockaddr * src __unused,struct sockaddr * dst,struct proc * p,uint32_t ifscope __unused,sae_associd_t aid __unused,sae_connid_t * pcid,uint32_t flags __unused,void * arg __unused,uint32_t arglen __unused,struct uio * uio,user_ssize_t * bytes_written)3784 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3785 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3786 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3787 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3788 {
3789 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3790 }
3791
3792 static errno_t
flow_divert_data_out(struct socket * so,int flags,mbuf_t data,struct sockaddr * to,mbuf_t control,struct proc * p)3793 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3794 {
3795 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3796 int error = 0;
3797 struct inpcb *inp;
3798 #if CONTENT_FILTER
3799 struct m_tag *cfil_tag = NULL;
3800 #endif
3801
3802 if (!SO_IS_DIVERTED(so)) {
3803 return EINVAL;
3804 }
3805
3806 inp = sotoinpcb(so);
3807 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3808 error = ECONNRESET;
3809 goto done;
3810 }
3811
3812 if (control && mbuf_len(control) > 0) {
3813 error = EINVAL;
3814 goto done;
3815 }
3816
3817 if (flags & MSG_OOB) {
3818 error = EINVAL;
3819 goto done; /* We don't support OOB data */
3820 }
3821
3822 if ((fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) && SOCK_TYPE(so) == SOCK_DGRAM) {
3823 /* The provider considers this datagram flow to be closed, so no data can be sent */
3824 FDLOG0(LOG_INFO, fd_cb, "provider is no longer accepting writes, cannot send data");
3825 error = EHOSTUNREACH;
3826 goto done;
3827 }
3828
3829 #if CONTENT_FILTER
3830 /*
3831 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3832 * retrieve the CFIL saved remote address from the mbuf and use it.
3833 */
3834 if (to == NULL && CFIL_DGRAM_FILTERED(so)) {
3835 struct sockaddr *cfil_faddr = NULL;
3836 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3837 if (cfil_tag) {
3838 to = (struct sockaddr *)(void *)cfil_faddr;
3839 }
3840 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3841 }
3842 #endif
3843
3844 /* Implicit connect */
3845 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3846 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3847
3848 error = flow_divert_connect_out_internal(so, to, p, true);
3849 if (error) {
3850 goto done;
3851 }
3852 } else {
3853 error = flow_divert_check_no_cellular(fd_cb) ||
3854 flow_divert_check_no_expensive(fd_cb) ||
3855 flow_divert_check_no_constrained(fd_cb);
3856 if (error) {
3857 goto done;
3858 }
3859 }
3860
3861 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3862
3863 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3864 error = flow_divert_send_app_data(fd_cb, data, to);
3865
3866 data = NULL;
3867
3868 if (error) {
3869 goto done;
3870 }
3871
3872 if (flags & PRUS_EOF) {
3873 flow_divert_shutdown(so);
3874 }
3875
3876 done:
3877 if (data) {
3878 mbuf_freem(data);
3879 }
3880 if (control) {
3881 mbuf_free(control);
3882 }
3883 #if CONTENT_FILTER
3884 if (cfil_tag) {
3885 m_tag_free(cfil_tag);
3886 }
3887 #endif
3888
3889 return error;
3890 }
3891
3892 static int
flow_divert_preconnect(struct socket * so)3893 flow_divert_preconnect(struct socket *so)
3894 {
3895 int error = 0;
3896 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3897
3898 if (!SO_IS_DIVERTED(so)) {
3899 return EINVAL;
3900 }
3901
3902 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3903 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3904 error = flow_divert_send_connect_packet(so->so_fd_pcb);
3905 if (error) {
3906 return error;
3907 }
3908
3909 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3910 }
3911
3912 soclearfastopen(so);
3913
3914 return error;
3915 }
3916
3917 static void
flow_divert_set_protosw(struct socket * so)3918 flow_divert_set_protosw(struct socket *so)
3919 {
3920 if (SOCK_DOM(so) == PF_INET) {
3921 so->so_proto = &g_flow_divert_in_protosw;
3922 } else {
3923 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3924 }
3925 }
3926
3927 static void
flow_divert_set_udp_protosw(struct socket * so)3928 flow_divert_set_udp_protosw(struct socket *so)
3929 {
3930 if (SOCK_DOM(so) == PF_INET) {
3931 so->so_proto = &g_flow_divert_in_udp_protosw;
3932 } else {
3933 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3934 }
3935 }
3936
3937 errno_t
flow_divert_implicit_data_out(struct socket * so,int flags,mbuf_t data,struct sockaddr * to,mbuf_t control,struct proc * p)3938 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3939 {
3940 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3941 struct inpcb *inp;
3942 int error = 0;
3943
3944 inp = sotoinpcb(so);
3945 if (inp == NULL) {
3946 return EINVAL;
3947 }
3948
3949 if (fd_cb == NULL) {
3950 error = flow_divert_pcb_init(so);
3951 fd_cb = so->so_fd_pcb;
3952 if (error != 0 || fd_cb == NULL) {
3953 goto done;
3954 }
3955 }
3956 return flow_divert_data_out(so, flags, data, to, control, p);
3957
3958 done:
3959 if (data) {
3960 mbuf_freem(data);
3961 }
3962 if (control) {
3963 mbuf_free(control);
3964 }
3965
3966 return error;
3967 }
3968
3969 static errno_t
flow_divert_pcb_init_internal(struct socket * so,uint32_t ctl_unit,uint32_t aggregate_unit)3970 flow_divert_pcb_init_internal(struct socket *so, uint32_t ctl_unit, uint32_t aggregate_unit)
3971 {
3972 errno_t error = 0;
3973 struct flow_divert_pcb *fd_cb = NULL;
3974 uint32_t agg_unit = aggregate_unit;
3975 uint32_t policy_control_unit = ctl_unit;
3976 bool is_aggregate = false;
3977
3978 if (so->so_flags & SOF_FLOW_DIVERT) {
3979 return EALREADY;
3980 }
3981
3982 fd_cb = flow_divert_pcb_create(so);
3983 if (fd_cb == NULL) {
3984 return ENOMEM;
3985 }
3986
3987 do {
3988 uint32_t group_unit = flow_divert_derive_kernel_control_unit(&policy_control_unit, &agg_unit, &is_aggregate);
3989 if (group_unit == 0 || group_unit >= GROUP_COUNT_MAX) {
3990 FDLOG0(LOG_ERR, fd_cb, "No valid group is available, cannot init flow divert");
3991 error = EINVAL;
3992 break;
3993 }
3994
3995 error = flow_divert_add_to_group(fd_cb, group_unit);
3996 if (error == 0) {
3997 so->so_fd_pcb = fd_cb;
3998 so->so_flags |= SOF_FLOW_DIVERT;
3999 fd_cb->control_group_unit = group_unit;
4000 fd_cb->policy_control_unit = ctl_unit;
4001 fd_cb->aggregate_unit = agg_unit;
4002 if (is_aggregate) {
4003 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT;
4004 } else {
4005 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT;
4006 }
4007
4008 if (SOCK_TYPE(so) == SOCK_STREAM) {
4009 flow_divert_set_protosw(so);
4010 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
4011 flow_divert_set_udp_protosw(so);
4012 }
4013
4014 FDLOG0(LOG_INFO, fd_cb, "Created");
4015 } else if (error != ENOENT) {
4016 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
4017 }
4018 } while (error == ENOENT);
4019
4020 if (error != 0) {
4021 FDRELEASE(fd_cb);
4022 }
4023
4024 return error;
4025 }
4026
4027 errno_t
flow_divert_pcb_init(struct socket * so)4028 flow_divert_pcb_init(struct socket *so)
4029 {
4030 struct inpcb *inp = sotoinpcb(so);
4031 uint32_t aggregate_units = 0;
4032 uint32_t ctl_unit = necp_socket_get_flow_divert_control_unit(inp, &aggregate_units);
4033 return flow_divert_pcb_init_internal(so, ctl_unit, aggregate_units);
4034 }
4035
4036 errno_t
flow_divert_token_set(struct socket * so,struct sockopt * sopt)4037 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
4038 {
4039 uint32_t ctl_unit = 0;
4040 uint32_t key_unit = 0;
4041 uint32_t aggregate_unit = 0;
4042 int error = 0;
4043 int hmac_error = 0;
4044 mbuf_t token = NULL;
4045
4046 if (so->so_flags & SOF_FLOW_DIVERT) {
4047 error = EALREADY;
4048 goto done;
4049 }
4050
4051 if (g_init_result) {
4052 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
4053 error = ENOPROTOOPT;
4054 goto done;
4055 }
4056
4057 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
4058 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
4059 (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6)) {
4060 error = EINVAL;
4061 goto done;
4062 } else {
4063 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
4064 struct tcpcb *tp = sototcpcb(so);
4065 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
4066 error = EINVAL;
4067 goto done;
4068 }
4069 }
4070 }
4071
4072 error = soopt_getm(sopt, &token);
4073 if (error) {
4074 token = NULL;
4075 goto done;
4076 }
4077
4078 error = soopt_mcopyin(sopt, token);
4079 if (error) {
4080 token = NULL;
4081 goto done;
4082 }
4083
4084 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
4085 if (!error) {
4086 key_unit = ntohl(key_unit);
4087 if (key_unit >= GROUP_COUNT_MAX) {
4088 key_unit = 0;
4089 }
4090 } else if (error != ENOENT) {
4091 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
4092 goto done;
4093 } else {
4094 key_unit = 0;
4095 }
4096
4097 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
4098 if (error) {
4099 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
4100 goto done;
4101 }
4102
4103 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_AGGREGATE_UNIT, sizeof(aggregate_unit), (void *)&aggregate_unit, NULL);
4104 if (error && error != ENOENT) {
4105 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the aggregate unit from the token: %d", error);
4106 goto done;
4107 }
4108
4109 /* A valid kernel control unit is required */
4110 ctl_unit = ntohl(ctl_unit);
4111 aggregate_unit = ntohl(aggregate_unit);
4112
4113 if (ctl_unit > 0 && ctl_unit < GROUP_COUNT_MAX) {
4114 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
4115 if (hmac_error && hmac_error != ENOENT) {
4116 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
4117 error = hmac_error;
4118 goto done;
4119 }
4120 }
4121
4122 error = flow_divert_pcb_init_internal(so, ctl_unit, aggregate_unit);
4123 if (error == 0) {
4124 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
4125 int log_level = LOG_NOTICE;
4126
4127 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
4128 if (error == 0) {
4129 fd_cb->log_level = (uint8_t)log_level;
4130 }
4131 error = 0;
4132
4133 fd_cb->connect_token = token;
4134 token = NULL;
4135
4136 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN;
4137 }
4138
4139 if (hmac_error == 0) {
4140 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
4141 if (fd_cb != NULL) {
4142 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
4143 }
4144 }
4145
4146 done:
4147 if (token != NULL) {
4148 mbuf_freem(token);
4149 }
4150
4151 return error;
4152 }
4153
4154 errno_t
flow_divert_token_get(struct socket * so,struct sockopt * sopt)4155 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
4156 {
4157 uint32_t ctl_unit;
4158 int error = 0;
4159 uint8_t hmac[SHA_DIGEST_LENGTH];
4160 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
4161 mbuf_t token = NULL;
4162 struct flow_divert_group *control_group = NULL;
4163
4164 if (!SO_IS_DIVERTED(so)) {
4165 error = EINVAL;
4166 goto done;
4167 }
4168
4169 if (fd_cb->group == NULL) {
4170 error = EINVAL;
4171 goto done;
4172 }
4173
4174 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
4175 if (error) {
4176 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
4177 goto done;
4178 }
4179
4180 ctl_unit = htonl(fd_cb->group->ctl_unit);
4181
4182 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
4183 if (error) {
4184 goto done;
4185 }
4186
4187 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
4188 if (error) {
4189 goto done;
4190 }
4191
4192 if (fd_cb->app_data != NULL) {
4193 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_data);
4194 if (error) {
4195 goto done;
4196 }
4197 }
4198
4199 control_group = flow_divert_group_lookup(fd_cb->control_group_unit, fd_cb);
4200 if (control_group != NULL) {
4201 lck_rw_lock_shared(&control_group->lck);
4202 ctl_unit = htonl(control_group->ctl_unit);
4203 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
4204 if (!error) {
4205 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
4206 }
4207 lck_rw_done(&control_group->lck);
4208 FDGRP_RELEASE(control_group);
4209 } else {
4210 error = ENOPROTOOPT;
4211 }
4212
4213 if (error) {
4214 goto done;
4215 }
4216
4217 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
4218 if (error) {
4219 goto done;
4220 }
4221
4222 if (sopt->sopt_val == USER_ADDR_NULL) {
4223 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
4224 sopt->sopt_valsize = mbuf_pkthdr_len(token);
4225 goto done;
4226 }
4227
4228 error = soopt_mcopyout(sopt, token);
4229 if (error) {
4230 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
4231 goto done;
4232 }
4233
4234 done:
4235 if (token != NULL) {
4236 mbuf_freem(token);
4237 }
4238
4239 return error;
4240 }
4241
4242 void
flow_divert_group_destroy(struct flow_divert_group * group)4243 flow_divert_group_destroy(struct flow_divert_group *group)
4244 {
4245 lck_rw_lock_exclusive(&group->lck);
4246
4247 FDLOG(LOG_NOTICE, &nil_pcb, "Destroying group %u", group->ctl_unit);
4248
4249 if (group->token_key != NULL) {
4250 memset(group->token_key, 0, group->token_key_size);
4251 kfree_data(group->token_key, group->token_key_size);
4252 group->token_key = NULL;
4253 group->token_key_size = 0;
4254 }
4255
4256 /* Re-set the current trie */
4257 if (group->signing_id_trie.memory != NULL) {
4258 kfree_data_addr(group->signing_id_trie.memory);
4259 }
4260 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
4261 group->signing_id_trie.root = NULL_TRIE_IDX;
4262
4263 lck_rw_done(&group->lck);
4264
4265 zfree(flow_divert_group_zone, group);
4266 }
4267
4268 static errno_t
flow_divert_kctl_connect(kern_ctl_ref kctlref __unused,struct sockaddr_ctl * sac,void ** unitinfo)4269 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
4270 {
4271 struct flow_divert_group *new_group = NULL;
4272 int error = 0;
4273
4274 if (sac->sc_unit >= GROUP_COUNT_MAX) {
4275 error = EINVAL;
4276 goto done;
4277 }
4278
4279 *unitinfo = NULL;
4280
4281 new_group = zalloc_flags(flow_divert_group_zone, Z_WAITOK | Z_ZERO);
4282 lck_rw_init(&new_group->lck, &flow_divert_mtx_grp, &flow_divert_mtx_attr);
4283 RB_INIT(&new_group->pcb_tree);
4284 new_group->ctl_unit = sac->sc_unit;
4285 MBUFQ_INIT(&new_group->send_queue);
4286 new_group->signing_id_trie.root = NULL_TRIE_IDX;
4287 new_group->ref_count = 1;
4288
4289 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4290
4291 if (g_flow_divert_groups == NULL) {
4292 g_flow_divert_groups = kalloc_type(struct flow_divert_group *,
4293 GROUP_COUNT_MAX, Z_WAITOK | Z_ZERO | Z_NOFAIL);
4294 }
4295
4296 if (g_flow_divert_groups[sac->sc_unit] != NULL) {
4297 error = EALREADY;
4298 } else {
4299 g_flow_divert_groups[sac->sc_unit] = new_group;
4300 g_active_group_count++;
4301 }
4302
4303 lck_rw_done(&g_flow_divert_group_lck);
4304
4305 done:
4306 if (error == 0) {
4307 *unitinfo = new_group;
4308 } else if (new_group != NULL) {
4309 zfree(flow_divert_group_zone, new_group);
4310 }
4311 return error;
4312 }
4313
4314 static errno_t
flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused,uint32_t unit,void * unitinfo)4315 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
4316 {
4317 struct flow_divert_group *group = NULL;
4318 errno_t error = 0;
4319
4320 if (unit >= GROUP_COUNT_MAX) {
4321 return EINVAL;
4322 }
4323
4324 if (unitinfo == NULL) {
4325 return 0;
4326 }
4327
4328 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
4329
4330 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4331
4332 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
4333 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
4334 g_flow_divert_groups, g_active_group_count);
4335 }
4336
4337 group = g_flow_divert_groups[unit];
4338
4339 if (group != (struct flow_divert_group *)unitinfo) {
4340 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
4341 }
4342
4343 g_flow_divert_groups[unit] = NULL;
4344 g_active_group_count--;
4345
4346 if (g_active_group_count == 0) {
4347 kfree_type(struct flow_divert_group *,
4348 GROUP_COUNT_MAX, g_flow_divert_groups);
4349 g_flow_divert_groups = NULL;
4350 }
4351
4352 lck_rw_done(&g_flow_divert_group_lck);
4353
4354 if (group != NULL) {
4355 flow_divert_close_all(group);
4356 FDGRP_RELEASE(group);
4357 } else {
4358 error = EINVAL;
4359 }
4360
4361 return error;
4362 }
4363
4364 static errno_t
flow_divert_kctl_send(__unused kern_ctl_ref kctlref,uint32_t unit,__unused void * unitinfo,mbuf_t m,__unused int flags)4365 flow_divert_kctl_send(__unused kern_ctl_ref kctlref, uint32_t unit, __unused void *unitinfo, mbuf_t m, __unused int flags)
4366 {
4367 errno_t error = 0;
4368 struct flow_divert_group *group = flow_divert_group_lookup(unit, NULL);
4369 if (group != NULL) {
4370 error = flow_divert_input(m, group);
4371 FDGRP_RELEASE(group);
4372 } else {
4373 error = ENOENT;
4374 }
4375 return error;
4376 }
4377
4378 static void
flow_divert_kctl_rcvd(__unused kern_ctl_ref kctlref,uint32_t unit,__unused void * unitinfo,__unused int flags)4379 flow_divert_kctl_rcvd(__unused kern_ctl_ref kctlref, uint32_t unit, __unused void *unitinfo, __unused int flags)
4380 {
4381 struct flow_divert_group *group = flow_divert_group_lookup(unit, NULL);
4382 if (group == NULL) {
4383 return;
4384 }
4385
4386 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
4387 struct flow_divert_pcb *fd_cb;
4388 SLIST_HEAD(, flow_divert_pcb) tmp_list;
4389
4390 lck_rw_lock_exclusive(&group->lck);
4391
4392 while (!MBUFQ_EMPTY(&group->send_queue)) {
4393 mbuf_t next_packet;
4394 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
4395 next_packet = MBUFQ_FIRST(&group->send_queue);
4396 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
4397 if (error) {
4398 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
4399 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
4400 lck_rw_done(&group->lck);
4401 return;
4402 }
4403 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
4404 }
4405
4406 SLIST_INIT(&tmp_list);
4407
4408 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
4409 FDRETAIN(fd_cb);
4410 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
4411 }
4412
4413 lck_rw_done(&group->lck);
4414
4415 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
4416 FDLOCK(fd_cb);
4417 if (fd_cb->so != NULL) {
4418 socket_lock(fd_cb->so, 0);
4419 if (fd_cb->group != NULL) {
4420 flow_divert_send_buffered_data(fd_cb, FALSE);
4421 }
4422 socket_unlock(fd_cb->so, 0);
4423 }
4424 FDUNLOCK(fd_cb);
4425 FDRELEASE(fd_cb);
4426 }
4427 }
4428
4429 FDGRP_RELEASE(group);
4430 }
4431
4432 static int
flow_divert_kctl_init(void)4433 flow_divert_kctl_init(void)
4434 {
4435 struct kern_ctl_reg ctl_reg;
4436 int result;
4437
4438 memset(&ctl_reg, 0, sizeof(ctl_reg));
4439
4440 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
4441 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
4442 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
4443 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
4444
4445 ctl_reg.ctl_connect = flow_divert_kctl_connect;
4446 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
4447 ctl_reg.ctl_send = flow_divert_kctl_send;
4448 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
4449
4450 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
4451
4452 if (result) {
4453 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
4454 return result;
4455 }
4456
4457 return 0;
4458 }
4459
4460 void
flow_divert_init(void)4461 flow_divert_init(void)
4462 {
4463 memset(&nil_pcb, 0, sizeof(nil_pcb));
4464 nil_pcb.log_level = LOG_NOTICE;
4465
4466 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4467
4468 VERIFY(g_tcp_protosw != NULL);
4469
4470 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4471 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4472
4473 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4474 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
4475 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4476 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4477 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4478 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4479 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
4480 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
4481
4482 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4483 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4484
4485 /*
4486 * Socket filters shouldn't attach/detach to/from this protosw
4487 * since pr_protosw is to be used instead, which points to the
4488 * real protocol; if they do, it is a bug and we should panic.
4489 */
4490 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4491 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4492 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4493 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4494
4495 /* UDP */
4496 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4497 VERIFY(g_udp_protosw != NULL);
4498
4499 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4500 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4501
4502 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4503 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
4504 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4505 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4506 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4507 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4508 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4509 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4510 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4511 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4512
4513 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4514 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4515
4516 /*
4517 * Socket filters shouldn't attach/detach to/from this protosw
4518 * since pr_protosw is to be used instead, which points to the
4519 * real protocol; if they do, it is a bug and we should panic.
4520 */
4521 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4522 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4523 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4524 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4525
4526 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4527
4528 VERIFY(g_tcp6_protosw != NULL);
4529
4530 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4531 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4532
4533 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4534 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
4535 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4536 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4537 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4538 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4539 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
4540 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
4541
4542 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4543 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4544 /*
4545 * Socket filters shouldn't attach/detach to/from this protosw
4546 * since pr_protosw is to be used instead, which points to the
4547 * real protocol; if they do, it is a bug and we should panic.
4548 */
4549 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4550 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4551 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4552 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4553
4554 /* UDP6 */
4555 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4556
4557 VERIFY(g_udp6_protosw != NULL);
4558
4559 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4560 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4561
4562 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4563 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4564 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4565 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4566 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4567 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4568 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4569 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4570 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4571 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4572
4573 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4574 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4575 /*
4576 * Socket filters shouldn't attach/detach to/from this protosw
4577 * since pr_protosw is to be used instead, which points to the
4578 * real protocol; if they do, it is a bug and we should panic.
4579 */
4580 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4581 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4582 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4583 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4584
4585 g_init_result = flow_divert_kctl_init();
4586 if (g_init_result) {
4587 goto done;
4588 }
4589
4590 done:
4591 if (g_init_result != 0) {
4592 if (g_flow_divert_kctl_ref != NULL) {
4593 ctl_deregister(g_flow_divert_kctl_ref);
4594 g_flow_divert_kctl_ref = NULL;
4595 }
4596 }
4597 }
4598