Lines Matching refs:fd_cb
227 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_lookup() local
232 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item); in flow_divert_pcb_lookup()
233 FDRETAIN(fd_cb); in flow_divert_pcb_lookup()
236 return fd_cb; in flow_divert_pcb_lookup()
240 flow_divert_group_lookup(uint32_t ctl_unit, struct flow_divert_pcb *fd_cb) in flow_divert_group_lookup() argument
245 if (fd_cb != NULL) { in flow_divert_group_lookup()
246 FDLOG0(LOG_ERR, fd_cb, "No active groups, flow divert cannot be used for this socket"); in flow_divert_group_lookup()
249 FDLOG(LOG_ERR, fd_cb, "Cannot lookup group with invalid control unit (%u)", ctl_unit); in flow_divert_group_lookup()
253 if (fd_cb != NULL) { in flow_divert_group_lookup()
254 …FDLOG(LOG_ERR, fd_cb, "Group for control unit %u is NULL, flow divert cannot be used for this sock… in flow_divert_group_lookup()
265 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, struct flow_divert_group *group) in flow_divert_pcb_insert() argument
270 if (NULL == RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb)) { in flow_divert_pcb_insert()
271 fd_cb->group = group; in flow_divert_pcb_insert()
272 fd_cb->control_group_unit = group->ctl_unit; in flow_divert_pcb_insert()
273 FDRETAIN(fd_cb); /* The group now has a reference */ in flow_divert_pcb_insert()
275 …FDLOG(LOG_ERR, fd_cb, "Group %u already contains a PCB with hash %u", group->ctl_unit, fd_cb->hash… in flow_divert_pcb_insert()
279 FDLOG(LOG_ERR, fd_cb, "Group %u is defunct, cannot insert", group->ctl_unit); in flow_divert_pcb_insert()
287 flow_divert_add_to_group(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit) in flow_divert_add_to_group() argument
295 group = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_add_to_group()
312 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed); in flow_divert_add_to_group()
321 if (NULL != RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb)) { in flow_divert_add_to_group()
330 error = flow_divert_pcb_insert(fd_cb, group); in flow_divert_add_to_group()
335 FDLOG0(LOG_ERR, fd_cb, "Failed to create a unique hash"); in flow_divert_add_to_group()
336 fd_cb->hash = 0; in flow_divert_add_to_group()
359 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_destroy() argument
361 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u", in flow_divert_pcb_destroy()
362 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received); in flow_divert_pcb_destroy()
364 if (fd_cb->connect_token != NULL) { in flow_divert_pcb_destroy()
365 mbuf_freem(fd_cb->connect_token); in flow_divert_pcb_destroy()
367 if (fd_cb->connect_packet != NULL) { in flow_divert_pcb_destroy()
368 mbuf_freem(fd_cb->connect_packet); in flow_divert_pcb_destroy()
370 if (fd_cb->app_data != NULL) { in flow_divert_pcb_destroy()
371 kfree_data(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_pcb_destroy()
373 if (fd_cb->original_remote_endpoint != NULL) { in flow_divert_pcb_destroy()
374 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_pcb_destroy()
376 zfree(flow_divert_pcb_zone, fd_cb); in flow_divert_pcb_destroy()
380 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_remove() argument
382 if (fd_cb->group != NULL) { in flow_divert_pcb_remove()
383 struct flow_divert_group *group = fd_cb->group; in flow_divert_pcb_remove()
385 …FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count… in flow_divert_pcb_remove()
386 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb); in flow_divert_pcb_remove()
387 fd_cb->group = NULL; in flow_divert_pcb_remove()
388 FDRELEASE(fd_cb); /* Release the group's reference */ in flow_divert_pcb_remove()
394 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet) in flow_divert_packet_init() argument
401 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_packet_init()
406 hdr.conn_id = htonl(fd_cb->hash); in flow_divert_packet_init()
411 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error); in flow_divert_packet_init()
609 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send) in flow_divert_add_data_statistics() argument
617 inp = sotoinpcb(fd_cb->so); in flow_divert_add_data_statistics()
644 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_cellular() argument
646 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_cellular()
655 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied"); in flow_divert_check_no_cellular()
663 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_expensive() argument
665 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_expensive()
674 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied"); in flow_divert_check_no_expensive()
682 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_constrained() argument
684 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_constrained()
693 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied"); in flow_divert_check_no_constrained()
701 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, bool tunnel, bool flush_snd) in flow_divert_update_closed_state() argument
704 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED; in flow_divert_update_closed_state()
705 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
706 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_update_closed_state()
709 sbflush(&fd_cb->so->so_snd); in flow_divert_update_closed_state()
714 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED; in flow_divert_update_closed_state()
715 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
716 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_update_closed_state()
995 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_… in flow_divert_add_proc_info() argument
1008 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid"); in flow_divert_add_proc_info()
1013 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_proc_info()
1014 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_proc_info()
1016 …uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs… in flow_divert_add_proc_info()
1018 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id); in flow_divert_add_proc_info()
1021 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id); in flow_divert_add_proc_info()
1027 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_proc_info()
1044 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error); in flow_divert_add_proc_info()
1056 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error); in flow_divert_add_proc_info()
1060 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash"); in flow_divert_add_proc_info()
1073 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error); in flow_divert_add_proc_info()
1085 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const … in flow_divert_add_all_proc_info() argument
1097 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid); in flow_divert_add_all_proc_info()
1114 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1115 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_all_proc_info()
1120 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1136 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true); in flow_divert_add_all_proc_info()
1142 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false); in flow_divert_add_all_proc_info()
1165 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet) in flow_divert_send_packet() argument
1169 if (fd_cb->group == NULL) { in flow_divert_send_packet()
1170 FDLOG0(LOG_INFO, fd_cb, "no provider, cannot send packet"); in flow_divert_send_packet()
1171 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, false); in flow_divert_send_packet()
1172 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT)); in flow_divert_send_packet()
1173 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_packet()
1178 fd_cb->so->so_error = (uint16_t)error; in flow_divert_send_packet()
1182 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_send_packet()
1184 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) { in flow_divert_send_packet()
1185 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR); in flow_divert_send_packet()
1194 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) { in flow_divert_send_packet()
1195 lck_rw_lock_exclusive(&fd_cb->group->lck); in flow_divert_send_packet()
1197 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet); in flow_divert_send_packet()
1199 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits); in flow_divert_send_packet()
1202 lck_rw_done(&fd_cb->group->lck); in flow_divert_send_packet()
1227 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket… in flow_divert_create_connect_packet() argument
1241 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet); in flow_divert_create_connect_packet()
1246 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) { in flow_divert_create_connect_packet()
1247 …int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0… in flow_divert_create_connect_packet()
1251 …flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_… in flow_divert_create_connect_packet()
1252 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id); in flow_divert_create_connect_packet()
1257 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet); in flow_divert_create_connect_packet()
1264 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error); in flow_divert_create_connect_packet()
1270 sizeof(fd_cb->so->so_traffic_class), in flow_divert_create_connect_packet()
1271 &fd_cb->so->so_traffic_class); in flow_divert_create_connect_packet()
1276 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_create_connect_packet()
1278 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_create_connect_packet()
1293 if (fd_cb->connect_token != NULL) { in flow_divert_create_connect_packet()
1294 unsigned int token_len = m_length(fd_cb->connect_token); in flow_divert_create_connect_packet()
1295 mbuf_concatenate(connect_packet, fd_cb->connect_token); in flow_divert_create_connect_packet()
1297 fd_cb->connect_token = NULL; in flow_divert_create_connect_packet()
1307 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_create_connect_packet()
1308 …_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->… in flow_divert_create_connect_packet()
1377 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_packet() argument
1380 mbuf_t connect_packet = fd_cb->connect_packet; in flow_divert_send_connect_packet()
1386 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet"); in flow_divert_send_connect_packet()
1390 error = flow_divert_send_packet(fd_cb, connect_packet); in flow_divert_send_connect_packet()
1395 fd_cb->connect_packet = saved_connect_packet; in flow_divert_send_connect_packet()
1409 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_result() argument
1415 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet); in flow_divert_send_connect_result()
1417 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error); in flow_divert_send_connect_result()
1421 rbuff_space = fd_cb->so->so_rcv.sb_hiwat; in flow_divert_send_connect_result()
1434 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_send_connect_result()
1435 …t_packet_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->… in flow_divert_send_connect_result()
1441 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_connect_result()
1455 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how) in flow_divert_send_close() argument
1461 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet); in flow_divert_send_close()
1463 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error); in flow_divert_send_close()
1469 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error); in flow_divert_send_close()
1476 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error); in flow_divert_send_close()
1480 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_close()
1494 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb) in flow_divert_tunnel_how_closed() argument
1496 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == in flow_divert_tunnel_how_closed()
1499 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) { in flow_divert_tunnel_how_closed()
1501 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) { in flow_divert_tunnel_how_closed()
1513 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb) in flow_divert_send_close_if_needed() argument
1518 if (fd_cb->so->so_snd.sb_cc == 0) { in flow_divert_send_close_if_needed()
1519 …if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_… in flow_divert_send_close_if_needed()
1523 …if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRIT… in flow_divert_send_close_if_needed()
1534 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how); in flow_divert_send_close_if_needed()
1535 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) { in flow_divert_send_close_if_needed()
1538 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_send_close_if_needed()
1541 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_send_close_if_needed()
1546 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) { in flow_divert_send_close_if_needed()
1547 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT)); in flow_divert_send_close_if_needed()
1552 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len) in flow_divert_send_data_packet() argument
1558 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_data_packet()
1560 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_data_packet()
1571 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_data_packet()
1573 fd_cb->bytes_sent += data_len; in flow_divert_send_data_packet()
1574 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_data_packet()
1591 flow_divert_send_datagram_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struc… in flow_divert_send_datagram_packet() argument
1597 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_datagram_packet()
1599 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_datagram_packet()
1606 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error); in flow_divert_send_datagram_packet()
1613 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_IS_FRAGMENT) failed: %d", err… in flow_divert_send_datagram_packet()
1620 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_DATAGRAM_SIZE) failed: %d", e… in flow_divert_send_datagram_packet()
1631 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_datagram_packet()
1633 fd_cb->bytes_sent += data_len; in flow_divert_send_datagram_packet()
1634 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_datagram_packet()
1651 flow_divert_send_fragmented_datagram(struct flow_divert_pcb *fd_cb, mbuf_t datagram, size_t datagra… in flow_divert_send_fragmented_datagram() argument
1671 …error = flow_divert_send_datagram_packet(fd_cb, next_data, to_send, (first ? toaddr : NULL), TRUE,… in flow_divert_send_fragmented_datagram()
1693 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) in flow_divert_send_buffered_data() argument
1700 to_send = fd_cb->so->so_snd.sb_cc; in flow_divert_send_buffered_data()
1701 buffer = fd_cb->so->so_snd.sb_mb; in flow_divert_send_buffered_data()
1704 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send); in flow_divert_send_buffered_data()
1709 if (!force && (to_send > fd_cb->send_window)) { in flow_divert_send_buffered_data()
1710 to_send = fd_cb->send_window; in flow_divert_send_buffered_data()
1713 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_buffered_data()
1725 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1729 error = flow_divert_send_data_packet(fd_cb, data, data_len); in flow_divert_send_buffered_data()
1739 sbdrop(&fd_cb->so->so_snd, (int)sent); in flow_divert_send_buffered_data()
1740 sowwakeup(fd_cb->so); in flow_divert_send_buffered_data()
1741 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_buffered_data()
1760 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain."); in flow_divert_send_buffered_data()
1766 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); in flow_divert_send_buffered_data()
1769 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1776 error = flow_divert_send_datagram_packet(fd_cb, data, data_len, toaddr, FALSE, 0); in flow_divert_send_buffered_data()
1778 error = flow_divert_send_fragmented_datagram(fd_cb, data, data_len, toaddr); in flow_divert_send_buffered_data()
1790 (void) sbdroprecord(&(fd_cb->so->so_snd)); in flow_divert_send_buffered_data()
1795 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent); in flow_divert_send_buffered_data()
1796 if (fd_cb->send_window >= sent) { in flow_divert_send_buffered_data()
1797 fd_cb->send_window -= sent; in flow_divert_send_buffered_data()
1799 fd_cb->send_window = 0; in flow_divert_send_buffered_data()
1805 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr) in flow_divert_send_app_data() argument
1810 if (to_send > fd_cb->send_window) { in flow_divert_send_app_data()
1811 to_send = fd_cb->send_window; in flow_divert_send_app_data()
1814 if (fd_cb->so->so_snd.sb_cc > 0) { in flow_divert_send_app_data()
1818 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_app_data()
1836 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_send_app_data()
1844 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len); in flow_divert_send_app_data()
1853 fd_cb->send_window -= sent; in flow_divert_send_app_data()
1858 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1859 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) { in flow_divert_send_app_data()
1860 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %… in flow_divert_send_app_data()
1861 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1870 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1871 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) { in flow_divert_send_app_data()
1872 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_wind… in flow_divert_send_app_data()
1873 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1880 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_app_data()
1885 send_dgram_error = flow_divert_send_datagram_packet(fd_cb, data, data_size, toaddr, FALSE, 0); in flow_divert_send_app_data()
1887 send_dgram_error = flow_divert_send_fragmented_datagram(fd_cb, data, data_size, toaddr); in flow_divert_send_app_data()
1891 …FDLOG(LOG_NOTICE, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = … in flow_divert_send_app_data()
1893 if (data_size >= fd_cb->send_window) { in flow_divert_send_app_data()
1894 fd_cb->send_window = 0; in flow_divert_send_app_data()
1896 fd_cb->send_window -= data_size; in flow_divert_send_app_data()
1904 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1907 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &append_error)) { in flow_divert_send_app_data()
1908 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1910 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, append_error); in flow_divert_send_app_data()
1913 if (!sbappendrecord(&fd_cb->so->so_snd, data)) { in flow_divert_send_app_data()
1914 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1916 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1920 …FDLOG(LOG_ERR, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = %lu… in flow_divert_send_app_data()
1930 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb) in flow_divert_send_read_notification() argument
1935 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet); in flow_divert_send_read_notification()
1937 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error); in flow_divert_send_read_notification()
1941 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_read_notification()
1955 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class) in flow_divert_send_traffic_class_update() argument
1960 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet); in flow_divert_send_traffic_class_update()
1962 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error); in flow_divert_send_traffic_class_update()
1968 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error); in flow_divert_send_traffic_class_update()
1972 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_traffic_class_update()
1986 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint) in flow_divert_set_local_endpoint() argument
1988 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_local_endpoint()
1991 …if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR… in flow_divert_set_local_endpoint()
1992 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2001 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) { in flow_divert_set_local_endpoint()
2002 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2012 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint) in flow_divert_set_remote_endpoint() argument
2014 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_remote_endpoint()
2063 flow_divert_try_next_group(struct flow_divert_pcb *fd_cb) in flow_divert_try_next_group() argument
2066 uint32_t policy_control_unit = fd_cb->policy_control_unit; in flow_divert_try_next_group()
2068 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2073 …uint32_t next_ctl_unit = flow_divert_derive_kernel_control_unit(&policy_control_unit, &(fd_cb->agg… in flow_divert_try_next_group()
2075 if (fd_cb->control_group_unit == next_ctl_unit) { in flow_divert_try_next_group()
2076 …FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling fl… in flow_divert_try_next_group()
2082 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert"); in flow_divert_try_next_group()
2087 next_group = flow_divert_group_lookup(next_ctl_unit, fd_cb); in flow_divert_try_next_group()
2089 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit); in flow_divert_try_next_group()
2093 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", fd_cb->control_group_unit, next_ctl_unit); in flow_divert_try_next_group()
2095 error = flow_divert_pcb_insert(fd_cb, next_group); in flow_divert_try_next_group()
2098 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2100 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2104 } while (fd_cb->group == NULL); in flow_divert_try_next_group()
2106 if (fd_cb->group == NULL) { in flow_divert_try_next_group()
2110 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_try_next_group()
2112 …FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", fd_cb->… in flow_divert_try_next_group()
2113 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2121 flow_divert_disable(struct flow_divert_pcb *fd_cb) in flow_divert_disable() argument
2127 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint; in flow_divert_disable()
2128 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT); in flow_divert_disable()
2131 so = fd_cb->so; in flow_divert_disable()
2136 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert"); in flow_divert_disable()
2140 inp->inp_vflag = fd_cb->original_vflag; in flow_divert_disable()
2147 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) { in flow_divert_disable()
2152 inp->inp_last_outifp = fd_cb->original_last_outifp; in flow_divert_disable()
2153 inp->in6p_last_outifp = fd_cb->original_last_outifp6; in flow_divert_disable()
2159 fd_cb->so = NULL; in flow_divert_disable()
2161 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_disable()
2175 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error); in flow_divert_disable()
2193 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer"); in flow_divert_disable()
2211 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued TCP data using the socket's original protocol: %d", e… in flow_divert_disable()
2263 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer"); in flow_divert_disable()
2268 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data"); in flow_divert_disable()
2274 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header"); in flow_divert_disable()
2299 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued UDP data using the socket's original protocol: %d", e… in flow_divert_disable()
2315 flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address) in flow_divert_scope() argument
2323 so = fd_cb->so; in flow_divert_scope()
2349 …FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error); in flow_divert_scope()
2353 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) { in flow_divert_scope()
2357 …error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr)… in flow_divert_scope()
2360 …error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), I… in flow_divert_scope()
2364 …FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr return… in flow_divert_scope()
2392 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) in flow_divert_handle_connect_result() argument
2408 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error); in flow_divert_handle_connect_result()
2413 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error); in flow_divert_handle_connect_result()
2417 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error); in flow_divert_handle_connect_result()
2423 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result"); in flow_divert_handle_connect_result()
2428 FDLOG0(LOG_INFO, fd_cb, "No local address provided"); in flow_divert_handle_connect_result()
2433 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_connect_result()
2438 FDLOG0(LOG_INFO, fd_cb, "No output if index provided"); in flow_divert_handle_connect_result()
2443 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result"); in flow_divert_handle_connect_result()
2448 FDLOCK(fd_cb); in flow_divert_handle_connect_result()
2449 if (fd_cb->so != NULL) { in flow_divert_handle_connect_result()
2451 struct socket *so = fd_cb->so; in flow_divert_handle_connect_result()
2457 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result"); in flow_divert_handle_connect_result()
2462 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result"); in flow_divert_handle_connect_result()
2477 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2480 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port; in flow_divert_handle_connect_result()
2486 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2489 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port; in flow_divert_handle_connect_result()
2494 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid); in flow_divert_handle_connect_result()
2495 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_connect_result()
2503 flow_divert_set_remote_endpoint(fd_cb, SA(&remote_endpoint)); in flow_divert_handle_connect_result()
2512 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length); in flow_divert_handle_connect_result()
2513 if (fd_cb->app_data != NULL) { in flow_divert_handle_connect_result()
2514 kfree_data(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_connect_result()
2516 fd_cb->app_data = app_data; in flow_divert_handle_connect_result()
2517 fd_cb->app_data_length = app_data_length; in flow_divert_handle_connect_result()
2519 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet"… in flow_divert_handle_connect_result()
2523 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_connect_result()
2531 if (fd_cb->group == NULL) { in flow_divert_handle_connect_result()
2542 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit); in flow_divert_handle_connect_result()
2547 grp = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_handle_connect_result()
2553 flow_divert_pcb_remove(fd_cb); in flow_divert_handle_connect_result()
2554 insert_error = flow_divert_pcb_insert(fd_cb, grp); in flow_divert_handle_connect_result()
2563 fd_cb->send_window = ntohl(send_window); in flow_divert_handle_connect_result()
2567 FDLOG0(LOG_INFO, fd_cb, "sending connect result"); in flow_divert_handle_connect_result()
2568 error = flow_divert_send_connect_result(fd_cb); in flow_divert_handle_connect_result()
2572 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) { in flow_divert_handle_connect_result()
2573 error = flow_divert_try_next_group(fd_cb); in flow_divert_handle_connect_result()
2574 if (error && fd_cb->policy_control_unit == 0) { in flow_divert_handle_connect_result()
2575 flow_divert_disable(fd_cb); in flow_divert_handle_connect_result()
2583 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_handle_connect_result()
2585 flow_divert_send_close_if_needed(fd_cb); in flow_divert_handle_connect_result()
2587 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_handle_connect_result()
2590 flow_divert_disconnect_socket(so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT)); in flow_divert_handle_connect_result()
2605 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_connect_result()
2610 if (fd_cb->connect_packet != NULL) { in flow_divert_handle_connect_result()
2611 mbuf_freem(fd_cb->connect_packet); in flow_divert_handle_connect_result()
2612 fd_cb->connect_packet = NULL; in flow_divert_handle_connect_result()
2616 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_handle_connect_result()
2620 FDUNLOCK(fd_cb); in flow_divert_handle_connect_result()
2624 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) in flow_divert_handle_close() argument
2632 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error); in flow_divert_handle_close()
2638 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error); in flow_divert_handle_close()
2644 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how); in flow_divert_handle_close()
2646 FDLOCK(fd_cb); in flow_divert_handle_close()
2647 if (fd_cb->so != NULL) { in flow_divert_handle_close()
2648 …bool is_connected = (SOCK_TYPE(fd_cb->so) == SOCK_STREAM || !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_… in flow_divert_handle_close()
2649 socket_lock(fd_cb->so, 0); in flow_divert_handle_close()
2651 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_close()
2652 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider"); in flow_divert_handle_close()
2656 fd_cb->so->so_error = (uint16_t)ntohl(close_error); in flow_divert_handle_close()
2658 flow_divert_update_closed_state(fd_cb, how, true, true); in flow_divert_handle_close()
2661 how = flow_divert_tunnel_how_closed(fd_cb); in flow_divert_handle_close()
2663 flow_divert_disconnect_socket(fd_cb->so, is_connected); in flow_divert_handle_close()
2665 socantrcvmore(fd_cb->so); in flow_divert_handle_close()
2667 socantsendmore(fd_cb->so); in flow_divert_handle_close()
2670 socket_unlock(fd_cb->so, 0); in flow_divert_handle_close()
2672 FDUNLOCK(fd_cb); in flow_divert_handle_close()
2676 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb) in flow_divert_create_control_mbuf() argument
2678 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_create_control_mbuf()
2683 fd_cb->local_endpoint.sa.sa_family == AF_INET && in flow_divert_create_control_mbuf()
2685 …return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_… in flow_divert_create_control_mbuf()
2687 fd_cb->local_endpoint.sa.sa_family == AF_INET6 && in flow_divert_create_control_mbuf()
2691 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr; in flow_divert_create_control_mbuf()
2699 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset) in flow_divert_handle_data() argument
2703 FDLOCK(fd_cb); in flow_divert_handle_data()
2704 if (fd_cb->so != NULL) { in flow_divert_handle_data()
2712 socket_lock(fd_cb->so, 0); in flow_divert_handle_data()
2714 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_data()
2715 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data"); in flow_divert_handle_data()
2719 if (sbspace(&fd_cb->so->so_rcv) == 0) { in flow_divert_handle_data()
2721 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_handle_data()
2722 …FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some d… in flow_divert_handle_data()
2726 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_handle_data()
2733 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_data()
2743 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid"); in flow_divert_handle_data()
2751 if (fd_cb->so->so_state & SS_CANTRCVMORE) { in flow_divert_handle_data()
2752 …FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size… in flow_divert_handle_data()
2756 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) { in flow_divert_handle_data()
2757 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so)); in flow_divert_handle_data()
2761 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size); in flow_divert_handle_data()
2765 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_handle_data()
2769 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_handle_data()
2770 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0); in flow_divert_handle_data()
2779 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) { in flow_divert_handle_data()
2780 error = in6_mapped_peeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2782 error = in_getpeeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2786 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address."); in flow_divert_handle_data()
2789 mctl = flow_divert_create_control_mbuf(fd_cb); in flow_divert_handle_data()
2791 appended = sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error); in flow_divert_handle_data()
2795 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error); in flow_divert_handle_data()
2802 fd_cb->bytes_received += data_size; in flow_divert_handle_data()
2803 flow_divert_add_data_statistics(fd_cb, data_size, FALSE); in flow_divert_handle_data()
2807 sorwakeup(fd_cb->so); in flow_divert_handle_data()
2810 socket_unlock(fd_cb->so, 0); in flow_divert_handle_data()
2812 FDUNLOCK(fd_cb); in flow_divert_handle_data()
2818 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) in flow_divert_handle_read_notification() argument
2825 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error); in flow_divert_handle_read_notification()
2829 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count)); in flow_divert_handle_read_notification()
2831 FDLOCK(fd_cb); in flow_divert_handle_read_notification()
2832 if (fd_cb->so != NULL) { in flow_divert_handle_read_notification()
2833 socket_lock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2835 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_read_notification()
2836 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification"); in flow_divert_handle_read_notification()
2840 fd_cb->send_window += ntohl(read_count); in flow_divert_handle_read_notification()
2841 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_read_notification()
2843 socket_unlock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2845 FDUNLOCK(fd_cb); in flow_divert_handle_read_notification()
2906 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) in flow_divert_handle_properties_update() argument
2912 FDLOG0(LOG_INFO, fd_cb, "received a properties update"); in flow_divert_handle_properties_update()
2916 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update"); in flow_divert_handle_properties_update()
2921 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update"); in flow_divert_handle_properties_update()
2924 FDLOCK(fd_cb); in flow_divert_handle_properties_update()
2925 if (fd_cb->so != NULL) { in flow_divert_handle_properties_update()
2926 socket_lock(fd_cb->so, 0); in flow_divert_handle_properties_update()
2928 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_properties_update()
2929 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update"); in flow_divert_handle_properties_update()
2934 flow_divert_scope(fd_cb, out_if_index, true); in flow_divert_handle_properties_update()
2935 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_properties_update()
2944 if (fd_cb->app_data != NULL) { in flow_divert_handle_properties_update()
2945 kfree_data(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_properties_update()
2947 fd_cb->app_data = app_data; in flow_divert_handle_properties_update()
2948 fd_cb->app_data_length = app_data_length; in flow_divert_handle_properties_update()
2950 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update pack… in flow_divert_handle_properties_update()
2954 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_properties_update()
2958 socket_unlock(fd_cb->so, 0); in flow_divert_handle_properties_update()
2960 FDUNLOCK(fd_cb); in flow_divert_handle_properties_update()
3134 struct flow_divert_pcb *fd_cb; in flow_divert_input() local
3166 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */ in flow_divert_input()
3167 if (fd_cb == NULL) { in flow_divert_input()
3176 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3179 flow_divert_handle_close(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3182 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3185 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3188 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3191 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type); in flow_divert_input()
3195 FDRELEASE(fd_cb); in flow_divert_input()
3205 struct flow_divert_pcb *fd_cb; in flow_divert_close_all() local
3214 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_close_all()
3215 FDRETAIN(fd_cb); in flow_divert_close_all()
3216 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_close_all()
3224 fd_cb = SLIST_FIRST(&tmp_list); in flow_divert_close_all()
3225 FDLOCK(fd_cb); in flow_divert_close_all()
3227 if (fd_cb->so != NULL) { in flow_divert_close_all()
3228 socket_lock(fd_cb->so, 0); in flow_divert_close_all()
3229 flow_divert_pcb_remove(fd_cb); in flow_divert_close_all()
3230 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_close_all()
3231 fd_cb->so->so_error = ECONNABORTED; in flow_divert_close_all()
3232 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT)); in flow_divert_close_all()
3233 socket_unlock(fd_cb->so, 0); in flow_divert_close_all()
3235 FDUNLOCK(fd_cb); in flow_divert_close_all()
3236 FDRELEASE(fd_cb); in flow_divert_close_all()
3243 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_detach() local
3252 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count); in flow_divert_detach()
3254 if (fd_cb->group != NULL) { in flow_divert_detach()
3256 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_detach()
3258 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_detach()
3259 flow_divert_send_close_if_needed(fd_cb); in flow_divert_detach()
3261 flow_divert_pcb_remove(fd_cb); in flow_divert_detach()
3265 FDLOCK(fd_cb); in flow_divert_detach()
3266 fd_cb->so = NULL; in flow_divert_detach()
3267 FDUNLOCK(fd_cb); in flow_divert_detach()
3270 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_detach()
3276 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_close() local
3282 FDLOG0(LOG_INFO, fd_cb, "Closing"); in flow_divert_close()
3289 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_close()
3290 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_close()
3291 flow_divert_send_close_if_needed(fd_cb); in flow_divert_close()
3294 flow_divert_pcb_remove(fd_cb); in flow_divert_close()
3313 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_shutdown() local
3319 FDLOG0(LOG_INFO, fd_cb, "Can't send more"); in flow_divert_shutdown()
3323 flow_divert_update_closed_state(fd_cb, SHUT_WR, false, true); in flow_divert_shutdown()
3324 flow_divert_send_close_if_needed(fd_cb); in flow_divert_shutdown()
3332 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_rcvd() local
3340 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space); in flow_divert_rcvd()
3341 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) && in flow_divert_rcvd()
3343 flow_divert_send_read_notification(fd_cb) == 0) { in flow_divert_rcvd()
3344 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification"); in flow_divert_rcvd()
3345 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_rcvd()
3481 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_ctloutput() local
3488 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_ctloutput()
3489 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class); in flow_divert_ctloutput()
3504 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out_internal() local
3515 if (fd_cb->group == NULL) { in flow_divert_connect_out_internal()
3533 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_connect_out_internal()
3538 FDLOG0(LOG_INFO, fd_cb, "Connecting"); in flow_divert_connect_out_internal()
3540 if (fd_cb->connect_packet == NULL) { in flow_divert_connect_out_internal()
3545 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet"); in flow_divert_connect_out_internal()
3551 FDLOG0(LOG_ERR, fd_cb, "Destination address is not valid when creating connect packet"); in flow_divert_connect_out_internal()
3556 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0); in flow_divert_connect_out_internal()
3557 if (fd_cb->original_remote_endpoint == NULL) { in flow_divert_connect_out_internal()
3558 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint"); in flow_divert_connect_out_internal()
3562 fd_cb->original_vflag = inp->inp_vflag; in flow_divert_connect_out_internal()
3563 fd_cb->original_last_outifp = inp->inp_last_outifp; in flow_divert_connect_out_internal()
3564 fd_cb->original_last_outifp6 = inp->in6p_last_outifp; in flow_divert_connect_out_internal()
3589 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6); in flow_divert_connect_out_internal()
3590 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6; in flow_divert_connect_out_internal()
3591 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport; in flow_divert_connect_out_internal()
3592 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp); in flow_divert_connect_out_internal()
3594 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error); in flow_divert_connect_out_internal()
3595 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || IN6_IS_ADDR_UNSPECIFIED(&(satosin6(to)->s… in flow_divert_connect_out_internal()
3606 if (IN6_IS_SCOPE_EMBED(&(fd_cb->local_endpoint.sin6.sin6_addr)) && in flow_divert_connect_out_internal()
3608 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] != 0) { in flow_divert_connect_out_internal()
3609 …fd_cb->local_endpoint.sin6.sin6_scope_id = ntohs(fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1]… in flow_divert_connect_out_internal()
3610 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] = 0; in flow_divert_connect_out_internal()
3622 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in); in flow_divert_connect_out_internal()
3623 fd_cb->local_endpoint.sin.sin_family = AF_INET; in flow_divert_connect_out_internal()
3624 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport; in flow_divert_connect_out_internal()
3625 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0); in flow_divert_connect_out_internal()
3627 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error); in flow_divert_connect_out_internal()
3628 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || satosin(to)->sin_addr.s_addr == INADDR_AN… in flow_divert_connect_out_internal()
3639 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family); in flow_divert_connect_out_internal()
3642 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_connect_out_internal()
3643 flow_divert_check_no_expensive(fd_cb) || in flow_divert_connect_out_internal()
3644 flow_divert_check_no_constrained(fd_cb); in flow_divert_connect_out_internal()
3653 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR; in flow_divert_connect_out_internal()
3656 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet); in flow_divert_connect_out_internal()
3662 flow_divert_set_remote_endpoint(fd_cb, to); in flow_divert_connect_out_internal()
3663 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_connect_out_internal()
3667 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT; in flow_divert_connect_out_internal()
3671 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive"); in flow_divert_connect_out_internal()
3675 fd_cb->connect_packet = connect_packet; in flow_divert_connect_out_internal()
3678 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet"); in flow_divert_connect_out_internal()
3682 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_connect_out_internal()
3687 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_connect_out_internal()
3690 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) { in flow_divert_connect_out_internal()
3707 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out() local
3708 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error); in flow_divert_connect_out()
3731 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connectx_out_common() local
3732 if (fd_cb != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_TOKEN) && in flow_divert_connectx_out_common()
3801 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_data_out() local
3818 if ((fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) && SOCK_TYPE(so) == SOCK_DGRAM) { in flow_divert_data_out()
3820 FDLOG0(LOG_INFO, fd_cb, "provider is no longer accepting writes, cannot send data"); in flow_divert_data_out()
3836 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to); in flow_divert_data_out()
3841 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_data_out()
3842 FDLOG0(LOG_INFO, fd_cb, "implicit connect"); in flow_divert_data_out()
3849 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_data_out()
3850 flow_divert_check_no_expensive(fd_cb) || in flow_divert_data_out()
3851 flow_divert_check_no_constrained(fd_cb); in flow_divert_data_out()
3857 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data)); in flow_divert_data_out()
3859 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data); in flow_divert_data_out()
3860 error = flow_divert_send_app_data(fd_cb, data, to); in flow_divert_data_out()
3892 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_preconnect() local
3898 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_preconnect()
3899 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet"); in flow_divert_preconnect()
3905 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_preconnect()
3936 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out() local
3945 if (fd_cb == NULL) { in flow_divert_implicit_data_out()
3947 fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out()
3948 if (error != 0 || fd_cb == NULL) { in flow_divert_implicit_data_out()
3969 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_init_internal() local
3978 fd_cb = flow_divert_pcb_create(so); in flow_divert_pcb_init_internal()
3979 if (fd_cb == NULL) { in flow_divert_pcb_init_internal()
3986 FDLOG0(LOG_ERR, fd_cb, "No valid group is available, cannot init flow divert"); in flow_divert_pcb_init_internal()
3991 error = flow_divert_add_to_group(fd_cb, group_unit); in flow_divert_pcb_init_internal()
3993 so->so_fd_pcb = fd_cb; in flow_divert_pcb_init_internal()
3995 fd_cb->control_group_unit = group_unit; in flow_divert_pcb_init_internal()
3996 fd_cb->policy_control_unit = ctl_unit; in flow_divert_pcb_init_internal()
3997 fd_cb->aggregate_unit = agg_unit; in flow_divert_pcb_init_internal()
3999 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4001 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4010 FDLOG0(LOG_INFO, fd_cb, "Created"); in flow_divert_pcb_init_internal()
4012 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error); in flow_divert_pcb_init_internal()
4017 FDRELEASE(fd_cb); in flow_divert_pcb_init_internal()
4120 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4125 fd_cb->log_level = (uint8_t)log_level; in flow_divert_token_set()
4129 fd_cb->connect_token = token; in flow_divert_token_set()
4132 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN; in flow_divert_token_set()
4136 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4137 if (fd_cb != NULL) { in flow_divert_token_set()
4138 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC; in flow_divert_token_set()
4156 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_get() local
4165 if (fd_cb->group == NULL) { in flow_divert_token_get()
4172 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_token_get()
4176 ctl_unit = htonl(fd_cb->group->ctl_unit); in flow_divert_token_get()
4183 …error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb-… in flow_divert_token_get()
4188 if (fd_cb->app_data != NULL) { in flow_divert_token_get()
4189 …_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_d… in flow_divert_token_get()
4195 control_group = flow_divert_group_lookup(fd_cb->control_group_unit, fd_cb); in flow_divert_token_get()
4383 struct flow_divert_pcb *fd_cb; in flow_divert_kctl_rcvd() local
4404 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_kctl_rcvd()
4405 FDRETAIN(fd_cb); in flow_divert_kctl_rcvd()
4406 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_kctl_rcvd()
4411 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { in flow_divert_kctl_rcvd()
4412 FDLOCK(fd_cb); in flow_divert_kctl_rcvd()
4413 if (fd_cb->so != NULL) { in flow_divert_kctl_rcvd()
4414 socket_lock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4415 if (fd_cb->group != NULL) { in flow_divert_kctl_rcvd()
4416 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_kctl_rcvd()
4418 socket_unlock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4420 FDUNLOCK(fd_cb); in flow_divert_kctl_rcvd()
4421 FDRELEASE(fd_cb); in flow_divert_kctl_rcvd()