Lines Matching refs:fd_cb
225 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_lookup() local
230 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item); in flow_divert_pcb_lookup()
231 FDRETAIN(fd_cb); in flow_divert_pcb_lookup()
234 return fd_cb; in flow_divert_pcb_lookup()
238 flow_divert_group_lookup(uint32_t ctl_unit, struct flow_divert_pcb *fd_cb) in flow_divert_group_lookup() argument
243 if (fd_cb != NULL) { in flow_divert_group_lookup()
244 FDLOG0(LOG_ERR, fd_cb, "No active groups, flow divert cannot be used for this socket"); in flow_divert_group_lookup()
247 FDLOG(LOG_ERR, fd_cb, "Cannot lookup group with invalid control unit (%u)", ctl_unit); in flow_divert_group_lookup()
250 if (fd_cb != NULL) { in flow_divert_group_lookup()
251 …FDLOG0(LOG_ERR, fd_cb, "No active non-in-process groups, flow divert cannot be used for this socke… in flow_divert_group_lookup()
256 if (fd_cb != NULL) { in flow_divert_group_lookup()
257 …FDLOG(LOG_ERR, fd_cb, "Group for control unit %u is NULL, flow divert cannot be used for this sock… in flow_divert_group_lookup()
265 if (fd_cb != NULL) { in flow_divert_group_lookup()
266 … FDLOG0(LOG_ERR, fd_cb, "No active in-process groups, flow divert cannot be used for this socket"); in flow_divert_group_lookup()
277 if (fd_cb != NULL) { in flow_divert_group_lookup()
278 …FDLOG(LOG_ERR, fd_cb, "Group for control unit %u not found, flow divert cannot be used for this so… in flow_divert_group_lookup()
280 } else if (fd_cb != NULL && in flow_divert_group_lookup()
281 (fd_cb->so == NULL || in flow_divert_group_lookup()
282 group_cursor->in_process_pid != fd_cb->so->last_pid)) { in flow_divert_group_lookup()
283 FDLOG(LOG_ERR, fd_cb, "Cannot access group for control unit %u, mismatched PID (%u != %u)", in flow_divert_group_lookup()
284 ctl_unit, group_cursor->in_process_pid, fd_cb->so ? fd_cb->so->last_pid : 0); in flow_divert_group_lookup()
296 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, struct flow_divert_group *group) in flow_divert_pcb_insert() argument
301 if (NULL == RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb)) { in flow_divert_pcb_insert()
302 fd_cb->group = group; in flow_divert_pcb_insert()
303 fd_cb->control_group_unit = group->ctl_unit; in flow_divert_pcb_insert()
304 FDRETAIN(fd_cb); /* The group now has a reference */ in flow_divert_pcb_insert()
306 …FDLOG(LOG_ERR, fd_cb, "Group %u already contains a PCB with hash %u", group->ctl_unit, fd_cb->hash… in flow_divert_pcb_insert()
310 FDLOG(LOG_ERR, fd_cb, "Group %u is defunct, cannot insert", group->ctl_unit); in flow_divert_pcb_insert()
318 flow_divert_add_to_group(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit) in flow_divert_add_to_group() argument
326 group = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_add_to_group()
343 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed); in flow_divert_add_to_group()
352 if (NULL != RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb)) { in flow_divert_add_to_group()
361 error = flow_divert_pcb_insert(fd_cb, group); in flow_divert_add_to_group()
366 FDLOG0(LOG_ERR, fd_cb, "Failed to create a unique hash"); in flow_divert_add_to_group()
367 fd_cb->hash = 0; in flow_divert_add_to_group()
390 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_destroy() argument
392 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %llu, tunnel tx %llu, tunnel rx %llu", in flow_divert_pcb_destroy()
393 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received); in flow_divert_pcb_destroy()
395 if (fd_cb->connect_token != NULL) { in flow_divert_pcb_destroy()
396 mbuf_freem(fd_cb->connect_token); in flow_divert_pcb_destroy()
398 if (fd_cb->connect_packet != NULL) { in flow_divert_pcb_destroy()
399 mbuf_freem(fd_cb->connect_packet); in flow_divert_pcb_destroy()
401 if (fd_cb->app_data != NULL) { in flow_divert_pcb_destroy()
402 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_pcb_destroy()
404 if (fd_cb->original_remote_endpoint != NULL) { in flow_divert_pcb_destroy()
405 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_pcb_destroy()
407 zfree(flow_divert_pcb_zone, fd_cb); in flow_divert_pcb_destroy()
411 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_remove() argument
413 if (fd_cb->group != NULL) { in flow_divert_pcb_remove()
414 struct flow_divert_group *group = fd_cb->group; in flow_divert_pcb_remove()
416 …FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count… in flow_divert_pcb_remove()
417 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb); in flow_divert_pcb_remove()
418 fd_cb->group = NULL; in flow_divert_pcb_remove()
419 FDRELEASE(fd_cb); /* Release the group's reference */ in flow_divert_pcb_remove()
425 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_ref_t *packet) in flow_divert_packet_init() argument
432 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_packet_init()
437 hdr.conn_id = htonl(fd_cb->hash); in flow_divert_packet_init()
442 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error); in flow_divert_packet_init()
640 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send) in flow_divert_add_data_statistics() argument
646 inp = sotoinpcb(fd_cb->so); in flow_divert_add_data_statistics()
671 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_cellular() argument
673 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_cellular()
682 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied"); in flow_divert_check_no_cellular()
690 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_expensive() argument
692 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_expensive()
701 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied"); in flow_divert_check_no_expensive()
709 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_constrained() argument
711 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_constrained()
720 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied"); in flow_divert_check_no_constrained()
728 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, bool tunnel, bool flush_snd) in flow_divert_update_closed_state() argument
731 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED; in flow_divert_update_closed_state()
732 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
733 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_update_closed_state()
736 sbflush(&fd_cb->so->so_snd); in flow_divert_update_closed_state()
741 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED; in flow_divert_update_closed_state()
742 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
743 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_update_closed_state()
1024 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id __null… in flow_divert_add_proc_info() argument
1037 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid"); in flow_divert_add_proc_info()
1042 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_proc_info()
1043 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_proc_info()
1046 …uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)__unsaf… in flow_divert_add_proc_info()
1048 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id); in flow_divert_add_proc_info()
1051 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id); in flow_divert_add_proc_info()
1057 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_proc_info()
1074 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error); in flow_divert_add_proc_info()
1086 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error); in flow_divert_add_proc_info()
1090 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash"); in flow_divert_add_proc_info()
1103 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error); in flow_divert_add_proc_info()
1115 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const … in flow_divert_add_all_proc_info() argument
1127 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid); in flow_divert_add_all_proc_info()
1147 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1148 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_all_proc_info()
1153 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1169 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true); in flow_divert_add_all_proc_info()
1175 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false); in flow_divert_add_all_proc_info()
1198 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet) in flow_divert_send_packet() argument
1202 if (fd_cb->group == NULL) { in flow_divert_send_packet()
1203 FDLOG0(LOG_INFO, fd_cb, "no provider, cannot send packet"); in flow_divert_send_packet()
1204 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, false); in flow_divert_send_packet()
1205 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_send_packet()
1206 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_packet()
1211 fd_cb->so->so_error = (uint16_t)error; in flow_divert_send_packet()
1215 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_send_packet()
1217 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) { in flow_divert_send_packet()
1218 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR); in flow_divert_send_packet()
1227 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) { in flow_divert_send_packet()
1228 lck_rw_lock_exclusive(&fd_cb->group->lck); in flow_divert_send_packet()
1230 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet); in flow_divert_send_packet()
1232 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits); in flow_divert_send_packet()
1235 lck_rw_done(&fd_cb->group->lck); in flow_divert_send_packet()
1260 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket… in flow_divert_create_connect_packet() argument
1274 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet); in flow_divert_create_connect_packet()
1279 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) { in flow_divert_create_connect_packet()
1280 …int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0… in flow_divert_create_connect_packet()
1284 …flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_… in flow_divert_create_connect_packet()
1285 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id); in flow_divert_create_connect_packet()
1291 …error = flow_divert_add_all_proc_info(fd_cb, so, p, NULL == signing_id ? NULL : __unsafe_null_term… in flow_divert_create_connect_packet()
1298 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error); in flow_divert_create_connect_packet()
1304 sizeof(fd_cb->so->so_traffic_class), in flow_divert_create_connect_packet()
1305 &fd_cb->so->so_traffic_class); in flow_divert_create_connect_packet()
1310 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_create_connect_packet()
1312 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_create_connect_packet()
1327 if (fd_cb->connect_token != NULL) { in flow_divert_create_connect_packet()
1328 unsigned int token_len = m_length(fd_cb->connect_token); in flow_divert_create_connect_packet()
1329 mbuf_concatenate(connect_packet, fd_cb->connect_token); in flow_divert_create_connect_packet()
1331 fd_cb->connect_token = NULL; in flow_divert_create_connect_packet()
1341 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_create_connect_packet()
1342 …_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->… in flow_divert_create_connect_packet()
1413 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_packet() argument
1416 mbuf_ref_t connect_packet = fd_cb->connect_packet; in flow_divert_send_connect_packet()
1422 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet"); in flow_divert_send_connect_packet()
1426 error = flow_divert_send_packet(fd_cb, connect_packet); in flow_divert_send_connect_packet()
1431 fd_cb->connect_packet = saved_connect_packet; in flow_divert_send_connect_packet()
1445 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_result() argument
1451 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet); in flow_divert_send_connect_result()
1453 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error); in flow_divert_send_connect_result()
1457 rbuff_space = fd_cb->so->so_rcv.sb_hiwat; in flow_divert_send_connect_result()
1470 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_send_connect_result()
1471 …t_packet_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->… in flow_divert_send_connect_result()
1477 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_connect_result()
1491 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how) in flow_divert_send_close() argument
1497 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet); in flow_divert_send_close()
1499 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error); in flow_divert_send_close()
1505 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error); in flow_divert_send_close()
1512 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error); in flow_divert_send_close()
1516 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_close()
1530 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb) in flow_divert_tunnel_how_closed() argument
1532 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == in flow_divert_tunnel_how_closed()
1535 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) { in flow_divert_tunnel_how_closed()
1537 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) { in flow_divert_tunnel_how_closed()
1549 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb) in flow_divert_send_close_if_needed() argument
1554 if (fd_cb->so->so_snd.sb_cc == 0) { in flow_divert_send_close_if_needed()
1555 …if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_… in flow_divert_send_close_if_needed()
1559 …if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRIT… in flow_divert_send_close_if_needed()
1570 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how); in flow_divert_send_close_if_needed()
1571 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) { in flow_divert_send_close_if_needed()
1574 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_send_close_if_needed()
1577 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_send_close_if_needed()
1582 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) { in flow_divert_send_close_if_needed()
1583 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_send_close_if_needed()
1588 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_len) in flow_divert_send_data_packet() argument
1594 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_data_packet()
1596 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_data_packet()
1607 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_data_packet()
1609 fd_cb->bytes_sent += data_len; in flow_divert_send_data_packet()
1610 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_data_packet()
1627 flow_divert_send_datagram_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_len, s… in flow_divert_send_datagram_packet() argument
1633 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_datagram_packet()
1635 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_datagram_packet()
1642 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error); in flow_divert_send_datagram_packet()
1649 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_IS_FRAGMENT) failed: %d", err… in flow_divert_send_datagram_packet()
1656 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_DATAGRAM_SIZE) failed: %d", e… in flow_divert_send_datagram_packet()
1667 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_datagram_packet()
1669 fd_cb->bytes_sent += data_len; in flow_divert_send_datagram_packet()
1670 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_datagram_packet()
1687 flow_divert_send_fragmented_datagram(struct flow_divert_pcb *fd_cb, mbuf_ref_t datagram, size_t dat… in flow_divert_send_fragmented_datagram() argument
1707 …error = flow_divert_send_datagram_packet(fd_cb, next_data, to_send, (first ? toaddr : NULL), TRUE,… in flow_divert_send_fragmented_datagram()
1729 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) in flow_divert_send_buffered_data() argument
1736 to_send = fd_cb->so->so_snd.sb_cc; in flow_divert_send_buffered_data()
1737 buffer = fd_cb->so->so_snd.sb_mb; in flow_divert_send_buffered_data()
1740 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send); in flow_divert_send_buffered_data()
1745 if (!force && (to_send > fd_cb->send_window)) { in flow_divert_send_buffered_data()
1746 to_send = fd_cb->send_window; in flow_divert_send_buffered_data()
1749 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_buffered_data()
1761 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1765 error = flow_divert_send_data_packet(fd_cb, data, data_len); in flow_divert_send_buffered_data()
1775 sbdrop(&fd_cb->so->so_snd, (int)sent); in flow_divert_send_buffered_data()
1776 sowwakeup(fd_cb->so); in flow_divert_send_buffered_data()
1777 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_buffered_data()
1796 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain."); in flow_divert_send_buffered_data()
1802 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); in flow_divert_send_buffered_data()
1805 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1812 error = flow_divert_send_datagram_packet(fd_cb, data, data_len, toaddr, FALSE, 0); in flow_divert_send_buffered_data()
1814 error = flow_divert_send_fragmented_datagram(fd_cb, data, data_len, toaddr); in flow_divert_send_buffered_data()
1826 (void) sbdroprecord(&(fd_cb->so->so_snd)); in flow_divert_send_buffered_data()
1831 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent); in flow_divert_send_buffered_data()
1832 if (fd_cb->send_window >= sent) { in flow_divert_send_buffered_data()
1833 fd_cb->send_window -= sent; in flow_divert_send_buffered_data()
1835 fd_cb->send_window = 0; in flow_divert_send_buffered_data()
1841 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_size, struct … in flow_divert_send_app_data() argument
1846 if (to_send > fd_cb->send_window) { in flow_divert_send_app_data()
1847 to_send = fd_cb->send_window; in flow_divert_send_app_data()
1850 if (fd_cb->so->so_snd.sb_cc > 0) { in flow_divert_send_app_data()
1854 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_app_data()
1873 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_send_app_data()
1883 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len); in flow_divert_send_app_data()
1892 if (fd_cb->send_window >= sent) { in flow_divert_send_app_data()
1893 fd_cb->send_window -= sent; in flow_divert_send_app_data()
1895 fd_cb->send_window = 0; in flow_divert_send_app_data()
1901 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1902 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) { in flow_divert_send_app_data()
1903 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %… in flow_divert_send_app_data()
1904 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1913 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1914 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) { in flow_divert_send_app_data()
1915 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_wind… in flow_divert_send_app_data()
1916 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1923 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_app_data()
1927 send_dgram_error = flow_divert_send_datagram_packet(fd_cb, data, data_size, toaddr, FALSE, 0); in flow_divert_send_app_data()
1929 send_dgram_error = flow_divert_send_fragmented_datagram(fd_cb, data, data_size, toaddr); in flow_divert_send_app_data()
1933 …FDLOG(LOG_NOTICE, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = … in flow_divert_send_app_data()
1935 if (data_size >= fd_cb->send_window) { in flow_divert_send_app_data()
1936 fd_cb->send_window = 0; in flow_divert_send_app_data()
1938 fd_cb->send_window -= data_size; in flow_divert_send_app_data()
1946 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1949 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &append_error)) { in flow_divert_send_app_data()
1950 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1952 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, append_error); in flow_divert_send_app_data()
1955 if (!sbappendrecord(&fd_cb->so->so_snd, data)) { in flow_divert_send_app_data()
1956 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1958 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1962 …FDLOG(LOG_ERR, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = %lu… in flow_divert_send_app_data()
1972 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb) in flow_divert_send_read_notification() argument
1977 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet); in flow_divert_send_read_notification()
1979 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error); in flow_divert_send_read_notification()
1983 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_read_notification()
1997 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class) in flow_divert_send_traffic_class_update() argument
2002 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet); in flow_divert_send_traffic_class_update()
2004 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error); in flow_divert_send_traffic_class_update()
2010 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error); in flow_divert_send_traffic_class_update()
2014 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_traffic_class_update()
2028 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint) in flow_divert_set_local_endpoint() argument
2030 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_local_endpoint()
2033 …if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR… in flow_divert_set_local_endpoint()
2034 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2043 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) { in flow_divert_set_local_endpoint()
2044 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2054 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint) in flow_divert_set_remote_endpoint() argument
2056 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_remote_endpoint()
2161 flow_divert_try_next_group(struct flow_divert_pcb *fd_cb) in flow_divert_try_next_group() argument
2164 uint32_t policy_control_unit = fd_cb->policy_control_unit; in flow_divert_try_next_group()
2166 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2171 …uint32_t next_ctl_unit = flow_divert_derive_kernel_control_unit(0, &policy_control_unit, &(fd_cb->… in flow_divert_try_next_group()
2173 if (fd_cb->control_group_unit == next_ctl_unit) { in flow_divert_try_next_group()
2174 …FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling fl… in flow_divert_try_next_group()
2180 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert"); in flow_divert_try_next_group()
2185 next_group = flow_divert_group_lookup(next_ctl_unit, fd_cb); in flow_divert_try_next_group()
2187 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit); in flow_divert_try_next_group()
2191 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", fd_cb->control_group_unit, next_ctl_unit); in flow_divert_try_next_group()
2193 error = flow_divert_pcb_insert(fd_cb, next_group); in flow_divert_try_next_group()
2196 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2198 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2202 } while (fd_cb->group == NULL); in flow_divert_try_next_group()
2204 if (fd_cb->group == NULL) { in flow_divert_try_next_group()
2208 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_try_next_group()
2210 …FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", fd_cb->… in flow_divert_try_next_group()
2211 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2219 flow_divert_disable(struct flow_divert_pcb *fd_cb) in flow_divert_disable() argument
2225 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint; in flow_divert_disable()
2226 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT); in flow_divert_disable()
2229 so = fd_cb->so; in flow_divert_disable()
2234 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert"); in flow_divert_disable()
2238 inp->inp_vflag = fd_cb->original_vflag; in flow_divert_disable()
2245 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) { in flow_divert_disable()
2250 inp->inp_last_outifp = fd_cb->original_last_outifp; in flow_divert_disable()
2251 inp->in6p_last_outifp = fd_cb->original_last_outifp6; in flow_divert_disable()
2257 fd_cb->so = NULL; in flow_divert_disable()
2259 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_disable()
2273 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error); in flow_divert_disable()
2291 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer"); in flow_divert_disable()
2309 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued TCP data using the socket's original protocol: %d", e… in flow_divert_disable()
2361 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer"); in flow_divert_disable()
2366 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data"); in flow_divert_disable()
2372 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header"); in flow_divert_disable()
2397 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued UDP data using the socket's original protocol: %d", e… in flow_divert_disable()
2413 flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address) in flow_divert_scope() argument
2421 so = fd_cb->so; in flow_divert_scope()
2447 …FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error); in flow_divert_scope()
2451 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) { in flow_divert_scope()
2455 …error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr)… in flow_divert_scope()
2458 …error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), I… in flow_divert_scope()
2462 …FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr return… in flow_divert_scope()
2490 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_connect_result() argument
2506 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error); in flow_divert_handle_connect_result()
2511 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error); in flow_divert_handle_connect_result()
2515 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error); in flow_divert_handle_connect_result()
2521 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result"); in flow_divert_handle_connect_result()
2526 FDLOG0(LOG_INFO, fd_cb, "No local address provided"); in flow_divert_handle_connect_result()
2531 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_connect_result()
2536 FDLOG0(LOG_INFO, fd_cb, "No output if index provided"); in flow_divert_handle_connect_result()
2541 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result"); in flow_divert_handle_connect_result()
2546 FDLOCK(fd_cb); in flow_divert_handle_connect_result()
2547 if (fd_cb->so != NULL) { in flow_divert_handle_connect_result()
2549 struct socket *so = fd_cb->so; in flow_divert_handle_connect_result()
2555 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result"); in flow_divert_handle_connect_result()
2560 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result"); in flow_divert_handle_connect_result()
2575 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2578 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port; in flow_divert_handle_connect_result()
2584 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2587 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port; in flow_divert_handle_connect_result()
2592 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid); in flow_divert_handle_connect_result()
2593 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_connect_result()
2601 flow_divert_set_remote_endpoint(fd_cb, SA(&remote_endpoint)); in flow_divert_handle_connect_result()
2610 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length); in flow_divert_handle_connect_result()
2611 if (fd_cb->app_data != NULL) { in flow_divert_handle_connect_result()
2612 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_connect_result()
2614 fd_cb->app_data = app_data; in flow_divert_handle_connect_result()
2615 fd_cb->app_data_length = app_data_length; in flow_divert_handle_connect_result()
2617 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet"… in flow_divert_handle_connect_result()
2621 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_connect_result()
2629 if (fd_cb->group == NULL) { in flow_divert_handle_connect_result()
2640 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit); in flow_divert_handle_connect_result()
2645 grp = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_handle_connect_result()
2651 flow_divert_pcb_remove(fd_cb); in flow_divert_handle_connect_result()
2652 insert_error = flow_divert_pcb_insert(fd_cb, grp); in flow_divert_handle_connect_result()
2661 fd_cb->send_window = ntohl(send_window); in flow_divert_handle_connect_result()
2665 FDLOG0(LOG_INFO, fd_cb, "sending connect result"); in flow_divert_handle_connect_result()
2666 error = flow_divert_send_connect_result(fd_cb); in flow_divert_handle_connect_result()
2670 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) { in flow_divert_handle_connect_result()
2671 error = flow_divert_try_next_group(fd_cb); in flow_divert_handle_connect_result()
2672 if (error && fd_cb->policy_control_unit == 0) { in flow_divert_handle_connect_result()
2673 flow_divert_disable(fd_cb); in flow_divert_handle_connect_result()
2681 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_handle_connect_result()
2683 flow_divert_send_close_if_needed(fd_cb); in flow_divert_handle_connect_result()
2685 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_handle_connect_result()
2688 flow_divert_disconnect_socket(so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_handle_connect_result()
2703 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_connect_result()
2708 if (fd_cb->connect_packet != NULL) { in flow_divert_handle_connect_result()
2709 mbuf_freem(fd_cb->connect_packet); in flow_divert_handle_connect_result()
2710 fd_cb->connect_packet = NULL; in flow_divert_handle_connect_result()
2714 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_handle_connect_result()
2718 FDUNLOCK(fd_cb); in flow_divert_handle_connect_result()
2722 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_close() argument
2730 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error); in flow_divert_handle_close()
2736 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error); in flow_divert_handle_close()
2742 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how); in flow_divert_handle_close()
2744 FDLOCK(fd_cb); in flow_divert_handle_close()
2745 if (fd_cb->so != NULL) { in flow_divert_handle_close()
2746 …bool is_connected = (SOCK_TYPE(fd_cb->so) == SOCK_STREAM || !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_… in flow_divert_handle_close()
2747 socket_lock(fd_cb->so, 0); in flow_divert_handle_close()
2749 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_close()
2750 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider"); in flow_divert_handle_close()
2754 fd_cb->so->so_error = (uint16_t)ntohl(close_error); in flow_divert_handle_close()
2756 flow_divert_update_closed_state(fd_cb, how, true, true); in flow_divert_handle_close()
2759 how = flow_divert_tunnel_how_closed(fd_cb); in flow_divert_handle_close()
2761 flow_divert_disconnect_socket(fd_cb->so, is_connected, true); in flow_divert_handle_close()
2763 socantrcvmore(fd_cb->so); in flow_divert_handle_close()
2765 socantsendmore(fd_cb->so); in flow_divert_handle_close()
2768 socket_unlock(fd_cb->so, 0); in flow_divert_handle_close()
2770 FDUNLOCK(fd_cb); in flow_divert_handle_close()
2774 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb) in flow_divert_create_control_mbuf() argument
2776 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_create_control_mbuf()
2781 fd_cb->local_endpoint.sa.sa_family == AF_INET && in flow_divert_create_control_mbuf()
2783 …return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_… in flow_divert_create_control_mbuf()
2785 fd_cb->local_endpoint.sa.sa_family == AF_INET6 && in flow_divert_create_control_mbuf()
2789 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr; in flow_divert_create_control_mbuf()
2797 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, size_t offset) in flow_divert_handle_data() argument
2801 FDLOCK(fd_cb); in flow_divert_handle_data()
2802 if (fd_cb->so != NULL) { in flow_divert_handle_data()
2810 socket_lock(fd_cb->so, 0); in flow_divert_handle_data()
2812 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_data()
2813 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data"); in flow_divert_handle_data()
2817 if (sbspace(&fd_cb->so->so_rcv) == 0) { in flow_divert_handle_data()
2819 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_handle_data()
2820 …FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some d… in flow_divert_handle_data()
2824 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_handle_data()
2831 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_data()
2841 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid"); in flow_divert_handle_data()
2849 if (fd_cb->so->so_state & SS_CANTRCVMORE) { in flow_divert_handle_data()
2850 …FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size… in flow_divert_handle_data()
2854 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) { in flow_divert_handle_data()
2855 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so)); in flow_divert_handle_data()
2859 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size); in flow_divert_handle_data()
2863 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_handle_data()
2867 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_handle_data()
2868 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0); in flow_divert_handle_data()
2877 if (SOCK_CHECK_DOM(fd_cb->so, AF_INET6)) { in flow_divert_handle_data()
2878 error = in6_mapped_peeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2880 error = in_getpeeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2884 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address."); in flow_divert_handle_data()
2887 mctl = flow_divert_create_control_mbuf(fd_cb); in flow_divert_handle_data()
2889 appended = sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error); in flow_divert_handle_data()
2893 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error); in flow_divert_handle_data()
2900 fd_cb->bytes_received += data_size; in flow_divert_handle_data()
2901 flow_divert_add_data_statistics(fd_cb, data_size, FALSE); in flow_divert_handle_data()
2905 sorwakeup(fd_cb->so); in flow_divert_handle_data()
2908 socket_unlock(fd_cb->so, 0); in flow_divert_handle_data()
2910 FDUNLOCK(fd_cb); in flow_divert_handle_data()
2916 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_read_notification() argument
2923 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error); in flow_divert_handle_read_notification()
2927 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count)); in flow_divert_handle_read_notification()
2929 FDLOCK(fd_cb); in flow_divert_handle_read_notification()
2930 if (fd_cb->so != NULL) { in flow_divert_handle_read_notification()
2931 socket_lock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2933 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_read_notification()
2934 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification"); in flow_divert_handle_read_notification()
2938 fd_cb->send_window += ntohl(read_count); in flow_divert_handle_read_notification()
2939 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_read_notification()
2941 socket_unlock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2943 FDUNLOCK(fd_cb); in flow_divert_handle_read_notification()
3008 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_properties_update() argument
3014 FDLOG0(LOG_INFO, fd_cb, "received a properties update"); in flow_divert_handle_properties_update()
3018 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update"); in flow_divert_handle_properties_update()
3023 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update"); in flow_divert_handle_properties_update()
3026 FDLOCK(fd_cb); in flow_divert_handle_properties_update()
3027 if (fd_cb->so != NULL) { in flow_divert_handle_properties_update()
3028 socket_lock(fd_cb->so, 0); in flow_divert_handle_properties_update()
3030 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_properties_update()
3031 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update"); in flow_divert_handle_properties_update()
3036 flow_divert_scope(fd_cb, out_if_index, true); in flow_divert_handle_properties_update()
3037 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_properties_update()
3046 if (fd_cb->app_data != NULL) { in flow_divert_handle_properties_update()
3047 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_properties_update()
3049 fd_cb->app_data = app_data; in flow_divert_handle_properties_update()
3050 fd_cb->app_data_length = app_data_length; in flow_divert_handle_properties_update()
3052 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update pack… in flow_divert_handle_properties_update()
3056 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_properties_update()
3060 socket_unlock(fd_cb->so, 0); in flow_divert_handle_properties_update()
3062 FDUNLOCK(fd_cb); in flow_divert_handle_properties_update()
3238 struct flow_divert_pcb *fd_cb; in flow_divert_handle_flow_states_request() local
3260 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_handle_flow_states_request()
3261 FDRETAIN(fd_cb); in flow_divert_handle_flow_states_request()
3262 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_handle_flow_states_request()
3267 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { in flow_divert_handle_flow_states_request()
3268 FDLOCK(fd_cb); in flow_divert_handle_flow_states_request()
3269 if (fd_cb->so != NULL) { in flow_divert_handle_flow_states_request()
3271 socket_lock(fd_cb->so, 0); in flow_divert_handle_flow_states_request()
3273 state.conn_id = fd_cb->hash; in flow_divert_handle_flow_states_request()
3274 state.bytes_written_by_app = fd_cb->bytes_written_by_app; in flow_divert_handle_flow_states_request()
3275 state.bytes_sent = fd_cb->bytes_sent; in flow_divert_handle_flow_states_request()
3276 state.bytes_received = fd_cb->bytes_received; in flow_divert_handle_flow_states_request()
3277 state.send_window = fd_cb->send_window; in flow_divert_handle_flow_states_request()
3278 state.send_buffer_bytes = fd_cb->so->so_snd.sb_cc; in flow_divert_handle_flow_states_request()
3282 FDLOG(LOG_ERR, fd_cb, "Failed to add a flow state: %d", error); in flow_divert_handle_flow_states_request()
3285 socket_unlock(fd_cb->so, 0); in flow_divert_handle_flow_states_request()
3287 FDUNLOCK(fd_cb); in flow_divert_handle_flow_states_request()
3288 FDRELEASE(fd_cb); in flow_divert_handle_flow_states_request()
3303 struct flow_divert_pcb *fd_cb; in flow_divert_input() local
3338 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */ in flow_divert_input()
3339 if (fd_cb == NULL) { in flow_divert_input()
3348 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3351 flow_divert_handle_close(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3354 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3357 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3360 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3363 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type); in flow_divert_input()
3367 FDRELEASE(fd_cb); in flow_divert_input()
3377 struct flow_divert_pcb *fd_cb; in flow_divert_close_all() local
3386 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_close_all()
3387 FDRETAIN(fd_cb); in flow_divert_close_all()
3388 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_close_all()
3396 fd_cb = SLIST_FIRST(&tmp_list); in flow_divert_close_all()
3397 FDLOCK(fd_cb); in flow_divert_close_all()
3399 if (fd_cb->so != NULL) { in flow_divert_close_all()
3400 socket_lock(fd_cb->so, 0); in flow_divert_close_all()
3401 flow_divert_pcb_remove(fd_cb); in flow_divert_close_all()
3402 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_close_all()
3403 fd_cb->so->so_error = ECONNABORTED; in flow_divert_close_all()
3404 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_close_all()
3405 socket_unlock(fd_cb->so, 0); in flow_divert_close_all()
3407 FDUNLOCK(fd_cb); in flow_divert_close_all()
3408 FDRELEASE(fd_cb); in flow_divert_close_all()
3415 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_detach() local
3424 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count); in flow_divert_detach()
3426 if (fd_cb->group != NULL) { in flow_divert_detach()
3428 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_detach()
3430 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_detach()
3431 flow_divert_send_close_if_needed(fd_cb); in flow_divert_detach()
3433 flow_divert_pcb_remove(fd_cb); in flow_divert_detach()
3437 FDLOCK(fd_cb); in flow_divert_detach()
3438 fd_cb->so = NULL; in flow_divert_detach()
3439 FDUNLOCK(fd_cb); in flow_divert_detach()
3442 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_detach()
3448 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_close() local
3454 FDLOG0(LOG_INFO, fd_cb, "Closing"); in flow_divert_close()
3461 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_close()
3462 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_close()
3463 flow_divert_send_close_if_needed(fd_cb); in flow_divert_close()
3466 flow_divert_pcb_remove(fd_cb); in flow_divert_close()
3485 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_shutdown() local
3491 FDLOG0(LOG_INFO, fd_cb, "Can't send more"); in flow_divert_shutdown()
3495 flow_divert_update_closed_state(fd_cb, SHUT_WR, false, true); in flow_divert_shutdown()
3496 flow_divert_send_close_if_needed(fd_cb); in flow_divert_shutdown()
3504 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_rcvd() local
3512 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space); in flow_divert_rcvd()
3513 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) && in flow_divert_rcvd()
3515 flow_divert_send_read_notification(fd_cb) == 0) { in flow_divert_rcvd()
3516 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification"); in flow_divert_rcvd()
3517 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_rcvd()
3657 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_ctloutput() local
3664 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_ctloutput()
3665 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class); in flow_divert_ctloutput()
3680 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out_internal() local
3691 if (fd_cb->group == NULL) { in flow_divert_connect_out_internal()
3709 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_connect_out_internal()
3714 FDLOG0(LOG_INFO, fd_cb, "Connecting"); in flow_divert_connect_out_internal()
3716 if (fd_cb->connect_packet == NULL) { in flow_divert_connect_out_internal()
3721 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet"); in flow_divert_connect_out_internal()
3727 FDLOG0(LOG_ERR, fd_cb, "Destination address is not valid when creating connect packet"); in flow_divert_connect_out_internal()
3732 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0); in flow_divert_connect_out_internal()
3733 if (fd_cb->original_remote_endpoint == NULL) { in flow_divert_connect_out_internal()
3734 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint"); in flow_divert_connect_out_internal()
3738 fd_cb->original_vflag = inp->inp_vflag; in flow_divert_connect_out_internal()
3739 fd_cb->original_last_outifp = inp->inp_last_outifp; in flow_divert_connect_out_internal()
3740 fd_cb->original_last_outifp6 = inp->in6p_last_outifp; in flow_divert_connect_out_internal()
3765 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6); in flow_divert_connect_out_internal()
3766 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6; in flow_divert_connect_out_internal()
3767 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport; in flow_divert_connect_out_internal()
3768 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp); in flow_divert_connect_out_internal()
3770 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error); in flow_divert_connect_out_internal()
3771 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || IN6_IS_ADDR_UNSPECIFIED(&(satosin6(to)->s… in flow_divert_connect_out_internal()
3782 if (IN6_IS_SCOPE_EMBED(&(fd_cb->local_endpoint.sin6.sin6_addr)) && in flow_divert_connect_out_internal()
3784 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] != 0) { in flow_divert_connect_out_internal()
3785 …fd_cb->local_endpoint.sin6.sin6_scope_id = ntohs(fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1]… in flow_divert_connect_out_internal()
3786 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] = 0; in flow_divert_connect_out_internal()
3798 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in); in flow_divert_connect_out_internal()
3799 fd_cb->local_endpoint.sin.sin_family = AF_INET; in flow_divert_connect_out_internal()
3800 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport; in flow_divert_connect_out_internal()
3801 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0); in flow_divert_connect_out_internal()
3803 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error); in flow_divert_connect_out_internal()
3804 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || satosin(to)->sin_addr.s_addr == INADDR_AN… in flow_divert_connect_out_internal()
3815 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family); in flow_divert_connect_out_internal()
3818 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_connect_out_internal()
3819 flow_divert_check_no_expensive(fd_cb) || in flow_divert_connect_out_internal()
3820 flow_divert_check_no_constrained(fd_cb); in flow_divert_connect_out_internal()
3829 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR; in flow_divert_connect_out_internal()
3832 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet); in flow_divert_connect_out_internal()
3838 flow_divert_set_remote_endpoint(fd_cb, to); in flow_divert_connect_out_internal()
3839 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_connect_out_internal()
3843 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT; in flow_divert_connect_out_internal()
3847 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive"); in flow_divert_connect_out_internal()
3851 fd_cb->connect_packet = connect_packet; in flow_divert_connect_out_internal()
3854 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet"); in flow_divert_connect_out_internal()
3858 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_connect_out_internal()
3863 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_connect_out_internal()
3866 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) { in flow_divert_connect_out_internal()
3883 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out() local
3884 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error); in flow_divert_connect_out()
3907 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connectx_out_common() local
3908 if (fd_cb != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_TOKEN) && in flow_divert_connectx_out_common()
3983 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_data_out() local
4000 if ((fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) && SOCK_TYPE(so) == SOCK_DGRAM) { in flow_divert_data_out()
4002 FDLOG0(LOG_INFO, fd_cb, "provider is no longer accepting writes, cannot send data"); in flow_divert_data_out()
4018 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to); in flow_divert_data_out()
4023 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_data_out()
4024 FDLOG0(LOG_INFO, fd_cb, "implicit connect"); in flow_divert_data_out()
4031 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_data_out()
4032 flow_divert_check_no_expensive(fd_cb) || in flow_divert_data_out()
4033 flow_divert_check_no_constrained(fd_cb); in flow_divert_data_out()
4049 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", data_size); in flow_divert_data_out()
4050 fd_cb->bytes_written_by_app += data_size; in flow_divert_data_out()
4052 error = flow_divert_send_app_data(fd_cb, data, data_size, to); in flow_divert_data_out()
4085 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_preconnect() local
4091 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_preconnect()
4092 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet"); in flow_divert_preconnect()
4098 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_preconnect()
4129 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out() local
4138 if (fd_cb == NULL) { in flow_divert_implicit_data_out()
4140 fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out()
4141 if (error != 0 || fd_cb == NULL) { in flow_divert_implicit_data_out()
4162 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_init_internal() local
4171 fd_cb = flow_divert_pcb_create(so); in flow_divert_pcb_init_internal()
4172 if (fd_cb == NULL) { in flow_divert_pcb_init_internal()
4179 FDLOG0(LOG_ERR, fd_cb, "No valid group is available, cannot init flow divert"); in flow_divert_pcb_init_internal()
4184 error = flow_divert_add_to_group(fd_cb, group_unit); in flow_divert_pcb_init_internal()
4186 so->so_fd_pcb = fd_cb; in flow_divert_pcb_init_internal()
4188 fd_cb->control_group_unit = group_unit; in flow_divert_pcb_init_internal()
4189 fd_cb->policy_control_unit = ctl_unit; in flow_divert_pcb_init_internal()
4190 fd_cb->aggregate_unit = agg_unit; in flow_divert_pcb_init_internal()
4192 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4194 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4203 FDLOG0(LOG_INFO, fd_cb, "Created"); in flow_divert_pcb_init_internal()
4205 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error); in flow_divert_pcb_init_internal()
4210 FDRELEASE(fd_cb); in flow_divert_pcb_init_internal()
4313 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4318 fd_cb->log_level = (uint8_t)log_level; in flow_divert_token_set()
4322 fd_cb->connect_token = token; in flow_divert_token_set()
4325 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN; in flow_divert_token_set()
4329 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4330 if (fd_cb != NULL) { in flow_divert_token_set()
4331 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC; in flow_divert_token_set()
4349 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_get() local
4358 if (fd_cb->group == NULL) { in flow_divert_token_get()
4365 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_token_get()
4369 ctl_unit = htonl(fd_cb->group->ctl_unit); in flow_divert_token_get()
4376 …error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb-… in flow_divert_token_get()
4381 if (fd_cb->app_data != NULL) { in flow_divert_token_get()
4382 …_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_d… in flow_divert_token_get()
4388 control_group = flow_divert_group_lookup(fd_cb->control_group_unit, fd_cb); in flow_divert_token_get()
4649 struct flow_divert_pcb *fd_cb; in flow_divert_kctl_rcvd() local
4670 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_kctl_rcvd()
4671 FDRETAIN(fd_cb); in flow_divert_kctl_rcvd()
4672 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_kctl_rcvd()
4677 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { in flow_divert_kctl_rcvd()
4678 FDLOCK(fd_cb); in flow_divert_kctl_rcvd()
4679 if (fd_cb->so != NULL) { in flow_divert_kctl_rcvd()
4680 socket_lock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4681 if (fd_cb->group != NULL) { in flow_divert_kctl_rcvd()
4682 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_kctl_rcvd()
4684 socket_unlock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4686 FDUNLOCK(fd_cb); in flow_divert_kctl_rcvd()
4687 FDRELEASE(fd_cb); in flow_divert_kctl_rcvd()