Lines Matching refs:fd_cb
226 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_lookup() local
231 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item); in flow_divert_pcb_lookup()
232 FDRETAIN(fd_cb); in flow_divert_pcb_lookup()
235 return fd_cb; in flow_divert_pcb_lookup()
239 flow_divert_group_lookup(uint32_t ctl_unit, struct flow_divert_pcb *fd_cb) in flow_divert_group_lookup() argument
244 if (fd_cb != NULL) { in flow_divert_group_lookup()
245 FDLOG0(LOG_ERR, fd_cb, "No active groups, flow divert cannot be used for this socket"); in flow_divert_group_lookup()
248 FDLOG(LOG_ERR, fd_cb, "Cannot lookup group with invalid control unit (%u)", ctl_unit); in flow_divert_group_lookup()
251 if (fd_cb != NULL) { in flow_divert_group_lookup()
252 …FDLOG0(LOG_ERR, fd_cb, "No active non-in-process groups, flow divert cannot be used for this socke… in flow_divert_group_lookup()
257 if (fd_cb != NULL) { in flow_divert_group_lookup()
258 …FDLOG(LOG_ERR, fd_cb, "Group for control unit %u is NULL, flow divert cannot be used for this sock… in flow_divert_group_lookup()
266 if (fd_cb != NULL) { in flow_divert_group_lookup()
267 … FDLOG0(LOG_ERR, fd_cb, "No active in-process groups, flow divert cannot be used for this socket"); in flow_divert_group_lookup()
278 if (fd_cb != NULL) { in flow_divert_group_lookup()
279 …FDLOG(LOG_ERR, fd_cb, "Group for control unit %u not found, flow divert cannot be used for this so… in flow_divert_group_lookup()
281 } else if (fd_cb != NULL && in flow_divert_group_lookup()
282 (fd_cb->so == NULL || in flow_divert_group_lookup()
283 group_cursor->in_process_pid != fd_cb->so->last_pid)) { in flow_divert_group_lookup()
284 FDLOG(LOG_ERR, fd_cb, "Cannot access group for control unit %u, mismatched PID (%u != %u)", in flow_divert_group_lookup()
285 ctl_unit, group_cursor->in_process_pid, fd_cb->so ? fd_cb->so->last_pid : 0); in flow_divert_group_lookup()
297 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, struct flow_divert_group *group) in flow_divert_pcb_insert() argument
302 if (NULL == RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb)) { in flow_divert_pcb_insert()
303 fd_cb->group = group; in flow_divert_pcb_insert()
304 fd_cb->control_group_unit = group->ctl_unit; in flow_divert_pcb_insert()
305 FDRETAIN(fd_cb); /* The group now has a reference */ in flow_divert_pcb_insert()
307 …FDLOG(LOG_ERR, fd_cb, "Group %u already contains a PCB with hash %u", group->ctl_unit, fd_cb->hash… in flow_divert_pcb_insert()
311 FDLOG(LOG_ERR, fd_cb, "Group %u is defunct, cannot insert", group->ctl_unit); in flow_divert_pcb_insert()
319 flow_divert_add_to_group(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit) in flow_divert_add_to_group() argument
327 group = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_add_to_group()
344 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed); in flow_divert_add_to_group()
353 if (NULL != RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb)) { in flow_divert_add_to_group()
362 error = flow_divert_pcb_insert(fd_cb, group); in flow_divert_add_to_group()
367 FDLOG0(LOG_ERR, fd_cb, "Failed to create a unique hash"); in flow_divert_add_to_group()
368 fd_cb->hash = 0; in flow_divert_add_to_group()
391 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_destroy() argument
393 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %llu, tunnel tx %llu, tunnel rx %llu", in flow_divert_pcb_destroy()
394 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received); in flow_divert_pcb_destroy()
396 if (fd_cb->connect_token != NULL) { in flow_divert_pcb_destroy()
397 mbuf_freem(fd_cb->connect_token); in flow_divert_pcb_destroy()
399 if (fd_cb->connect_packet != NULL) { in flow_divert_pcb_destroy()
400 mbuf_freem(fd_cb->connect_packet); in flow_divert_pcb_destroy()
402 if (fd_cb->app_data != NULL) { in flow_divert_pcb_destroy()
403 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_pcb_destroy()
405 if (fd_cb->original_remote_endpoint != NULL) { in flow_divert_pcb_destroy()
406 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_pcb_destroy()
408 zfree(flow_divert_pcb_zone, fd_cb); in flow_divert_pcb_destroy()
412 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb) in flow_divert_pcb_remove() argument
414 if (fd_cb->group != NULL) { in flow_divert_pcb_remove()
415 struct flow_divert_group *group = fd_cb->group; in flow_divert_pcb_remove()
417 …FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count… in flow_divert_pcb_remove()
418 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb); in flow_divert_pcb_remove()
419 fd_cb->group = NULL; in flow_divert_pcb_remove()
420 FDRELEASE(fd_cb); /* Release the group's reference */ in flow_divert_pcb_remove()
426 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_ref_t *packet) in flow_divert_packet_init() argument
433 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_packet_init()
438 hdr.conn_id = htonl(fd_cb->hash); in flow_divert_packet_init()
443 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error); in flow_divert_packet_init()
641 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send) in flow_divert_add_data_statistics() argument
647 inp = sotoinpcb(fd_cb->so); in flow_divert_add_data_statistics()
672 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_cellular() argument
674 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_cellular()
683 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied"); in flow_divert_check_no_cellular()
691 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_expensive() argument
693 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_expensive()
702 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied"); in flow_divert_check_no_expensive()
710 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb) in flow_divert_check_no_constrained() argument
712 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_check_no_constrained()
721 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied"); in flow_divert_check_no_constrained()
729 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, bool tunnel, bool flush_snd) in flow_divert_update_closed_state() argument
732 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED; in flow_divert_update_closed_state()
733 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
734 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_update_closed_state()
737 sbflush(&fd_cb->so->so_snd); in flow_divert_update_closed_state()
742 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED; in flow_divert_update_closed_state()
743 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_update_closed_state()
744 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_update_closed_state()
1025 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id __null… in flow_divert_add_proc_info() argument
1038 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid"); in flow_divert_add_proc_info()
1043 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_proc_info()
1044 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_proc_info()
1047 …uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)__unsaf… in flow_divert_add_proc_info()
1049 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id); in flow_divert_add_proc_info()
1052 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id); in flow_divert_add_proc_info()
1058 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_proc_info()
1075 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error); in flow_divert_add_proc_info()
1087 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error); in flow_divert_add_proc_info()
1091 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash"); in flow_divert_add_proc_info()
1104 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error); in flow_divert_add_proc_info()
1116 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const … in flow_divert_add_all_proc_info() argument
1128 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid); in flow_divert_add_all_proc_info()
1148 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1149 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) { in flow_divert_add_all_proc_info()
1154 lck_rw_done(&fd_cb->group->lck); in flow_divert_add_all_proc_info()
1170 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true); in flow_divert_add_all_proc_info()
1176 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false); in flow_divert_add_all_proc_info()
1199 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet) in flow_divert_send_packet() argument
1203 if (fd_cb->group == NULL) { in flow_divert_send_packet()
1204 FDLOG0(LOG_INFO, fd_cb, "no provider, cannot send packet"); in flow_divert_send_packet()
1205 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, false); in flow_divert_send_packet()
1206 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_send_packet()
1207 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_packet()
1212 fd_cb->so->so_error = (uint16_t)error; in flow_divert_send_packet()
1216 lck_rw_lock_shared(&fd_cb->group->lck); in flow_divert_send_packet()
1218 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) { in flow_divert_send_packet()
1219 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR); in flow_divert_send_packet()
1228 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) { in flow_divert_send_packet()
1229 lck_rw_lock_exclusive(&fd_cb->group->lck); in flow_divert_send_packet()
1231 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet); in flow_divert_send_packet()
1233 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits); in flow_divert_send_packet()
1236 lck_rw_done(&fd_cb->group->lck); in flow_divert_send_packet()
1261 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket… in flow_divert_create_connect_packet() argument
1275 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet); in flow_divert_create_connect_packet()
1280 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) { in flow_divert_create_connect_packet()
1281 …int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0… in flow_divert_create_connect_packet()
1285 …flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_… in flow_divert_create_connect_packet()
1286 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id); in flow_divert_create_connect_packet()
1292 …error = flow_divert_add_all_proc_info(fd_cb, so, p, NULL == signing_id ? NULL : __unsafe_null_term… in flow_divert_create_connect_packet()
1299 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error); in flow_divert_create_connect_packet()
1305 sizeof(fd_cb->so->so_traffic_class), in flow_divert_create_connect_packet()
1306 &fd_cb->so->so_traffic_class); in flow_divert_create_connect_packet()
1311 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_create_connect_packet()
1313 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_create_connect_packet()
1328 if (fd_cb->connect_token != NULL) { in flow_divert_create_connect_packet()
1329 unsigned int token_len = m_length(fd_cb->connect_token); in flow_divert_create_connect_packet()
1330 mbuf_concatenate(connect_packet, fd_cb->connect_token); in flow_divert_create_connect_packet()
1332 fd_cb->connect_token = NULL; in flow_divert_create_connect_packet()
1342 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_create_connect_packet()
1343 …end_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, SA_BYTES(&(fd… in flow_divert_create_connect_packet()
1414 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_packet() argument
1417 mbuf_ref_t connect_packet = fd_cb->connect_packet; in flow_divert_send_connect_packet()
1423 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet"); in flow_divert_send_connect_packet()
1427 error = flow_divert_send_packet(fd_cb, connect_packet); in flow_divert_send_connect_packet()
1432 fd_cb->connect_packet = saved_connect_packet; in flow_divert_send_connect_packet()
1446 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb) in flow_divert_send_connect_result() argument
1452 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet); in flow_divert_send_connect_result()
1454 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error); in flow_divert_send_connect_result()
1458 rbuff_space = fd_cb->so->so_rcv.sb_hiwat; in flow_divert_send_connect_result()
1471 …if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET… in flow_divert_send_connect_result()
1472 …cket_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, SA_BYTES(&(fd… in flow_divert_send_connect_result()
1478 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_connect_result()
1492 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how) in flow_divert_send_close() argument
1498 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet); in flow_divert_send_close()
1500 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error); in flow_divert_send_close()
1506 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error); in flow_divert_send_close()
1513 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error); in flow_divert_send_close()
1517 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_close()
1531 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb) in flow_divert_tunnel_how_closed() argument
1533 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == in flow_divert_tunnel_how_closed()
1536 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) { in flow_divert_tunnel_how_closed()
1538 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) { in flow_divert_tunnel_how_closed()
1550 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb) in flow_divert_send_close_if_needed() argument
1555 if (fd_cb->so->so_snd.sb_cc == 0) { in flow_divert_send_close_if_needed()
1556 …if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_… in flow_divert_send_close_if_needed()
1560 …if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRIT… in flow_divert_send_close_if_needed()
1571 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how); in flow_divert_send_close_if_needed()
1572 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) { in flow_divert_send_close_if_needed()
1575 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; in flow_divert_send_close_if_needed()
1578 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; in flow_divert_send_close_if_needed()
1583 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) { in flow_divert_send_close_if_needed()
1584 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_send_close_if_needed()
1589 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_len) in flow_divert_send_data_packet() argument
1595 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_data_packet()
1597 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_data_packet()
1608 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_data_packet()
1610 fd_cb->bytes_sent += data_len; in flow_divert_send_data_packet()
1611 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_data_packet()
1628 flow_divert_send_datagram_packet(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_len, s… in flow_divert_send_datagram_packet() argument
1634 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); in flow_divert_send_datagram_packet()
1636 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); in flow_divert_send_datagram_packet()
1643 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error); in flow_divert_send_datagram_packet()
1650 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_IS_FRAGMENT) failed: %d", err… in flow_divert_send_datagram_packet()
1657 …FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_append_tlv(FLOW_DIVERT_TLV_DATAGRAM_SIZE) failed: %d", e… in flow_divert_send_datagram_packet()
1668 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_datagram_packet()
1670 fd_cb->bytes_sent += data_len; in flow_divert_send_datagram_packet()
1671 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); in flow_divert_send_datagram_packet()
1688 flow_divert_send_fragmented_datagram(struct flow_divert_pcb *fd_cb, mbuf_ref_t datagram, size_t dat… in flow_divert_send_fragmented_datagram() argument
1708 …error = flow_divert_send_datagram_packet(fd_cb, next_data, to_send, (first ? toaddr : NULL), TRUE,… in flow_divert_send_fragmented_datagram()
1730 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) in flow_divert_send_buffered_data() argument
1737 to_send = fd_cb->so->so_snd.sb_cc; in flow_divert_send_buffered_data()
1738 buffer = fd_cb->so->so_snd.sb_mb; in flow_divert_send_buffered_data()
1741 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send); in flow_divert_send_buffered_data()
1746 if (!force && (to_send > fd_cb->send_window)) { in flow_divert_send_buffered_data()
1747 to_send = fd_cb->send_window; in flow_divert_send_buffered_data()
1750 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_buffered_data()
1762 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1766 error = flow_divert_send_data_packet(fd_cb, data, data_len); in flow_divert_send_buffered_data()
1776 sbdrop(&fd_cb->so->so_snd, (int)sent); in flow_divert_send_buffered_data()
1777 sowwakeup(fd_cb->so); in flow_divert_send_buffered_data()
1778 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_buffered_data()
1797 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain."); in flow_divert_send_buffered_data()
1803 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); in flow_divert_send_buffered_data()
1806 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); in flow_divert_send_buffered_data()
1813 error = flow_divert_send_datagram_packet(fd_cb, data, data_len, toaddr, FALSE, 0); in flow_divert_send_buffered_data()
1815 error = flow_divert_send_fragmented_datagram(fd_cb, data, data_len, toaddr); in flow_divert_send_buffered_data()
1827 (void) sbdroprecord(&(fd_cb->so->so_snd)); in flow_divert_send_buffered_data()
1832 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent); in flow_divert_send_buffered_data()
1833 if (fd_cb->send_window >= sent) { in flow_divert_send_buffered_data()
1834 fd_cb->send_window -= sent; in flow_divert_send_buffered_data()
1836 fd_cb->send_window = 0; in flow_divert_send_buffered_data()
1842 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_ref_t data, size_t data_size, struct … in flow_divert_send_app_data() argument
1847 if (to_send > fd_cb->send_window) { in flow_divert_send_app_data()
1848 to_send = fd_cb->send_window; in flow_divert_send_app_data()
1851 if (fd_cb->so->so_snd.sb_cc > 0) { in flow_divert_send_app_data()
1855 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_send_app_data()
1874 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_send_app_data()
1884 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len); in flow_divert_send_app_data()
1893 if (fd_cb->send_window >= sent) { in flow_divert_send_app_data()
1894 fd_cb->send_window -= sent; in flow_divert_send_app_data()
1896 fd_cb->send_window = 0; in flow_divert_send_app_data()
1902 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1903 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) { in flow_divert_send_app_data()
1904 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %… in flow_divert_send_app_data()
1905 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1914 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1915 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) { in flow_divert_send_app_data()
1916 …FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_wind… in flow_divert_send_app_data()
1917 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1924 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_send_app_data()
1928 send_dgram_error = flow_divert_send_datagram_packet(fd_cb, data, data_size, toaddr, FALSE, 0); in flow_divert_send_app_data()
1930 send_dgram_error = flow_divert_send_fragmented_datagram(fd_cb, data, data_size, toaddr); in flow_divert_send_app_data()
1934 …FDLOG(LOG_NOTICE, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = … in flow_divert_send_app_data()
1936 if (data_size >= fd_cb->send_window) { in flow_divert_send_app_data()
1937 fd_cb->send_window = 0; in flow_divert_send_app_data()
1939 fd_cb->send_window -= data_size; in flow_divert_send_app_data()
1947 if (sbspace(&fd_cb->so->so_snd) > 0) { in flow_divert_send_app_data()
1950 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &append_error)) { in flow_divert_send_app_data()
1951 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1953 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, append_error); in flow_divert_send_app_data()
1956 if (!sbappendrecord(&fd_cb->so->so_snd, data)) { in flow_divert_send_app_data()
1957 FDLOG(LOG_ERR, fd_cb, in flow_divert_send_app_data()
1959 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); in flow_divert_send_app_data()
1963 …FDLOG(LOG_ERR, fd_cb, "flow_divert_send_datagram_packet failed with error %d, send data size = %lu… in flow_divert_send_app_data()
1973 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb) in flow_divert_send_read_notification() argument
1978 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet); in flow_divert_send_read_notification()
1980 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error); in flow_divert_send_read_notification()
1984 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_read_notification()
1998 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class) in flow_divert_send_traffic_class_update() argument
2003 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet); in flow_divert_send_traffic_class_update()
2005 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error); in flow_divert_send_traffic_class_update()
2011 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error); in flow_divert_send_traffic_class_update()
2015 error = flow_divert_send_packet(fd_cb, packet); in flow_divert_send_traffic_class_update()
2029 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint) in flow_divert_set_local_endpoint() argument
2031 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_local_endpoint()
2034 …if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR… in flow_divert_set_local_endpoint()
2035 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2044 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) { in flow_divert_set_local_endpoint()
2045 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR; in flow_divert_set_local_endpoint()
2055 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint) in flow_divert_set_remote_endpoint() argument
2057 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_set_remote_endpoint()
2162 flow_divert_try_next_group(struct flow_divert_pcb *fd_cb) in flow_divert_try_next_group() argument
2165 uint32_t policy_control_unit = fd_cb->policy_control_unit; in flow_divert_try_next_group()
2167 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2172 …uint32_t next_ctl_unit = flow_divert_derive_kernel_control_unit(0, &policy_control_unit, &(fd_cb->… in flow_divert_try_next_group()
2174 if (fd_cb->control_group_unit == next_ctl_unit) { in flow_divert_try_next_group()
2175 …FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling fl… in flow_divert_try_next_group()
2181 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert"); in flow_divert_try_next_group()
2186 next_group = flow_divert_group_lookup(next_ctl_unit, fd_cb); in flow_divert_try_next_group()
2188 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit); in flow_divert_try_next_group()
2192 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", fd_cb->control_group_unit, next_ctl_unit); in flow_divert_try_next_group()
2194 error = flow_divert_pcb_insert(fd_cb, next_group); in flow_divert_try_next_group()
2197 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2199 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_try_next_group()
2203 } while (fd_cb->group == NULL); in flow_divert_try_next_group()
2205 if (fd_cb->group == NULL) { in flow_divert_try_next_group()
2209 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_try_next_group()
2211 …FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", fd_cb->… in flow_divert_try_next_group()
2212 flow_divert_pcb_remove(fd_cb); in flow_divert_try_next_group()
2220 flow_divert_disable(struct flow_divert_pcb *fd_cb) in flow_divert_disable() argument
2226 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint; in flow_divert_disable()
2227 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT); in flow_divert_disable()
2230 so = fd_cb->so; in flow_divert_disable()
2235 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert"); in flow_divert_disable()
2239 inp->inp_vflag = fd_cb->original_vflag; in flow_divert_disable()
2246 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) { in flow_divert_disable()
2251 inp->inp_last_outifp = fd_cb->original_last_outifp; in flow_divert_disable()
2252 inp->in6p_last_outifp = fd_cb->original_last_outifp6; in flow_divert_disable()
2258 fd_cb->so = NULL; in flow_divert_disable()
2260 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_disable()
2274 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error); in flow_divert_disable()
2292 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer"); in flow_divert_disable()
2310 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued TCP data using the socket's original protocol: %d", e… in flow_divert_disable()
2362 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer"); in flow_divert_disable()
2367 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data"); in flow_divert_disable()
2373 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header"); in flow_divert_disable()
2398 …FDLOG(LOG_ERR, fd_cb, "Failed to send queued UDP data using the socket's original protocol: %d", e… in flow_divert_disable()
2414 flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address) in flow_divert_scope() argument
2422 so = fd_cb->so; in flow_divert_scope()
2448 …FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error); in flow_divert_scope()
2452 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) { in flow_divert_scope()
2456 …error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr)… in flow_divert_scope()
2459 …error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), I… in flow_divert_scope()
2463 …FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr return… in flow_divert_scope()
2491 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_connect_result() argument
2507 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error); in flow_divert_handle_connect_result()
2512 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error); in flow_divert_handle_connect_result()
2516 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error); in flow_divert_handle_connect_result()
2522 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result"); in flow_divert_handle_connect_result()
2527 FDLOG0(LOG_INFO, fd_cb, "No local address provided"); in flow_divert_handle_connect_result()
2532 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_connect_result()
2537 FDLOG0(LOG_INFO, fd_cb, "No output if index provided"); in flow_divert_handle_connect_result()
2542 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result"); in flow_divert_handle_connect_result()
2547 FDLOCK(fd_cb); in flow_divert_handle_connect_result()
2548 if (fd_cb->so != NULL) { in flow_divert_handle_connect_result()
2550 struct socket *so = fd_cb->so; in flow_divert_handle_connect_result()
2556 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result"); in flow_divert_handle_connect_result()
2561 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result"); in flow_divert_handle_connect_result()
2576 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2579 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port; in flow_divert_handle_connect_result()
2585 fd_cb->local_endpoint = local_endpoint; in flow_divert_handle_connect_result()
2588 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port; in flow_divert_handle_connect_result()
2593 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid); in flow_divert_handle_connect_result()
2594 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_connect_result()
2602 flow_divert_set_remote_endpoint(fd_cb, SA(&remote_endpoint)); in flow_divert_handle_connect_result()
2611 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length); in flow_divert_handle_connect_result()
2612 if (fd_cb->app_data != NULL) { in flow_divert_handle_connect_result()
2613 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_connect_result()
2615 fd_cb->app_data = app_data; in flow_divert_handle_connect_result()
2616 fd_cb->app_data_length = app_data_length; in flow_divert_handle_connect_result()
2618 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet"… in flow_divert_handle_connect_result()
2622 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_connect_result()
2630 if (fd_cb->group == NULL) { in flow_divert_handle_connect_result()
2641 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit); in flow_divert_handle_connect_result()
2646 grp = flow_divert_group_lookup(ctl_unit, fd_cb); in flow_divert_handle_connect_result()
2652 flow_divert_pcb_remove(fd_cb); in flow_divert_handle_connect_result()
2653 insert_error = flow_divert_pcb_insert(fd_cb, grp); in flow_divert_handle_connect_result()
2662 fd_cb->send_window = ntohl(send_window); in flow_divert_handle_connect_result()
2666 FDLOG0(LOG_INFO, fd_cb, "sending connect result"); in flow_divert_handle_connect_result()
2667 error = flow_divert_send_connect_result(fd_cb); in flow_divert_handle_connect_result()
2671 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) { in flow_divert_handle_connect_result()
2672 error = flow_divert_try_next_group(fd_cb); in flow_divert_handle_connect_result()
2673 if (error && fd_cb->policy_control_unit == 0) { in flow_divert_handle_connect_result()
2674 flow_divert_disable(fd_cb); in flow_divert_handle_connect_result()
2682 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_handle_connect_result()
2684 flow_divert_send_close_if_needed(fd_cb); in flow_divert_handle_connect_result()
2686 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_handle_connect_result()
2689 flow_divert_disconnect_socket(so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_handle_connect_result()
2704 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_connect_result()
2709 if (fd_cb->connect_packet != NULL) { in flow_divert_handle_connect_result()
2710 mbuf_freem(fd_cb->connect_packet); in flow_divert_handle_connect_result()
2711 fd_cb->connect_packet = NULL; in flow_divert_handle_connect_result()
2715 free_sockaddr(fd_cb->original_remote_endpoint); in flow_divert_handle_connect_result()
2719 FDUNLOCK(fd_cb); in flow_divert_handle_connect_result()
2723 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_close() argument
2731 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error); in flow_divert_handle_close()
2737 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error); in flow_divert_handle_close()
2743 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how); in flow_divert_handle_close()
2745 FDLOCK(fd_cb); in flow_divert_handle_close()
2746 if (fd_cb->so != NULL) { in flow_divert_handle_close()
2747 …bool is_connected = (SOCK_TYPE(fd_cb->so) == SOCK_STREAM || !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_… in flow_divert_handle_close()
2748 socket_lock(fd_cb->so, 0); in flow_divert_handle_close()
2750 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_close()
2751 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider"); in flow_divert_handle_close()
2755 fd_cb->so->so_error = (uint16_t)ntohl(close_error); in flow_divert_handle_close()
2757 flow_divert_update_closed_state(fd_cb, how, true, true); in flow_divert_handle_close()
2760 how = flow_divert_tunnel_how_closed(fd_cb); in flow_divert_handle_close()
2762 flow_divert_disconnect_socket(fd_cb->so, is_connected, true); in flow_divert_handle_close()
2764 socantrcvmore(fd_cb->so); in flow_divert_handle_close()
2766 socantsendmore(fd_cb->so); in flow_divert_handle_close()
2769 socket_unlock(fd_cb->so, 0); in flow_divert_handle_close()
2771 FDUNLOCK(fd_cb); in flow_divert_handle_close()
2775 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb) in flow_divert_create_control_mbuf() argument
2777 struct inpcb *inp = sotoinpcb(fd_cb->so); in flow_divert_create_control_mbuf()
2782 fd_cb->local_endpoint.sa.sa_family == AF_INET && in flow_divert_create_control_mbuf()
2784 …return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_… in flow_divert_create_control_mbuf()
2786 fd_cb->local_endpoint.sa.sa_family == AF_INET6 && in flow_divert_create_control_mbuf()
2790 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr; in flow_divert_create_control_mbuf()
2798 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, size_t offset) in flow_divert_handle_data() argument
2802 FDLOCK(fd_cb); in flow_divert_handle_data()
2803 if (fd_cb->so != NULL) { in flow_divert_handle_data()
2811 socket_lock(fd_cb->so, 0); in flow_divert_handle_data()
2813 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_data()
2814 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data"); in flow_divert_handle_data()
2818 if (sbspace(&fd_cb->so->so_rcv) == 0) { in flow_divert_handle_data()
2820 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_handle_data()
2821 …FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some d… in flow_divert_handle_data()
2825 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { in flow_divert_handle_data()
2832 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); in flow_divert_handle_data()
2842 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid"); in flow_divert_handle_data()
2850 if (fd_cb->so->so_state & SS_CANTRCVMORE) { in flow_divert_handle_data()
2851 …FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size… in flow_divert_handle_data()
2855 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) { in flow_divert_handle_data()
2856 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so)); in flow_divert_handle_data()
2860 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size); in flow_divert_handle_data()
2864 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); in flow_divert_handle_data()
2868 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) { in flow_divert_handle_data()
2869 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0); in flow_divert_handle_data()
2878 if (SOCK_CHECK_DOM(fd_cb->so, AF_INET6)) { in flow_divert_handle_data()
2879 error = in6_mapped_peeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2881 error = in_getpeeraddr(fd_cb->so, &append_sa); in flow_divert_handle_data()
2885 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address."); in flow_divert_handle_data()
2888 mctl = flow_divert_create_control_mbuf(fd_cb); in flow_divert_handle_data()
2890 appended = sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error); in flow_divert_handle_data()
2894 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error); in flow_divert_handle_data()
2901 fd_cb->bytes_received += data_size; in flow_divert_handle_data()
2902 flow_divert_add_data_statistics(fd_cb, data_size, FALSE); in flow_divert_handle_data()
2906 sorwakeup(fd_cb->so); in flow_divert_handle_data()
2909 socket_unlock(fd_cb->so, 0); in flow_divert_handle_data()
2911 FDUNLOCK(fd_cb); in flow_divert_handle_data()
2917 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_read_notification() argument
2924 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error); in flow_divert_handle_read_notification()
2928 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count)); in flow_divert_handle_read_notification()
2930 FDLOCK(fd_cb); in flow_divert_handle_read_notification()
2931 if (fd_cb->so != NULL) { in flow_divert_handle_read_notification()
2932 socket_lock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2934 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_read_notification()
2935 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification"); in flow_divert_handle_read_notification()
2939 fd_cb->send_window += ntohl(read_count); in flow_divert_handle_read_notification()
2940 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_handle_read_notification()
2942 socket_unlock(fd_cb->so, 0); in flow_divert_handle_read_notification()
2944 FDUNLOCK(fd_cb); in flow_divert_handle_read_notification()
3009 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_ref_t packet, int offset) in flow_divert_handle_properties_update() argument
3015 FDLOG0(LOG_INFO, fd_cb, "received a properties update"); in flow_divert_handle_properties_update()
3019 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update"); in flow_divert_handle_properties_update()
3024 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update"); in flow_divert_handle_properties_update()
3027 FDLOCK(fd_cb); in flow_divert_handle_properties_update()
3028 if (fd_cb->so != NULL) { in flow_divert_handle_properties_update()
3029 socket_lock(fd_cb->so, 0); in flow_divert_handle_properties_update()
3031 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) { in flow_divert_handle_properties_update()
3032 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update"); in flow_divert_handle_properties_update()
3037 flow_divert_scope(fd_cb, out_if_index, true); in flow_divert_handle_properties_update()
3038 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_handle_properties_update()
3047 if (fd_cb->app_data != NULL) { in flow_divert_handle_properties_update()
3048 kfree_data_sized_by(fd_cb->app_data, fd_cb->app_data_length); in flow_divert_handle_properties_update()
3050 fd_cb->app_data = app_data; in flow_divert_handle_properties_update()
3051 fd_cb->app_data_length = app_data_length; in flow_divert_handle_properties_update()
3053 …FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update pack… in flow_divert_handle_properties_update()
3057 …FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from th… in flow_divert_handle_properties_update()
3061 socket_unlock(fd_cb->so, 0); in flow_divert_handle_properties_update()
3063 FDUNLOCK(fd_cb); in flow_divert_handle_properties_update()
3239 struct flow_divert_pcb *fd_cb; in flow_divert_handle_flow_states_request() local
3261 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_handle_flow_states_request()
3262 FDRETAIN(fd_cb); in flow_divert_handle_flow_states_request()
3263 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_handle_flow_states_request()
3268 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { in flow_divert_handle_flow_states_request()
3269 FDLOCK(fd_cb); in flow_divert_handle_flow_states_request()
3270 if (fd_cb->so != NULL) { in flow_divert_handle_flow_states_request()
3272 socket_lock(fd_cb->so, 0); in flow_divert_handle_flow_states_request()
3274 state.conn_id = fd_cb->hash; in flow_divert_handle_flow_states_request()
3275 state.bytes_written_by_app = fd_cb->bytes_written_by_app; in flow_divert_handle_flow_states_request()
3276 state.bytes_sent = fd_cb->bytes_sent; in flow_divert_handle_flow_states_request()
3277 state.bytes_received = fd_cb->bytes_received; in flow_divert_handle_flow_states_request()
3278 state.send_window = fd_cb->send_window; in flow_divert_handle_flow_states_request()
3279 state.send_buffer_bytes = fd_cb->so->so_snd.sb_cc; in flow_divert_handle_flow_states_request()
3283 FDLOG(LOG_ERR, fd_cb, "Failed to add a flow state: %d", error); in flow_divert_handle_flow_states_request()
3286 socket_unlock(fd_cb->so, 0); in flow_divert_handle_flow_states_request()
3288 FDUNLOCK(fd_cb); in flow_divert_handle_flow_states_request()
3289 FDRELEASE(fd_cb); in flow_divert_handle_flow_states_request()
3304 struct flow_divert_pcb *fd_cb; in flow_divert_input() local
3339 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */ in flow_divert_input()
3340 if (fd_cb == NULL) { in flow_divert_input()
3349 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3352 flow_divert_handle_close(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3355 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3358 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3361 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr)); in flow_divert_input()
3364 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type); in flow_divert_input()
3368 FDRELEASE(fd_cb); in flow_divert_input()
3378 struct flow_divert_pcb *fd_cb; in flow_divert_close_all() local
3387 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_close_all()
3388 FDRETAIN(fd_cb); in flow_divert_close_all()
3389 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_close_all()
3397 fd_cb = SLIST_FIRST(&tmp_list); in flow_divert_close_all()
3398 FDLOCK(fd_cb); in flow_divert_close_all()
3400 if (fd_cb->so != NULL) { in flow_divert_close_all()
3401 socket_lock(fd_cb->so, 0); in flow_divert_close_all()
3402 flow_divert_pcb_remove(fd_cb); in flow_divert_close_all()
3403 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, true, true); in flow_divert_close_all()
3404 fd_cb->so->so_error = ECONNABORTED; in flow_divert_close_all()
3405 flow_divert_disconnect_socket(fd_cb->so, !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT), false); in flow_divert_close_all()
3406 socket_unlock(fd_cb->so, 0); in flow_divert_close_all()
3408 FDUNLOCK(fd_cb); in flow_divert_close_all()
3409 FDRELEASE(fd_cb); in flow_divert_close_all()
3416 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_detach() local
3425 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count); in flow_divert_detach()
3427 if (fd_cb->group != NULL) { in flow_divert_detach()
3429 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_detach()
3431 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_detach()
3432 flow_divert_send_close_if_needed(fd_cb); in flow_divert_detach()
3434 flow_divert_pcb_remove(fd_cb); in flow_divert_detach()
3438 FDLOCK(fd_cb); in flow_divert_detach()
3439 fd_cb->so = NULL; in flow_divert_detach()
3440 FDUNLOCK(fd_cb); in flow_divert_detach()
3443 FDRELEASE(fd_cb); /* Release the socket's reference */ in flow_divert_detach()
3449 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_close() local
3455 FDLOG0(LOG_INFO, fd_cb, "Closing"); in flow_divert_close()
3462 flow_divert_send_buffered_data(fd_cb, TRUE); in flow_divert_close()
3463 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, false, true); in flow_divert_close()
3464 flow_divert_send_close_if_needed(fd_cb); in flow_divert_close()
3467 flow_divert_pcb_remove(fd_cb); in flow_divert_close()
3486 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_shutdown() local
3492 FDLOG0(LOG_INFO, fd_cb, "Can't send more"); in flow_divert_shutdown()
3496 flow_divert_update_closed_state(fd_cb, SHUT_WR, false, true); in flow_divert_shutdown()
3497 flow_divert_send_close_if_needed(fd_cb); in flow_divert_shutdown()
3505 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_rcvd() local
3513 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space); in flow_divert_rcvd()
3514 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) && in flow_divert_rcvd()
3516 flow_divert_send_read_notification(fd_cb) == 0) { in flow_divert_rcvd()
3517 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification"); in flow_divert_rcvd()
3518 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED; in flow_divert_rcvd()
3658 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_ctloutput() local
3665 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_ctloutput()
3666 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class); in flow_divert_ctloutput()
3681 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out_internal() local
3692 if (fd_cb->group == NULL) { in flow_divert_connect_out_internal()
3710 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { in flow_divert_connect_out_internal()
3715 FDLOG0(LOG_INFO, fd_cb, "Connecting"); in flow_divert_connect_out_internal()
3717 if (fd_cb->connect_packet == NULL) { in flow_divert_connect_out_internal()
3722 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet"); in flow_divert_connect_out_internal()
3728 FDLOG0(LOG_ERR, fd_cb, "Destination address is not valid when creating connect packet"); in flow_divert_connect_out_internal()
3733 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0); in flow_divert_connect_out_internal()
3734 if (fd_cb->original_remote_endpoint == NULL) { in flow_divert_connect_out_internal()
3735 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint"); in flow_divert_connect_out_internal()
3739 fd_cb->original_vflag = inp->inp_vflag; in flow_divert_connect_out_internal()
3740 fd_cb->original_last_outifp = inp->inp_last_outifp; in flow_divert_connect_out_internal()
3741 fd_cb->original_last_outifp6 = inp->in6p_last_outifp; in flow_divert_connect_out_internal()
3766 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6); in flow_divert_connect_out_internal()
3767 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6; in flow_divert_connect_out_internal()
3768 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport; in flow_divert_connect_out_internal()
3769 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp); in flow_divert_connect_out_internal()
3771 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error); in flow_divert_connect_out_internal()
3772 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || IN6_IS_ADDR_UNSPECIFIED(&(satosin6(to)->s… in flow_divert_connect_out_internal()
3783 if (IN6_IS_SCOPE_EMBED(&(fd_cb->local_endpoint.sin6.sin6_addr)) && in flow_divert_connect_out_internal()
3785 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] != 0) { in flow_divert_connect_out_internal()
3786 …fd_cb->local_endpoint.sin6.sin6_scope_id = ntohs(fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1]… in flow_divert_connect_out_internal()
3787 fd_cb->local_endpoint.sin6.sin6_addr.s6_addr16[1] = 0; in flow_divert_connect_out_internal()
3799 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in); in flow_divert_connect_out_internal()
3800 fd_cb->local_endpoint.sin.sin_family = AF_INET; in flow_divert_connect_out_internal()
3801 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport; in flow_divert_connect_out_internal()
3802 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0); in flow_divert_connect_out_internal()
3804 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error); in flow_divert_connect_out_internal()
3805 …if (!(fd_cb->flags & FLOW_DIVERT_FLOW_IS_TRANSPARENT) || satosin(to)->sin_addr.s_addr == INADDR_AN… in flow_divert_connect_out_internal()
3816 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family); in flow_divert_connect_out_internal()
3819 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_connect_out_internal()
3820 flow_divert_check_no_expensive(fd_cb) || in flow_divert_connect_out_internal()
3821 flow_divert_check_no_constrained(fd_cb); in flow_divert_connect_out_internal()
3830 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR; in flow_divert_connect_out_internal()
3833 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet); in flow_divert_connect_out_internal()
3839 flow_divert_set_remote_endpoint(fd_cb, to); in flow_divert_connect_out_internal()
3840 flow_divert_set_local_endpoint(fd_cb, SA(&fd_cb->local_endpoint)); in flow_divert_connect_out_internal()
3844 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT; in flow_divert_connect_out_internal()
3848 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive"); in flow_divert_connect_out_internal()
3852 fd_cb->connect_packet = connect_packet; in flow_divert_connect_out_internal()
3855 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet"); in flow_divert_connect_out_internal()
3859 error = flow_divert_send_connect_packet(fd_cb); in flow_divert_connect_out_internal()
3864 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_connect_out_internal()
3867 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) { in flow_divert_connect_out_internal()
3884 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connect_out() local
3885 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error); in flow_divert_connect_out()
3908 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_connectx_out_common() local
3909 if (fd_cb != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_TOKEN) && in flow_divert_connectx_out_common()
3984 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_data_out() local
4001 if ((fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) && SOCK_TYPE(so) == SOCK_DGRAM) { in flow_divert_data_out()
4003 FDLOG0(LOG_INFO, fd_cb, "provider is no longer accepting writes, cannot send data"); in flow_divert_data_out()
4019 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to); in flow_divert_data_out()
4024 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_data_out()
4025 FDLOG0(LOG_INFO, fd_cb, "implicit connect"); in flow_divert_data_out()
4032 error = flow_divert_check_no_cellular(fd_cb) || in flow_divert_data_out()
4033 flow_divert_check_no_expensive(fd_cb) || in flow_divert_data_out()
4034 flow_divert_check_no_constrained(fd_cb); in flow_divert_data_out()
4050 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", data_size); in flow_divert_data_out()
4051 fd_cb->bytes_written_by_app += data_size; in flow_divert_data_out()
4053 error = flow_divert_send_app_data(fd_cb, data, data_size, to); in flow_divert_data_out()
4086 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_preconnect() local
4092 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { in flow_divert_preconnect()
4093 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet"); in flow_divert_preconnect()
4099 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; in flow_divert_preconnect()
4130 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out() local
4139 if (fd_cb == NULL) { in flow_divert_implicit_data_out()
4141 fd_cb = so->so_fd_pcb; in flow_divert_implicit_data_out()
4142 if (error != 0 || fd_cb == NULL) { in flow_divert_implicit_data_out()
4163 struct flow_divert_pcb *fd_cb = NULL; in flow_divert_pcb_init_internal() local
4172 fd_cb = flow_divert_pcb_create(so); in flow_divert_pcb_init_internal()
4173 if (fd_cb == NULL) { in flow_divert_pcb_init_internal()
4180 FDLOG0(LOG_ERR, fd_cb, "No valid group is available, cannot init flow divert"); in flow_divert_pcb_init_internal()
4185 error = flow_divert_add_to_group(fd_cb, group_unit); in flow_divert_pcb_init_internal()
4187 so->so_fd_pcb = fd_cb; in flow_divert_pcb_init_internal()
4189 fd_cb->control_group_unit = group_unit; in flow_divert_pcb_init_internal()
4190 fd_cb->policy_control_unit = ctl_unit; in flow_divert_pcb_init_internal()
4191 fd_cb->aggregate_unit = agg_unit; in flow_divert_pcb_init_internal()
4193 fd_cb->flags |= FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4195 fd_cb->flags &= ~FLOW_DIVERT_FLOW_IS_TRANSPARENT; in flow_divert_pcb_init_internal()
4204 FDLOG0(LOG_INFO, fd_cb, "Created"); in flow_divert_pcb_init_internal()
4206 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error); in flow_divert_pcb_init_internal()
4211 FDRELEASE(fd_cb); in flow_divert_pcb_init_internal()
4314 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4319 fd_cb->log_level = (uint8_t)log_level; in flow_divert_token_set()
4323 fd_cb->connect_token = token; in flow_divert_token_set()
4326 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN; in flow_divert_token_set()
4330 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_set() local
4331 if (fd_cb != NULL) { in flow_divert_token_set()
4332 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC; in flow_divert_token_set()
4350 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; in flow_divert_token_get() local
4359 if (fd_cb->group == NULL) { in flow_divert_token_get()
4366 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); in flow_divert_token_get()
4370 ctl_unit = htonl(fd_cb->group->ctl_unit); in flow_divert_token_get()
4377 …error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb-… in flow_divert_token_get()
4382 if (fd_cb->app_data != NULL) { in flow_divert_token_get()
4383 …_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_d… in flow_divert_token_get()
4389 control_group = flow_divert_group_lookup(fd_cb->control_group_unit, fd_cb); in flow_divert_token_get()
4650 struct flow_divert_pcb *fd_cb; in flow_divert_kctl_rcvd() local
4671 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { in flow_divert_kctl_rcvd()
4672 FDRETAIN(fd_cb); in flow_divert_kctl_rcvd()
4673 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); in flow_divert_kctl_rcvd()
4678 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { in flow_divert_kctl_rcvd()
4679 FDLOCK(fd_cb); in flow_divert_kctl_rcvd()
4680 if (fd_cb->so != NULL) { in flow_divert_kctl_rcvd()
4681 socket_lock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4682 if (fd_cb->group != NULL) { in flow_divert_kctl_rcvd()
4683 flow_divert_send_buffered_data(fd_cb, FALSE); in flow_divert_kctl_rcvd()
4685 socket_unlock(fd_cb->so, 0); in flow_divert_kctl_rcvd()
4687 FDUNLOCK(fd_cb); in flow_divert_kctl_rcvd()
4688 FDRELEASE(fd_cb); in flow_divert_kctl_rcvd()