/* * Copyright (c) 2010-2024 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // These includes appear in ntstat.h but we include them here first so they won't trigger // any clang diagnostic errors. #include #include #include #pragma clang diagnostic push #pragma clang diagnostic error "-Wpadded" #pragma clang diagnostic error "-Wpacked" // This header defines structures shared with user space, so we need to ensure there is // no compiler inserted padding in case the user space process isn't using the same // architecture as the kernel (example: i386 process with x86_64 kernel). #include #pragma clang diagnostic pop #include #include #include #include #include #include #include #include #include #include #include __private_extern__ int nstat_collect = 1; #define NSTAT_TRACE_ENABLED 0 #define NSTAT_FUZZ_TIMING 0 #if (DEBUG || DEVELOPMENT) SYSCTL_INT(_net, OID_AUTO, statistics, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_collect, 0, "Collect detailed statistics"); #endif /* (DEBUG || DEVELOPMENT) */ #if !XNU_TARGET_OS_OSX static int nstat_privcheck = 1; #else /* XNU_TARGET_OS_OSX */ static int nstat_privcheck = 0; #endif /* XNU_TARGET_OS_OSX */ SYSCTL_INT(_net, OID_AUTO, statistics_privcheck, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_privcheck, 0, "Entitlement check"); SYSCTL_NODE(_net, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "network statistics"); static int nstat_debug = 0; SYSCTL_INT(_net_stats, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_debug, 0, ""); static int nstat_debug_pid = 0; // Only log socket level debug for specified pid SYSCTL_INT(_net_stats, OID_AUTO, debug_pid, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_debug_pid, 0, ""); static int nstat_sendspace = 2048; SYSCTL_INT(_net_stats, OID_AUTO, sendspace, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_sendspace, 0, ""); static int nstat_recvspace = 8192; SYSCTL_INT(_net_stats, OID_AUTO, recvspace, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_recvspace, 0, ""); static struct nstat_stats nstat_stats; SYSCTL_STRUCT(_net_stats, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_LOCKED, &nstat_stats, nstat_stats, ""); static struct nstat_global_counts nstat_global_counts; SYSCTL_STRUCT(_net_stats, OID_AUTO, global_counts, CTLFLAG_RD | CTLFLAG_LOCKED, &nstat_global_counts, nstat_global_counts, ""); static u_int32_t nstat_lim_interval = 30 * 60; /* Report interval, seconds */ static u_int32_t nstat_lim_min_tx_pkts = 100; static u_int32_t nstat_lim_min_rx_pkts = 100; #if (DEBUG || DEVELOPMENT) SYSCTL_INT(_net_stats, OID_AUTO, lim_report_interval, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_interval, 0, "Low internet stat report interval"); SYSCTL_INT(_net_stats, OID_AUTO, lim_min_tx_pkts, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_tx_pkts, 0, "Low Internet, min transmit packets threshold"); SYSCTL_INT(_net_stats, OID_AUTO, lim_min_rx_pkts, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_rx_pkts, 0, "Low Internet, min receive packets threshold"); #endif /* DEBUG || DEVELOPMENT */ static int ntstat_progress_indicators(struct sysctl_req *); static int ntstat_get_metrics(struct sysctl_req *); static struct net_api_stats net_api_stats_before; static u_int64_t net_api_stats_last_report_time; #define NET_API_STATS_REPORT_INTERVAL (12 * 60 * 60) /* 12 hours, in seconds */ static u_int32_t net_api_stats_report_interval = NET_API_STATS_REPORT_INTERVAL; #if (DEBUG || DEVELOPMENT) SYSCTL_UINT(_net_stats, OID_AUTO, api_report_interval, CTLFLAG_RW | CTLFLAG_LOCKED, &net_api_stats_report_interval, 0, ""); #endif /* DEBUG || DEVELOPMENT */ static os_log_t nstat_log_handle = NULL; #define _NSTAT_LOG(handle, type, fmt, ...) do { \ os_log_with_type(handle, type, "%s - " fmt, __func__, ##__VA_ARGS__); \ } while (0) #define NSTAT_LOG(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_DEFAULT, fmt, ##__VA_ARGS__) #define NSTAT_LOG_DEBUG(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_DEBUG, fmt, ##__VA_ARGS__) #define NSTAT_LOG_INFO(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_INFO, fmt, ##__VA_ARGS__) #define NSTAT_LOG_ERROR(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_ERROR, fmt, ##__VA_ARGS__) #define NSTAT_DEBUG_SOCKET_PID_MATCHED(so) \ (so && (nstat_debug_pid == (so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid))) #define NSTAT_DEBUG_SOCKET_ON(so) \ ((nstat_debug && (!nstat_debug_pid || NSTAT_DEBUG_SOCKET_PID_MATCHED(so))) ? nstat_debug : 0) #define NSTAT_DEBUG_SOCKET_LOG(so, fmt, ...) \ if (NSTAT_DEBUG_SOCKET_ON(so)) { \ NSTAT_LOG_DEBUG("NSTAT_DEBUG_SOCKET : " fmt, (so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid), ##__VA_ARGS__); \ } enum{ NSTAT_FLAG_CLEANUP = (1 << 0), NSTAT_FLAG_REQCOUNTS = (1 << 1), NSTAT_FLAG_SUPPORTS_UPDATES = (1 << 2), NSTAT_FLAG_SYSINFO_SUBSCRIBED = (1 << 3), NSTAT_FLAG_SUPPORTS_DETAILS = (1 << 4), }; static int progress_indicators SYSCTL_HANDLER_ARGS { return ntstat_progress_indicators(req); } SYSCTL_PROC(_net_stats, OID_AUTO, progress, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, 0, 0, progress_indicators, "S", "Various items that indicate the current state of progress on the link"); #if !XNU_TARGET_OS_OSX #define QUERY_CONTINUATION_SRC_COUNT 50 #else /* XNU_TARGET_OS_OSX */ #define QUERY_CONTINUATION_SRC_COUNT 100 #endif /* XNU_TARGET_OS_OSX */ #define QUERY_CONTINUATION_MIN_SRC_COUNT 5 #ifndef ROUNDUP64 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t)) #endif #ifndef ADVANCE64 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n)) #endif /* * A note on data structures * * Each user space "client" of NetworkStatistics is represented by a nstat_client structure. * Communication between user space and kernel is currently via a kernel control socket, * but other mechanisms could theoretically be used * * Each ntstat_client has a queue of nstat_src structures. These typically represent flows * but could represent whatever the relevant provider makes available to be reported to user level. * The "nstat_src" name is perhaps a little unfortunate as these don't directly refer to the socket, channel * or whatever is the ultimate source of the information but instead map directly to each client's idea of * that source. They therefore contain things like the source reference quoted to the client, which is * unique only on a per client basis. Currently each client has source references which are allocated sequentially * starting from one * * There are multiple "providers", one each for things like TCP sockets, QUIC channels, routes etc. * The ntstat_src structure contains the nts_provider field to identify the particular provider and the * nts_cookie field, which is a pointer to a provider-dependent structure * * The nstat_src structure has a pointer to a provider dependent "cookie" structure * Some, for example the QUIC channel provider, may have a single instance of a structure * for all nstat_srcs. Optionally the provider code may use an nstat_locus structure to allow it * to quickly iterate all the associated nstat_srcs. The alternative is to iterate over the nstat_srcs * belonging to each of the nstat_clients and look for any matches. * * That gives rise to the example picture below, with notes as follows: * 1) The second nstat_client has been in existence for longer than the first one, * and the larger nstat_srcref numbers reflect flows that were assigned numbers earlier * but which have now closed. * 2) The nts_cookie field contains a pointer to a provider dependent structure, * here either a nstat_tu_shadow, nstat_sock_locus or rtentry, for channel based or socket based flows * or routes respectively * 3) The nstat_tu_shadow and nstat_sock_locus structures contain an embedded nstat_locus * 4) The nstat_tu_shadow structures are linked, and the links are traversed when collecting all * the associated flows when there is a new "watcher". In contrast, the nstat_sock_locus structures are not * linked and the ntstat code must understand how to traverse the relevant structures in the BSD socket code * 5) For simplicity, various linkages are not shown. In the nstat_src we have: * - nts_client is a back pointer to the owning nstat_client * - nts_locus is a pointer to the associated nstat_locus, or NULL if there is no such item * - nts_provider is a pointer to a provider-dependent structure with a list of function pointers * * * Generic Code . Per Provider Code * . * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . * . * nstat_clients ---+ . * | . Channel based flows * +-----------<---------+ . * | . * | . * V . * | nstat_client nstat_client . nstat_userprot_shad_head --+ * | +-----------------+ +-----------------+ . | * +->| ntc_next |---------->| ntc_next | . +-----------<--------------+ * +-----------------+ +-----------------+ . | * | | | | . | * +-----------------+ +-----------------+ . | * | ntc_src_queue |---+ | ntc_src_queue |---+ . | * +-----------------+ | +-----------------+ | . | * | | | | | | . | * | | . | * | | V | | V . V * | | . | * | | | | | | . | * +-----------------+ | +-----------------+ | . | * | | . | * | | . | * +---------<------------+ +--------<-------------+ . | * | | . | * | | . | * | | . | * | +-------------------------- | -------<-------------------.- | -------<-----------------+ * | | | . | | * | | nstat_src | nstat_src . | | * | | +-----------------+ | +-----------------+ . | | * +--->| nts_client_link |---+ +--->| nts_client_link |---+ . | | * | +-----------------+ | +-----------------+ | . | | * -->| nts_locus_link |---|------>| nts_locus_link | | . V ^ * +-----------------+ | +-----------------+ | . | | * | nts_locus | | | nts_locus | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_client | | | nts_client | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_srcref = 5 | | | nts_srcref = 28 | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_provider | | | nts_provider | | . | nstat_tu_shadow | * +-----------------+ | +-----------------+ | . | +-----------------+ | * | nts_cookie |-->|---+ | nts_coookie |-->|-.--->-->| shad_locus |---+ * +-----------------+ | | +-----------------+ | . | +-----------------+ * | nts_filter | | | | nts_filter | | . | | shad_link |---+ * +-----------------+ | | +-----------------+ | . | +-----------------+ | * | | | | | | | . | | | | * | | | | | | | . ^ +-----------------+ | * +-----------------+ | | +-----------------+ | . | | | | * nstat_src struct | | | . | +-----------------+ | * | +------------>------------|-.--+ | | | * | | . | | | * | | . | | | * | | . | | | * +---------<----------------+ +--------<-----------------+ . +-----------------+ | * | | . | * | | . | * | | . +--------<-----------------+ * | | . | * | | . | * | +-------------------------- | -------<-------------------.- | -------<-----------------+ * | | | . | | * | | nstat_src | nstat_src . | | * | | +-----------------+ | +-----------------+ . | | * +--->| nts_client_link |---+ +--->| nts_client_link |---+ . | | * | +-----------------+ | +-----------------+ | . | | * -->| nts_locus_link |---|------>| nts_locus_link | | . V ^ * +-----------------+ | +-----------------+ | . | | * | nts_locus | | | nts_locus | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_client | | | nts_client | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_srcref = 4 | | | nts_srcref = 27 | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_provider | | | nts_provider | | . | nstat_tu_shadow | * +-----------------+ | +-----------------+ | . | +-----------------+ | * | nts_cookie |-->|---+ | nts_coookie |-->|----->-->| shad_locus |---+ * +-----------------+ | | +-----------------+ | . | +-----------------+ * | nts_filter | | | | nts_filter | | . | | shad_link | * +-----------------+ | | +-----------------+ | . | +-----------------+ * | | | | | | | . | | | * | | | | | | | . ^ +-----------------+ * +-----------------+ | | +-----------------+ | . | | | * | | | . | +-----------------+ * | +------------>------------|----+ | | * | | . | | * | | . | | * | | . | | * +---------<----------------+ +--------<-----------------+ . +-----------------+ * | | . * | | . * | | . . . . . . . . . . . . . . . . . . . . . . . . . . . . . * | | . * | | . Socket based flows * | | . * | | . * | | . nstat_tcp_sock_locus_head -+ * | | . | * | | . +-----------<--------------+ * | | . | * | | . | * | +-------------------------- | -------<-------------------.- | -------<-----------------+ * | | | . | | * | | nstat_src | nstat_src . | | * | | +-----------------+ | +-----------------+ . | | * +--->| nts_client_link |---+ +--->| nts_client_link |---+ . | | * | +-----------------+ | +-----------------+ | . | | * -->| nts_locus_link |---|------>| nts_locus_link | | . V ^ * +-----------------+ | +-----------------+ | . | | * | nts_locus | | | nts_locus | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_client | | | nts_client | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_srcref = 3 | | | nts_srcref = 26 | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_provider | | | nts_provider | | . | nstat_sock_locus | inpcb * +-----------------+ | +-----------------+ | . | +-----------------+ | +-----------------+ * | nts_cookie |-->|---+ | nts_coookie |-->|-.--->-->| nsl_locus |---+ | | * +-----------------+ | | +-----------------+ | . | +-----------------+ | | * | nts_filter | | | | nts_filter | | . | | nsl_link |---+ | | * +-----------------+ | | +-----------------+ | . | +-----------------+ | | | * | | | | | | | . | | nsl_inp |-- |-->| | * | | | | | | | . ^ +-----------------+ | * +-----------------+ | | +-----------------+ | . | | | | | | * nstat_src struct | | | . | +-----------------+ | * | +------------>------------|-.--+ | nsl_pname | | | | * | | . | | | * | | . | | | | | * | | . | | | * +---------<----------------+ +--------<-----------------+ . +-----------------+ | | | * | | . | * | | . | | | * | | . | +-----------------+ * | | . | * | | . | * | | . +--------<-----------------+ * | | . | * | | . | * | +-------------------------- | -------<-------------------.- | -------<-----------------+ * | | | . | | * | | nstat_src | nstat_src . | | * | | +-----------------+ | +-----------------+ . | | * +--->| nts_client_link |---+ +--->| nts_client_link |---+ . | | * | +-----------------+ | +-----------------+ | . | | * -->| nts_locus_link |---|------>| nts_locus_link | | . V ^ * +-----------------+ | +-----------------+ | . | | * | nts_locus | | | nts_locus | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_client | | | nts_client | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_srcref = 3 | | | nts_srcref = 26 | | . | | * +-----------------+ | +-----------------+ | . | | * | nts_provider | | | nts_provider | | . | nstat_sock_locus | inpcb * +-----------------+ | +-----------------+ | . | +-----------------+ | +-----------------+ * | nts_cookie |-->|---+ | nts_coookie |-->|-.--->-->| nsl_locus |---+ | | * +-----------------+ | | +-----------------+ | . | +-----------------+ | | * | nts_filter | | | | nts_filter | | . | | nsl_link | | | * +-----------------+ | | +-----------------+ | . | +-----------------+ | | * | | | | | | | . | | nsl_inp |------>| | * | | | | | | | . ^ +-----------------+ * +-----------------+ | | +-----------------+ | . | | | | | * nstat_src struct | | | . | +-----------------+ * | +------------>------------|-.--+ | nsl_pname | | | * | | . | | * | | . | | | | * | | . | | * +---------<----------------+ +--------<-----------------+ . +-----------------+ | | * | | . * | | . | | * | | . +-----------------+ * | | . * | | . * | | . . . . . . . . . . . . . . . . . . . . . . . . . . . . . * | | . * | +-----------------+ | +-----------------+ . Routes * +--->| nts_client_link | +--->| nts_client_link | . * +-----------------+ +-----------------+ . * | nts_locus_link | | nts_locus_link | . * +-----------------+ +-----------------+ . * | nts_locus | | nts_locus | . * +-----------------+ +-----------------+ . * | nts_client | | nts_client | . * +-----------------+ +-----------------+ . * | nts_srcref = 1 | | nts_srcref = 24 | . * +-----------------+ +-----------------+ . * | nts_provider | | nts_provider | . retentry * +-----------------+ +-----------------+ . +-----------------+ * | nts_cookie |------+ | nts_cookie |--------->-->| | * +-----------------+ | +-----------------+ . | | | * | nts_filter | | | nts_filter | . | * +-----------------+ | +-----------------+ . | | | * | | | | | . | * | | | | | . | | | * +-----------------+ | +-----------------+ . | * | . | | | * +-------------------------------+ * . | | * . * . | | * . +-----------------+ * */ typedef TAILQ_HEAD(, nstat_src) tailq_head_nstat_src; typedef TAILQ_ENTRY(nstat_src) tailq_entry_nstat_src; typedef TAILQ_HEAD(, nstat_sock_locus) tailq_head_sock_locus; typedef TAILQ_ENTRY(nstat_sock_locus) tailq_entry_sock_locus; typedef TAILQ_HEAD(, nstat_tu_shadow) tailq_head_tu_shadow; typedef TAILQ_ENTRY(nstat_tu_shadow) tailq_entry_tu_shadow; typedef TAILQ_HEAD(, nstat_generic_shadow) tailq_head_generic_shadow; typedef TAILQ_ENTRY(nstat_generic_shadow) tailq_entry_generic_shadow; typedef TAILQ_HEAD(, nstat_procdetails) tailq_head_procdetails; typedef TAILQ_ENTRY(nstat_procdetails) tailq_entry_procdetails; static int metrics_collection SYSCTL_HANDLER_ARGS { return ntstat_get_metrics(req); } SYSCTL_PROC(_net_stats, OID_AUTO, metrics, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, 0, 0, metrics_collection, "S", "Various metrics for NetworkStatistics clients, individually or collectively"); typedef void *nstat_provider_cookie_t; typedef struct nstat_locus { tailq_head_nstat_src ntl_src_queue; } nstat_locus; struct nstat_procdetails { tailq_entry_procdetails pdet_link; int pdet_pid; u_int64_t pdet_upid; char pdet_procname[64]; uuid_t pdet_uuid; u_int32_t pdet_refcnt; u_int32_t pdet_magic; }; typedef struct nstat_provider_filter { u_int64_t npf_flags; u_int64_t npf_events; u_int64_t npf_extensions; pid_t npf_pid; uuid_t npf_uuid; } nstat_provider_filter; #define NUM_NSTAT_METRICS_UINT32_COUNTS ((__builtin_offsetof(struct nstat_metrics, nstat_last_uint32_count) - \ __builtin_offsetof(struct nstat_metrics, nstat_first_uint32_count)) / sizeof(uint32_t)) typedef struct nstat_trace_entry { u_int32_t nte_seqno; u_int32_t nte_event; u_int64_t nte_qualifier; } nstat_trace_entry; #define NSTAT_TRACE_ENTRIES_PER_CLIENT (16 * 1024) typedef struct nstat_cyclic_trace { uint32_t ntt_next_trace_id; int32_t ntt_pad; nstat_trace_entry ntt_entry[NSTAT_TRACE_ENTRIES_PER_CLIENT]; } nstat_cyclic_trace; typedef struct nstat_client { struct nstat_client *ntc_next; /* A bitmask to indicate whether a provider ever done NSTAT_MSG_TYPE_ADD_ALL_SRCS */ u_int32_t ntc_watching; /* A bitmask to indicate whether a provider ever done NSTAT_MSG_TYPE_ADD_SRC */ u_int32_t ntc_added_src; decl_lck_mtx_data(, ntc_user_mtx); // Mutual exclusion for user level requests on this ntc_client kern_ctl_ref ntc_kctl; u_int32_t ntc_unit; u_int32_t ntc_client_id; nstat_src_ref_t ntc_next_srcref; tailq_head_nstat_src ntc_src_queue; mbuf_t ntc_accumulated; u_int32_t ntc_flags; nstat_provider_filter ntc_provider_filters[NSTAT_PROVIDER_COUNT]; /* state maintained for partial query requests */ u_int64_t ntc_context; u_int64_t ntc_seq; /* For ease of debugging with lldb macros */ struct nstat_procdetails *ntc_procdetails; struct nstat_metrics ntc_metrics; #if NSTAT_TRACE_ENABLED nstat_cyclic_trace *ntc_trace; #endif } nstat_client; typedef struct nstat_provider { struct nstat_provider *next; nstat_provider_id_t nstat_provider_id; size_t nstat_descriptor_length; errno_t (*nstat_lookup)(const void *__sized_by (length)data, u_int32_t length, nstat_provider_cookie_t *out_cookie); int (*nstat_gone)(nstat_provider_cookie_t cookie); errno_t (*nstat_counts)(nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone); errno_t (*nstat_details)(nstat_provider_cookie_t cookie, struct nstat_detailed_counts *out_details, int *out_gone); errno_t (*nstat_watcher_add)(nstat_client *client, nstat_msg_add_all_srcs *req); void (*nstat_watcher_remove)(nstat_client *client); errno_t (*nstat_copy_descriptor)(nstat_provider_cookie_t cookie, void *__sized_by (len)data, size_t len); void (*nstat_release)(nstat_provider_cookie_t cookie); bool (*nstat_reporting_allowed)(nstat_provider_cookie_t cookie, nstat_provider_filter *filter, u_int64_t suppression_flags); bool (*nstat_cookie_equal)(nstat_provider_cookie_t cookie1, nstat_provider_cookie_t cookie2); size_t (*nstat_copy_extension)(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len); } nstat_provider; typedef struct nstat_src { tailq_entry_nstat_src nts_client_link; // All sources for the nstat_client, for iterating over. tailq_entry_nstat_src nts_locus_link; // All sources for the nstat_locus, for iterating over. nstat_locus *nts_locus; // The (optional) locus, with further details nstat_client *nts_client; // The nstat_client that this is a source for nstat_src_ref_t nts_srcref; // The reference quoted in any messages nstat_provider *nts_provider; // The "provider" for the source, e.g. for kernel TCP sockets nstat_provider_cookie_t nts_cookie; // Provider-specific futher information, uint32_t nts_filter; bool nts_reported; // At least one update/counts/desc message has been sent uint64_t nts_seq; // Keeping track when getting partial poll results } nstat_src; // The merge structures are intended to give a global picture of what may be asked for by the current set of clients // This is to avoid taking locks to check them all individually typedef struct nstat_merged_provider_filter { u_int64_t mf_events; // So far we only merge the events portion of any filters } nstat_merged_provider_filter; typedef struct nstat_merged_provider_filters { nstat_merged_provider_filter mpf_filters[NSTAT_PROVIDER_COUNT]; } nstat_merged_provider_filters; static errno_t nstat_client_send_counts(nstat_client *client, nstat_src *src, unsigned long long context, u_int16_t hdr_flags, int *gone); static int nstat_client_send_description(nstat_client *client, nstat_src *src, u_int64_t context, u_int16_t hdr_flags); static int nstat_client_send_update(nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone); static errno_t nstat_client_send_removed(nstat_client *client, nstat_src *src, u_int16_t hdr_flags); static errno_t nstat_client_send_goodbye(nstat_client *client, nstat_src *src); static void nstat_client_cleanup_source(nstat_client *client, nstat_src *src); static bool nstat_client_reporting_allowed(nstat_client *client, nstat_src *src, u_int64_t suppression_flags); static boolean_t nstat_client_begin_query(nstat_client *client, const nstat_msg_hdr *hdrp); static u_int16_t nstat_client_end_query(nstat_client *client, nstat_src *last_src, boolean_t partial); static void nstat_ifnet_report_lim_stats(void); static void nstat_net_api_report_stats(void); static errno_t nstat_set_provider_filter(nstat_client *client, nstat_msg_add_all_srcs *req); static errno_t nstat_client_send_event(nstat_client *client, nstat_src *src, u_int64_t event); static u_int32_t nstat_udp_watchers = 0; static u_int32_t nstat_tcp_watchers = 0; static nstat_merged_provider_filters merged_filters = {}; static nstat_client *nstat_clients = NULL; static uint64_t nstat_idle_time = 0; static uint32_t nstat_next_client_id = 3; // Deliberate offset from zero to reserve values 0 and 1 #if NSTAT_FUZZ_TIMING static uint32_t nstat_random_delay_insert_modulo = 5000; static uint32_t nstat_max_nsec_delay = (NSEC_PER_SEC / 1000); #endif // NSTAT_FUZZ_TIMING static struct nstat_metrics nstat_metrics; // For lldb macro usage static __unused const size_t nstat_trace_entries_per_client = NSTAT_TRACE_ENTRIES_PER_CLIENT; #if NSTAT_TRACE_ENABLED static nstat_cyclic_trace nstat_global_trace; #endif #if NSTAT_TRACE_ENABLED #define NSTAT_TRACE(field, client, qual) \ { \ nstat_trace(offsetof(struct nstat_metrics, field), client, (uint64_t)qual); \ } #else // NSTAT_TRACE_ENABLED #define NSTAT_TRACE(field, client, qual) #endif #if NSTAT_FUZZ_TIMING #define NSTAT_RANDOM_DELAY(client) \ { \ nstat_random_delay(client); \ } #else #define NSTAT_RANDOM_DELAY(client) #endif #define NSTAT_NOTE_QUAL(field, client, qual) \ { \ NSTAT_TRACE(field, client, qual); \ NSTAT_RANDOM_DELAY(client); \ client->ntc_metrics.field++; \ } #define NSTAT_NOTE_SRC(field, client, src) \ { \ NSTAT_TRACE(field, client, (src != NULL)? src->nts_srcref: 0); \ NSTAT_RANDOM_DELAY(client); \ client->ntc_metrics.field++; \ } #define NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(_currentfield, _maxfield) \ { \ uint64_t _prev_count = os_atomic_inc_orig(&nstat_global_counts._currentfield, relaxed); \ if (_prev_count >= os_atomic_load(&nstat_global_counts._maxfield, relaxed)) { \ os_atomic_store(&nstat_global_counts._maxfield, _prev_count + 1, relaxed); \ } \ } #define NSTAT_GLOBAL_COUNT_INCREMENT(_field) \ { \ os_atomic_inc_orig(&nstat_global_counts._field, relaxed); \ } #define NSTAT_GLOBAL_COUNT_DECREMENT(_field) \ { \ os_atomic_dec_orig(&nstat_global_counts._field, relaxed); \ } static errno_t nstat_client_send_counts(nstat_client *client, nstat_src *src, unsigned long long context, u_int16_t hdr_flags, int *gone); static int nstat_client_send_description(nstat_client *client, nstat_src *src, u_int64_t context, u_int16_t hdr_flags); static int nstat_client_send_update(nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone); static int nstat_client_send_details(nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone); static errno_t nstat_client_send_removed(nstat_client *client, nstat_src *src, u_int16_t hdr_flags); static errno_t nstat_client_send_goodbye(nstat_client *client, nstat_src *src); static void nstat_client_cleanup_source(nstat_client *client, nstat_src *src); static bool nstat_client_reporting_allowed(nstat_client *client, nstat_src *src, u_int64_t suppression_flags); static boolean_t nstat_client_begin_query(nstat_client *client, const nstat_msg_hdr *hdrp); static u_int16_t nstat_client_end_query(nstat_client *client, nstat_src *last_src, boolean_t partial); static void nstat_ifnet_report_lim_stats(void); static void nstat_net_api_report_stats(void); static errno_t nstat_client_send_event(nstat_client *client, nstat_src *src, u_int64_t event); static void nstat_client_register(void); /* * The lock order is as follows: * * nstat_rwlock * * Or: * * client->ntc_user_mtx * nstat_rwlock * * * The nstat_rwlock may be held in exclusive (writer) or shared (reader) mode. * * In general, events from the kernel, such as flow creation and deletion, * require that the lock be held in exclusive mode while "nstat_src" structures * are created or destroyed and appropriate linkages made or broken. * * In contrast, things that are driven from ntstat user level clients, * most obviously polls, typically only require shared mode locking. * There are some rare exceptions where a need to manipulate linkages * that go across multiple clients will require an upgrade to an exlusive lock. * * There is one other lock to consider. Calls from NetworkStatistics clients will first * obtain the per-client ntc_user_mtx. This ensures that only one such client thread * can be in progress at a time, making it easier to reason about correctness. */ static LCK_ATTR_DECLARE(nstat_lck_attr, 0, 0); static LCK_GRP_DECLARE(nstat_lck_grp, "network statistics kctl"); static LCK_RW_DECLARE_ATTR(nstat_rwlock, &nstat_lck_grp, &nstat_lck_attr); #define NSTAT_LOCK_EXCLUSIVE() \ if (lck_rw_try_lock_exclusive(&nstat_rwlock)) { \ NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_exclusive_lock_uncontended); \ } else { \ lck_rw_lock_exclusive(&nstat_rwlock); \ NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_exclusive_lock_contended); \ } #define NSTAT_LOCK_SHARED() \ if (lck_rw_try_lock_shared(&nstat_rwlock)) { \ NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_shared_lock_uncontended); \ } else { \ lck_rw_lock_shared(&nstat_rwlock); \ NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_shared_lock_contended); \ } #define NSTAT_LOCK_SHARED_TO_EXCLUSIVE() lck_rw_lock_shared_to_exclusive(&nstat_rwlock) #define NSTAT_LOCK_EXCLUSIVE_TO_SHARED() lck_rw_lock_exclusive_to_shared(&nstat_rwlock) #define NSTAT_TRY_LOCK_EXCLUSIVE() lck_rw_try_lock_exclusive(&nstat_rwlock) #define NSTAT_UNLOCK() lck_rw_done(&nstat_rwlock) #define NSTAT_UNLOCK_EXCLUSIVE() lck_rw_unlock_exclusive(&nstat_rwlock) #define NSTAT_UNLOCK_SHARED() lck_rw_unlock_shared(&nstat_rwlock) #define NSTAT_LOCK_WOULD_YIELD() lck_rw_lock_would_yield_shared(&nstat_rwlock) #define NSTAT_LOCK_YIELD() lck_rw_lock_yield_shared(&nstat_rwlock, FALSE) #define NSTAT_LOCK_YIELD_EXCLUSIVE() lck_rw_lock_yield_exclusive(&nstat_rwlock, LCK_RW_YIELD_ANY_WAITER) #define NSTAT_ASSERT_LOCKED_EXCLUSIVE() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_EXCLUSIVE) #define NSTAT_ASSERT_LOCKED_SHARED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_SHARED) #define NSTAT_ASSERT_LOCKED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_HELD) #define NSTAT_ASSERT_UNLOCKED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_NOTHELD) /* some extern definitions */ extern void tcp_report_stats(void); #if NSTAT_TRACE_ENABLED static void nstat_trace(int32_t trace_id, nstat_client *client, uint64_t qual) { nstat_cyclic_trace *trace; nstat_trace_entry *entry; uint32_t seqno; if (client == NULL) { trace = &nstat_global_trace; } else { trace = client->ntc_trace; } if (trace != NULL) { seqno = OSIncrementAtomic(&trace->ntt_next_trace_id); if (seqno > (NSTAT_TRACE_ENTRIES_PER_CLIENT - 2)) { seqno = NSTAT_TRACE_ENTRIES_PER_CLIENT - 2; } int32_t index = seqno % NSTAT_TRACE_ENTRIES_PER_CLIENT; entry = &trace->ntt_entry[index]; entry->nte_seqno = seqno; entry->nte_event = trace_id; entry->nte_qualifier = qual; } } #endif #if NSTAT_FUZZ_TIMING static void nstat_random_delay(nstat_client *client) { if (nstat_random_delay_insert_modulo) { uint64_t random; read_random(&random, sizeof(random)); if ((random % nstat_random_delay_insert_modulo) == 0) { #if NSTAT_TRACE_ENABLED nstat_trace(1, client, 0); #endif read_random(&random, sizeof(random)); uint64_t nsec_delay = (random % nstat_max_nsec_delay); delay_for_interval(nsec_delay, 1); #if NSTAT_TRACE_ENABLED nstat_trace(2, client, 0); #endif } } } #endif static void nstat_accumulate_client_metrics(struct nstat_metrics *dest, nstat_client *client) { if (dest->nstat_src_max < client->ntc_metrics.nstat_src_max) { dest->nstat_src_max = client->ntc_metrics.nstat_src_max; } // Most of the counts happen to be consecutive uint32_t values that can be picked up via pointer iteration rather than name uint32_t *srcptr = __unsafe_forge_bidi_indexable(uint32_t *, (uint32_t *)(void *)&client->ntc_metrics.nstat_first_uint32_count, (NUM_NSTAT_METRICS_UINT32_COUNTS * sizeof(uint32_t))); uint32_t *destptr = __unsafe_forge_bidi_indexable(uint32_t *, (uint32_t *)(void *)&dest->nstat_first_uint32_count, (NUM_NSTAT_METRICS_UINT32_COUNTS * sizeof(uint32_t))); for (int i = 0; i < NUM_NSTAT_METRICS_UINT32_COUNTS; i++) { destptr[i] += srcptr[i]; } } static int metrics_for_client_id(uint32_t req_id, struct nstat_client_info *client_info) { int err = 0; struct nstat_metrics *metrics = &client_info->nstat_metrics; bzero(client_info, sizeof(*client_info)); if (req_id == NSTAT_METRIC_ID_ACCUMULATED) { client_info->nstat_client_details.nstat_client_id = NSTAT_METRIC_ID_ACCUMULATED; *metrics = nstat_metrics; } else { NSTAT_LOCK_EXCLUSIVE(); if (req_id == NSTAT_METRIC_ID_GRAND_TOTAL) { client_info->nstat_client_details.nstat_client_id = NSTAT_METRIC_ID_GRAND_TOTAL; *metrics = nstat_metrics; nstat_client *client; for (client = nstat_clients; client; client = client->ntc_next) { nstat_accumulate_client_metrics(metrics, client); } } else { nstat_client *client; err = ERANGE; for (client = nstat_clients; client; client = client->ntc_next) { if (client->ntc_client_id <= req_id) { client_info->nstat_client_details.nstat_client_id = client->ntc_client_id; client_info->nstat_client_details.nstat_client_pid = client->ntc_procdetails->pdet_pid; client_info->nstat_client_details.nstat_client_watching = client->ntc_watching; client_info->nstat_client_details.nstat_client_added_src = client->ntc_added_src; *metrics = client->ntc_metrics; err = 0; break; } } } NSTAT_UNLOCK_EXCLUSIVE(); } return err; } static int ntstat_get_metrics(struct sysctl_req *req) { // The following assumes that the client_info structure is small such that stack allocation is reasonable struct nstat_client_info client_info = {}; int error = 0; struct nstat_metrics_req requested; if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { return EACCES; } if (req->newptr == USER_ADDR_NULL) { return EINVAL; } if (req->newlen < sizeof(requested)) { return EINVAL; } error = SYSCTL_IN(req, &requested, sizeof(requested)); if (error != 0) { return error; } if (requested.mr_version != NSTAT_METRIC_VERSION) { return ENOTSUP; } error = metrics_for_client_id(requested.mr_id, &client_info); if (error != 0) { return error; } error = SYSCTL_OUT(req, &client_info, sizeof(client_info)); return error; } static void nstat_copy_sa_out( const struct sockaddr *src, struct sockaddr *dst, int maxlen) { if (src->sa_len > maxlen) { return; } SOCKADDR_COPY(src, dst, src->sa_len); if (src->sa_family == AF_INET6 && src->sa_len >= sizeof(struct sockaddr_in6)) { struct sockaddr_in6 *sin6 = SIN6(dst); if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) { sin6->sin6_scope_id = (SIN6(src))->sin6_scope_id; if (in6_embedded_scope) { in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id); sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]); sin6->sin6_addr.s6_addr16[1] = 0; } } } } static void nstat_ip_to_sockaddr( const struct in_addr *ip, u_int16_t port, struct sockaddr_in *sin, u_int32_t maxlen) { if (maxlen < sizeof(struct sockaddr_in)) { return; } sin->sin_family = AF_INET; sin->sin_len = sizeof(*sin); sin->sin_port = port; sin->sin_addr = *ip; } u_int32_t nstat_ifnet_to_flags( struct ifnet *ifp) { u_int32_t flags = 0; u_int32_t functional_type = if_functional_type(ifp, FALSE); u_int32_t peer_egress_functional_type = IFRTYPE_FUNCTIONAL_UNKNOWN; // If known VPN, check the delegate interface to see if it has peer // egress interface type set to cell. peer_egress_functional_type = if_peer_egress_functional_type(ifp, !(IFNET_IS_VPN(ifp))); /* Panic if someone adds a functional type without updating ntstat. */ VERIFY(0 <= functional_type && functional_type <= IFRTYPE_FUNCTIONAL_LAST); switch (functional_type) { case IFRTYPE_FUNCTIONAL_UNKNOWN: flags |= NSTAT_IFNET_IS_UNKNOWN_TYPE; break; case IFRTYPE_FUNCTIONAL_LOOPBACK: flags |= NSTAT_IFNET_IS_LOOPBACK; break; case IFRTYPE_FUNCTIONAL_WIRED: case IFRTYPE_FUNCTIONAL_INTCOPROC: case IFRTYPE_FUNCTIONAL_MANAGEMENT: flags |= NSTAT_IFNET_IS_WIRED; break; case IFRTYPE_FUNCTIONAL_WIFI_INFRA: flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_WIFI_INFRA; break; case IFRTYPE_FUNCTIONAL_WIFI_AWDL: flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_AWDL; break; case IFRTYPE_FUNCTIONAL_CELLULAR: flags |= NSTAT_IFNET_IS_CELLULAR; break; case IFRTYPE_FUNCTIONAL_COMPANIONLINK: flags |= NSTAT_IFNET_IS_COMPANIONLINK; if (IFNET_IS_COMPANION_LINK_BLUETOOTH(ifp)) { flags |= NSTAT_IFNET_IS_COMPANIONLINK_BT; } break; } switch (peer_egress_functional_type) { case IFRTYPE_FUNCTIONAL_CELLULAR: flags |= NSTAT_IFNET_PEEREGRESSINTERFACE_IS_CELLULAR; break; } if (IFNET_IS_EXPENSIVE(ifp)) { flags |= NSTAT_IFNET_IS_EXPENSIVE; } if (IFNET_IS_CONSTRAINED(ifp)) { flags |= NSTAT_IFNET_IS_CONSTRAINED; if (IFNET_IS_ULTRA_CONSTRAINED(ifp)) { flags |= NSTAT_IFNET_IS_ULTRA_CONSTRAINED; } } if (ifnet_is_low_latency(ifp)) { flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_LLW; } if (IFNET_IS_VPN(ifp)) { flags |= NSTAT_IFNET_IS_VPN; } return flags; } static void nstat_update_local_flag_from_inpcb_route(const struct inpcb *inp, u_int32_t *flags) { if (inp != NULL && (inp->inp_flags2 & INP2_LAST_ROUTE_LOCAL)) { *flags |= NSTAT_IFNET_IS_LOCAL; } else { *flags |= NSTAT_IFNET_IS_NON_LOCAL; } } static u_int32_t nstat_inpcb_to_flags( const struct inpcb *inp) { u_int32_t flags = 0; if (inp != NULL) { if (inp->inp_last_outifp != NULL) { struct ifnet *ifp = inp->inp_last_outifp; flags = nstat_ifnet_to_flags(ifp); struct tcpcb *tp = intotcpcb(inp); if (tp) { if (tp->t_flags & TF_LOCAL) { flags |= NSTAT_IFNET_IS_LOCAL; } else { flags |= NSTAT_IFNET_IS_NON_LOCAL; } } else { nstat_update_local_flag_from_inpcb_route(inp, &flags); } } else { flags = NSTAT_IFNET_IS_UNKNOWN_TYPE; nstat_update_local_flag_from_inpcb_route(inp, &flags); } if (inp->inp_socket != NULL && (inp->inp_socket->so_flags1 & SOF1_CELLFALLBACK)) { flags |= NSTAT_IFNET_VIA_CELLFALLBACK; } } return flags; } static void merge_current_event_filters(void) { // The nstat_rwlock is assumed locked NSTAT_ASSERT_LOCKED(); nstat_merged_provider_filters new_merge = {}; nstat_provider_type_t provider; nstat_client *client; for (client = nstat_clients; client; client = client->ntc_next) { for (provider = NSTAT_PROVIDER_NONE; provider <= NSTAT_PROVIDER_LAST; provider++) { new_merge.mpf_filters[provider].mf_events |= client->ntc_provider_filters[provider].npf_events; } } for (provider = NSTAT_PROVIDER_NONE; provider <= NSTAT_PROVIDER_LAST; provider++) { // This should do atomic updates of the 64 bit words, where memcpy would be undefined merged_filters.mpf_filters[provider].mf_events = new_merge.mpf_filters[provider].mf_events; } } static inline void nstat_src_remove_linkages(nstat_client *client, nstat_src *src) { NSTAT_ASSERT_LOCKED_EXCLUSIVE(); NSTAT_NOTE_SRC(nstat_src_removed_linkage, client, src); TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link); if (src->nts_locus != NULL) { TAILQ_REMOVE(&src->nts_locus->ntl_src_queue, src, nts_locus_link); src->nts_locus = NULL; } assert(client->ntc_metrics.nstat_src_current > 0); client->ntc_metrics.nstat_src_current--; } #pragma mark -- Network Statistic Providers -- static errno_t nstat_client_source_add(u_int64_t context, nstat_client *client, nstat_provider *provider, nstat_provider_cookie_t cookie, nstat_locus *locus); struct nstat_provider *nstat_providers = NULL; static struct nstat_provider* nstat_find_provider_by_id( nstat_provider_id_t id) { struct nstat_provider *provider; for (provider = nstat_providers; provider != NULL; provider = provider->next) { if (provider->nstat_provider_id == id) { break; } } return provider; } static errno_t nstat_lookup_entry( nstat_provider_id_t id, const void *__sized_by(length)data, u_int32_t length, nstat_provider **out_provider, nstat_provider_cookie_t *out_cookie) { *out_provider = nstat_find_provider_by_id(id); if (*out_provider == NULL) { return ENOENT; } return (*out_provider)->nstat_lookup(data, length, out_cookie); } static void nstat_client_sanitize_cookie( nstat_client *client, nstat_provider_id_t id, nstat_provider_cookie_t cookie) { nstat_src *src = NULL; // Scan the source list to find any duplicate entry and remove it. NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { nstat_provider *sp = src->nts_provider; if (sp->nstat_provider_id == id && sp->nstat_cookie_equal != NULL && sp->nstat_cookie_equal(src->nts_cookie, cookie)) { break; } } if (src) { nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); } NSTAT_UNLOCK_EXCLUSIVE(); if (src) { nstat_client_cleanup_source(NULL, src); } } static void nstat_init_route_provider(void); static void nstat_init_tcp_provider(void); static void nstat_init_udp_provider(void); #if SKYWALK static void nstat_init_userland_tcp_provider(void); static void nstat_init_userland_udp_provider(void); static void nstat_init_userland_quic_provider(void); #endif /* SKYWALK */ static void nstat_init_userland_conn_provider(void); static void nstat_init_udp_subflow_provider(void); static void nstat_init_ifnet_provider(void); __private_extern__ void nstat_init(void) { nstat_log_handle = os_log_create("com.apple.xnu.net", "ntstat"); nstat_global_counts.nstat_global_count_version = NSTAT_GLOBAL_COUNTS_VERSION; nstat_init_route_provider(); nstat_init_tcp_provider(); nstat_init_udp_provider(); #if SKYWALK nstat_init_userland_tcp_provider(); nstat_init_userland_udp_provider(); nstat_init_userland_quic_provider(); #endif /* SKYWALK */ nstat_init_userland_conn_provider(); nstat_init_udp_subflow_provider(); nstat_init_ifnet_provider(); nstat_client_register(); } #pragma mark -- Utilities -- #define NSTAT_PROCDETAILS_MAGIC 0xfeedc001 #define NSTAT_PROCDETAILS_UNMAGIC 0xdeadc001 static tailq_head_procdetails nstat_procdetails_head = TAILQ_HEAD_INITIALIZER(nstat_procdetails_head); static struct nstat_procdetails * nstat_retain_curprocdetails(void) { struct nstat_procdetails *procdetails = NULL; uint64_t upid = proc_uniqueid(current_proc()); NSTAT_LOCK_SHARED(); TAILQ_FOREACH(procdetails, &nstat_procdetails_head, pdet_link) { assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); if (procdetails->pdet_upid == upid) { OSIncrementAtomic(&procdetails->pdet_refcnt); break; } } NSTAT_UNLOCK_SHARED(); if (!procdetails) { // No need for paranoia on locking, it would be OK if there are duplicate structs on the list procdetails = kalloc_type(struct nstat_procdetails, Z_WAITOK | Z_NOFAIL); procdetails->pdet_pid = proc_selfpid(); procdetails->pdet_upid = upid; proc_selfname(procdetails->pdet_procname, sizeof(procdetails->pdet_procname)); proc_getexecutableuuid(current_proc(), procdetails->pdet_uuid, sizeof(uuid_t)); procdetails->pdet_refcnt = 1; procdetails->pdet_magic = NSTAT_PROCDETAILS_MAGIC; NSTAT_LOCK_EXCLUSIVE(); TAILQ_INSERT_HEAD(&nstat_procdetails_head, procdetails, pdet_link); NSTAT_UNLOCK_EXCLUSIVE(); NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_procdetails_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_procdetails_current, nstat_global_procdetails_max); } return procdetails; } static void nstat_release_procdetails(struct nstat_procdetails *procdetails) { assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); // These are harvested later to amortize costs OSDecrementAtomic(&procdetails->pdet_refcnt); } static void nstat_prune_procdetails(void) { struct nstat_procdetails *procdetails; struct nstat_procdetails *tmpdetails; tailq_head_procdetails dead_list; TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH_SAFE(procdetails, &nstat_procdetails_head, pdet_link, tmpdetails) { assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); if (procdetails->pdet_refcnt == 0) { // Pull it off the list TAILQ_REMOVE(&nstat_procdetails_head, procdetails, pdet_link); TAILQ_INSERT_TAIL(&dead_list, procdetails, pdet_link); } } NSTAT_UNLOCK_EXCLUSIVE(); while ((procdetails = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, procdetails, pdet_link); procdetails->pdet_magic = NSTAT_PROCDETAILS_UNMAGIC; NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_procdetails_current); kfree_type(struct nstat_procdetails, procdetails); } } #pragma mark -- Route Provider -- static nstat_provider nstat_route_provider; static errno_t nstat_route_lookup( const void *data, u_int32_t length, nstat_provider_cookie_t *out_cookie) { struct sockaddr *dst = NULL; struct sockaddr *mask = NULL; const nstat_route_add_param *param = (const nstat_route_add_param*)data; *out_cookie = NULL; if (length < sizeof(*param)) { return EINVAL; } if (param->dst.v4.sin_family == 0 || param->dst.v4.sin_family > AF_MAX || (param->mask.v4.sin_family != 0 && param->mask.v4.sin_family != param->dst.v4.sin_family)) { return EINVAL; } if (param->dst.v4.sin_len > sizeof(param->dst) || (param->mask.v4.sin_family && param->mask.v4.sin_len > sizeof(param->mask.v4.sin_len))) { return EINVAL; } if ((param->dst.v4.sin_family == AF_INET && param->dst.v4.sin_len < sizeof(struct sockaddr_in)) || (param->dst.v6.sin6_family == AF_INET6 && param->dst.v6.sin6_len < sizeof(struct sockaddr_in6))) { return EINVAL; } dst = __DECONST_SA(¶m->dst.v4); mask = param->mask.v4.sin_family ? __DECONST_SA(¶m->mask.v4) : NULL; struct radix_node_head *rnh = rt_tables[dst->sa_family]; if (rnh == NULL) { return EAFNOSUPPORT; } lck_mtx_lock(rnh_lock); struct rtentry *rt = rt_lookup(TRUE, dst, mask, rnh, param->ifindex); lck_mtx_unlock(rnh_lock); if (rt) { *out_cookie = (nstat_provider_cookie_t)rt; } return rt ? 0 : ENOENT; } static int nstat_route_gone( nstat_provider_cookie_t cookie) { struct rtentry *rt = (struct rtentry*)cookie; return ((rt->rt_flags & RTF_UP) == 0) ? 1 : 0; } static errno_t nstat_route_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct rtentry *rt = (struct rtentry*)cookie; struct nstat_counts *rt_stats = rt->rt_stats; if (out_gone) { *out_gone = 0; } if (out_gone && (rt->rt_flags & RTF_UP) == 0) { *out_gone = 1; } if (rt_stats) { out_counts->nstat_rxpackets = os_atomic_load(&rt_stats->nstat_rxpackets, relaxed); out_counts->nstat_rxbytes = os_atomic_load(&rt_stats->nstat_rxbytes, relaxed); out_counts->nstat_txpackets = os_atomic_load(&rt_stats->nstat_txpackets, relaxed); out_counts->nstat_txbytes = os_atomic_load(&rt_stats->nstat_txbytes, relaxed); out_counts->nstat_rxduplicatebytes = rt_stats->nstat_rxduplicatebytes; out_counts->nstat_rxoutoforderbytes = rt_stats->nstat_rxoutoforderbytes; out_counts->nstat_txretransmit = rt_stats->nstat_txretransmit; out_counts->nstat_connectattempts = rt_stats->nstat_connectattempts; out_counts->nstat_connectsuccesses = rt_stats->nstat_connectsuccesses; out_counts->nstat_min_rtt = rt_stats->nstat_min_rtt; out_counts->nstat_avg_rtt = rt_stats->nstat_avg_rtt; out_counts->nstat_var_rtt = rt_stats->nstat_var_rtt; out_counts->nstat_cell_rxbytes = out_counts->nstat_cell_txbytes = 0; } else { bzero(out_counts, sizeof(*out_counts)); } return 0; } static void nstat_route_release( nstat_provider_cookie_t cookie) { rtfree((struct rtentry*)cookie); } static u_int32_t nstat_route_watchers = 0; static int nstat_route_walktree_add( struct radix_node *rn, void *context) { errno_t result = 0; struct rtentry *rt = (struct rtentry *)rn; nstat_client *client = (nstat_client*)context; LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* RTF_UP can't change while rnh_lock is held */ if ((rt->rt_flags & RTF_UP) != 0) { /* Clear RTPRF_OURS if the route is still usable */ RT_LOCK(rt); if (rt_validate(rt)) { RT_ADDREF_LOCKED(rt); RT_UNLOCK(rt); } else { RT_UNLOCK(rt); rt = NULL; } /* Otherwise if RTF_CONDEMNED, treat it as if it were down */ if (rt == NULL) { return 0; } NSTAT_LOCK_EXCLUSIVE(); result = nstat_client_source_add(0, client, &nstat_route_provider, rt, NULL); NSTAT_UNLOCK_EXCLUSIVE(); if (result != 0) { rtfree_locked(rt); } } return result; } static errno_t nstat_route_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { int i; errno_t result = 0; lck_mtx_lock(rnh_lock); result = nstat_set_provider_filter(client, req); if (result == 0) { OSIncrementAtomic(&nstat_route_watchers); for (i = 1; i < AF_MAX; i++) { struct radix_node_head *rnh; rnh = rt_tables[i]; if (!rnh) { continue; } result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, client); if (result != 0) { // This is probably resource exhaustion. // There currently isn't a good way to recover from this. // Least bad seems to be to give up on the add-all but leave // the watcher in place. break; } } } lck_mtx_unlock(rnh_lock); return result; } __private_extern__ void nstat_route_new_entry( struct rtentry *rt) { if (nstat_route_watchers == 0) { return; } NSTAT_LOCK_EXCLUSIVE(); if ((rt->rt_flags & RTF_UP) != 0) { nstat_client *client; for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_watching & (1 << NSTAT_PROVIDER_ROUTE)) != 0) { // this client is watching routes // acquire a reference for the route RT_ADDREF(rt); // add the source, if that fails, release the reference if (nstat_client_source_add(0, client, &nstat_route_provider, rt, NULL) != 0) { RT_REMREF(rt); } } } } NSTAT_UNLOCK_EXCLUSIVE(); } static void nstat_route_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_route_watchers); } static errno_t nstat_route_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, size_t len) { nstat_route_descriptor *desc = (nstat_route_descriptor*)data; if (len < sizeof(*desc)) { return EINVAL; } bzero(desc, sizeof(*desc)); struct rtentry *rt = (struct rtentry*)cookie; desc->id = (uint64_t)VM_KERNEL_ADDRHASH(rt); desc->parent_id = (uint64_t)VM_KERNEL_ADDRHASH(rt->rt_parent); desc->gateway_id = (uint64_t)VM_KERNEL_ADDRHASH(rt->rt_gwroute); // key/dest struct sockaddr *sa; if ((sa = rt_key(rt))) { nstat_copy_sa_out(sa, &desc->dst.sa, sizeof(desc->dst)); } // mask if ((sa = rt_mask(rt))) { nstat_copy_sa_out(sa, &desc->mask.sa, sizeof(desc->mask)); } // gateway if ((sa = rt->rt_gateway)) { nstat_copy_sa_out(sa, &desc->gateway.sa, sizeof(desc->gateway)); } if (rt->rt_ifp) { desc->ifindex = rt->rt_ifp->if_index; } desc->flags = rt->rt_flags; return 0; } static bool nstat_route_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { bool retval = true; if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) { struct rtentry *rt = (struct rtentry*)cookie; struct ifnet *ifp = rt->rt_ifp; if (ifp) { uint32_t interface_properties = nstat_ifnet_to_flags(ifp); if ((filter->npf_flags & interface_properties) == 0) { retval = false; } } } return retval; } static bool nstat_route_cookie_equal( nstat_provider_cookie_t cookie1, nstat_provider_cookie_t cookie2) { struct rtentry *rt1 = (struct rtentry *)cookie1; struct rtentry *rt2 = (struct rtentry *)cookie2; return (rt1 == rt2) ? true : false; } static void nstat_init_route_provider(void) { bzero(&nstat_route_provider, sizeof(nstat_route_provider)); nstat_route_provider.nstat_descriptor_length = sizeof(nstat_route_descriptor); nstat_route_provider.nstat_provider_id = NSTAT_PROVIDER_ROUTE; nstat_route_provider.nstat_lookup = nstat_route_lookup; nstat_route_provider.nstat_gone = nstat_route_gone; nstat_route_provider.nstat_counts = nstat_route_counts; nstat_route_provider.nstat_release = nstat_route_release; nstat_route_provider.nstat_watcher_add = nstat_route_add_watcher; nstat_route_provider.nstat_watcher_remove = nstat_route_remove_watcher; nstat_route_provider.nstat_copy_descriptor = nstat_route_copy_descriptor; nstat_route_provider.nstat_reporting_allowed = nstat_route_reporting_allowed; nstat_route_provider.nstat_cookie_equal = nstat_route_cookie_equal; nstat_route_provider.next = nstat_providers; nstat_providers = &nstat_route_provider; } #pragma mark -- Route Collection -- __private_extern__ struct nstat_counts* nstat_route_attach( struct rtentry *rte) { struct nstat_counts * __single result = rte->rt_stats; if (result) { return result; } result = kalloc_type(struct nstat_counts, Z_WAITOK | Z_ZERO); if (!result) { return result; } if (!OSCompareAndSwapPtr(NULL, result, &rte->rt_stats)) { kfree_type(struct nstat_counts, result); result = rte->rt_stats; } return result; } __private_extern__ void nstat_route_detach( struct rtentry *rte) { if (rte->rt_stats) { kfree_type(struct nstat_counts, rte->rt_stats); rte->rt_stats = NULL; } } __private_extern__ void nstat_route_connect_attempt( struct rtentry *rte) { while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { OSIncrementAtomic(&stats->nstat_connectattempts); } rte = rte->rt_parent; } } __private_extern__ void nstat_route_connect_success( struct rtentry *rte) { // This route while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { OSIncrementAtomic(&stats->nstat_connectsuccesses); } rte = rte->rt_parent; } } __private_extern__ void nstat_route_tx( struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags) { while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { if ((flags & NSTAT_TX_FLAG_RETRANSMIT) != 0) { OSAddAtomic(bytes, &stats->nstat_txretransmit); } else { OSAddAtomic64((SInt64)packets, (SInt64*)&stats->nstat_txpackets); OSAddAtomic64((SInt64)bytes, (SInt64*)&stats->nstat_txbytes); } } rte = rte->rt_parent; } } __private_extern__ void nstat_route_rx( struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags) { while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { if (flags == 0) { OSAddAtomic64((SInt64)packets, (SInt64*)&stats->nstat_rxpackets); OSAddAtomic64((SInt64)bytes, (SInt64*)&stats->nstat_rxbytes); } else { if (flags & NSTAT_RX_FLAG_OUT_OF_ORDER) { OSAddAtomic(bytes, &stats->nstat_rxoutoforderbytes); } if (flags & NSTAT_RX_FLAG_DUPLICATE) { OSAddAtomic(bytes, &stats->nstat_rxduplicatebytes); } } } rte = rte->rt_parent; } } /* atomically average current value at _val_addr with _new_val and store */ #define NSTAT_EWMA_ATOMIC(_val_addr, _new_val, _decay) do { \ volatile uint32_t _old_val; \ volatile uint32_t _avg; \ do { \ _old_val = *_val_addr; \ if (_old_val == 0) \ { \ _avg = _new_val; \ } \ else \ { \ _avg = _old_val - (_old_val >> _decay) + (_new_val >> _decay); \ } \ if (_old_val == _avg) break; \ } while (!OSCompareAndSwap(_old_val, _avg, _val_addr)); \ } while (0); /* atomically compute minimum of current value at _val_addr with _new_val and store */ #define NSTAT_MIN_ATOMIC(_val_addr, _new_val) do { \ volatile uint32_t _old_val; \ do { \ _old_val = *_val_addr; \ if (_old_val != 0 && _old_val < _new_val) \ { \ break; \ } \ } while (!OSCompareAndSwap(_old_val, _new_val, _val_addr)); \ } while (0); __private_extern__ void nstat_route_rtt( struct rtentry *rte, u_int32_t rtt, u_int32_t rtt_var) { const uint32_t decay = 3; while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); } rte = rte->rt_parent; } } __private_extern__ void nstat_route_update( struct rtentry *rte, uint32_t connect_attempts, uint32_t connect_successes, uint32_t rx_packets, uint32_t rx_bytes, uint32_t rx_duplicatebytes, uint32_t rx_outoforderbytes, uint32_t tx_packets, uint32_t tx_bytes, uint32_t tx_retransmit, uint32_t rtt, uint32_t rtt_var) { const uint32_t decay = 3; while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { OSAddAtomic(connect_attempts, &stats->nstat_connectattempts); OSAddAtomic(connect_successes, &stats->nstat_connectsuccesses); OSAddAtomic64((SInt64)tx_packets, (SInt64*)&stats->nstat_txpackets); OSAddAtomic64((SInt64)tx_bytes, (SInt64*)&stats->nstat_txbytes); OSAddAtomic(tx_retransmit, &stats->nstat_txretransmit); OSAddAtomic64((SInt64)rx_packets, (SInt64*)&stats->nstat_rxpackets); OSAddAtomic64((SInt64)rx_bytes, (SInt64*)&stats->nstat_rxbytes); OSAddAtomic(rx_outoforderbytes, &stats->nstat_rxoutoforderbytes); OSAddAtomic(rx_duplicatebytes, &stats->nstat_rxduplicatebytes); if (rtt != 0) { NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); } } rte = rte->rt_parent; } } #pragma mark -- TCP Kernel Provider -- #define PNAME_MAX_LENGTH ((2 * MAXCOMLEN) + 1) /* * Due to the way the kernel deallocates a process (the process structure * might be gone by the time we get the PCB detach notification), * we need to cache the process name. Without this, proc_best_name_for_pid() * would return null and the process name would never be sent to userland. * * For UDP sockets, we also store the cached the connection tuples along with * the interface index. This is necessary because when UDP sockets are * disconnected, the connection tuples are forever lost from the inpcb, thus * we need to keep track of the last call to connect() in ntstat. * * There may be either zero or one of the TCP and UDP variants of the nstat_sock_locus * for any socket. There may be multiple nstat_src structures linked to the locus. * At the time of original implementation, it is not expected that the nstat_sock_locus * should live beyond the lifetime of the socket. There is therefore no reference counting * and the structures are disposed of when the socket closes. */ #define NSTAT_SOCK_LOCUS_MAGIC 0xfeedf001 #define NSTAT_SOCK_LOCUS_UNMAGIC 0xdeadf001 typedef struct nstat_sock_locus { nstat_locus nsl_locus; // The locus as used for generic processing tailq_entry_sock_locus nsl_link; // TCP and UDP sock_locus structures queued here struct inpcb *nsl_inp; // As per the associated socket uint32_t nsl_magic; // Debug aid pid_t nsl_pid; uint32_t nsl_ifnet_properties; char nsl_pname[2 * MAXCOMLEN + 1]; char nsl_is_tcp; // A boolean indicating TCP or UDP usage } nstat_sock_locus; // An extended version is used for UDP typedef struct nstat_extended_sock_locus { nstat_sock_locus nesl_sock_locus; union{ struct sockaddr_in v4; struct sockaddr_in6 v6; } nesl_local; union{ struct sockaddr_in v4; struct sockaddr_in6 v6; } nesl_remote; bool nesl_cached; unsigned int nesl_if_index; } nstat_extended_sock_locus; static tailq_head_sock_locus nstat_tcp_sock_locus_head = TAILQ_HEAD_INITIALIZER(nstat_tcp_sock_locus_head); static tailq_head_sock_locus nstat_udp_sock_locus_head = TAILQ_HEAD_INITIALIZER(nstat_udp_sock_locus_head); static bool nstat_tcpudp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter, bool is_UDP); static struct nstat_sock_locus * nstat_sock_locus_alloc_internal( struct inpcb *inp, bool locked) { struct nstat_sock_locus *sol; if (SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP) { struct nstat_extended_sock_locus *esol; esol = kalloc_type(struct nstat_extended_sock_locus, Z_WAITOK | Z_ZERO); if (!esol) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_udp_sck_locus_alloc_fails); return NULL; } NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_udp_sck_locus_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_udp_sck_locus_current, nstat_global_udp_sck_locus_max); if (in_pcb_checkstate(inp, WNT_ACQUIRE, locked) == WNT_STOPUSING) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_udp_sck_locus_stop_using); kfree_type(struct nstat_extended_sock_locus, esol); NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_udp_sck_locus_current); return NULL; } sol = &esol->nesl_sock_locus; } else { sol = kalloc_type(struct nstat_sock_locus, Z_WAITOK | Z_ZERO); if (!sol) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tcp_sck_locus_alloc_fails); return NULL; } NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tcp_sck_locus_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_tcp_sck_locus_current, nstat_global_tcp_sck_locus_max); // The acquire here may be redundant, it is balanced by a release when the pcb is detached // Copied from tucookie precedent, which had a different mechanism but similar constraints if (in_pcb_checkstate(inp, WNT_ACQUIRE, locked) == WNT_STOPUSING) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tcp_sck_locus_stop_using); kfree_type(struct nstat_sock_locus, sol); NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tcp_sck_locus_current); return NULL; } } sol->nsl_inp = inp; sol->nsl_magic = NSTAT_SOCK_LOCUS_MAGIC; bzero(sol->nsl_pname, sizeof(sol->nsl_pname)); pid_t initial_pid = inp->inp_socket->last_pid; proc_best_name_for_pid(initial_pid, sol->nsl_pname, sizeof(sol->nsl_pname)); if (sol->nsl_pname[0] != '\0') { sol->nsl_pid = initial_pid; } else { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_sck_fail_first_owner); } inp->inp_nstat_locus = sol; return sol; } static struct nstat_sock_locus * nstat_sock_locus_alloc_locked( struct inpcb *inp) { return nstat_sock_locus_alloc_internal(inp, true); } static void nstat_sock_locus_release( struct nstat_sock_locus *sol, int inplock) { // Note, caller should already hace cleared inp->inp_nstat_locus under lock // to prevent multiple calls to this cleanup function struct nstat_extended_sock_locus *esol = NULL; struct inpcb *inp = sol->nsl_inp; if (SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP) { esol = (struct nstat_extended_sock_locus *)sol; } sol->nsl_magic = NSTAT_SOCK_LOCUS_UNMAGIC; if (esol != NULL) { NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_udp_sck_locus_current); kfree_type(struct nstat_extended_sock_locus, esol); } else { NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tcp_sck_locus_current); kfree_type(struct nstat_sock_locus, sol); } in_pcb_checkstate(inp, WNT_RELEASE, inplock); } static size_t nstat_inp_domain_info(struct inpcb *inp, nstat_domain_info *domain_info, size_t len) { // Note, the caller has guaranteed that the buffer has been zeroed, there is no need to clear it again struct socket *so = inp->inp_socket; if (so == NULL) { return 0; } NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: Collecting stats"); if (domain_info == NULL) { return sizeof(nstat_domain_info); } if (len < sizeof(nstat_domain_info)) { return 0; } necp_copy_inp_domain_info(inp, so, domain_info); NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: Collected stats - domain <%s> owner <%s> ctxt <%s> bundle id <%s> " "is_tracker %d is_non_app_initiated %d is_silent %d", so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid, domain_info->domain_name, domain_info->domain_owner, domain_info->domain_tracker_ctxt, domain_info->domain_attributed_bundle_id, domain_info->is_tracker, domain_info->is_non_app_initiated, domain_info->is_silent); return sizeof(nstat_domain_info); } static size_t nstat_inp_bluetooth_counts(struct inpcb *inp, nstat_interface_counts *buf, size_t len) { // Note, the caller has guaranteed that the buffer has been zeroed, there is no need to clear it again struct socket *so = inp->inp_socket; if (so == NULL) { return 0; } if (buf == NULL) { uint64_t rxbytes = 0; uint64_t txbytes = 0; rxbytes = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_rxbytes, relaxed); txbytes = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_txbytes, relaxed); if ((rxbytes == 0) && (txbytes == 0)) { // It's more efficient to skip sending counts if they're only going to be zero return 0; } return sizeof(nstat_interface_counts); } if (len < sizeof(nstat_interface_counts)) { return 0; } // if the pcb is in the dead state, we should stop using it if (!(intotcpcb(inp)) || inp->inp_state == INPCB_STATE_DEAD) { return 0; } buf->nstat_rxpackets = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_rxpackets, relaxed); buf->nstat_rxbytes = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_rxbytes, relaxed); buf->nstat_txpackets = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_txpackets, relaxed); buf->nstat_txbytes = os_atomic_load(&inp->inp_mstat.ms_bluetooth.ts_txbytes, relaxed); return sizeof(nstat_interface_counts); } static nstat_provider nstat_tcp_provider; static errno_t nstat_tcp_lookup( __unused const void *data, __unused u_int32_t length, __unused nstat_provider_cookie_t *out_cookie) { // Looking up a specific connection is not supported. return ENOTSUP; } static int nstat_tcp_gone( nstat_provider_cookie_t cookie) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp; struct tcpcb *tp; return (!(inp = sol->nsl_inp) || !(tp = intotcpcb(inp)) || inp->inp_state == INPCB_STATE_DEAD) ? 1 : 0; } static errno_t nstat_tcp_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp; bzero(out_counts, sizeof(*out_counts)); if (out_gone) { *out_gone = 0; } // if the pcb is in the dead state, we should stop using it if (nstat_tcp_gone(cookie)) { if (out_gone) { *out_gone = 1; } if (!(inp = sol->nsl_inp) || !intotcpcb(inp)) { return EINVAL; } } inp = sol->nsl_inp; struct tcpcb *tp = intotcpcb(inp); out_counts->nstat_rxpackets = os_atomic_load(&inp->inp_mstat.ms_total.ts_rxpackets, relaxed); out_counts->nstat_rxbytes = os_atomic_load(&inp->inp_mstat.ms_total.ts_rxbytes, relaxed); out_counts->nstat_txpackets = os_atomic_load(&inp->inp_mstat.ms_total.ts_txpackets, relaxed); out_counts->nstat_txbytes = os_atomic_load(&inp->inp_mstat.ms_total.ts_txbytes, relaxed); out_counts->nstat_rxduplicatebytes = tp->t_stat.rxduplicatebytes; out_counts->nstat_rxoutoforderbytes = tp->t_stat.rxoutoforderbytes; out_counts->nstat_txretransmit = tp->t_stat.txretransmitbytes; out_counts->nstat_connectattempts = tp->t_state >= TCPS_SYN_SENT ? 1 : 0; out_counts->nstat_connectsuccesses = tp->t_state >= TCPS_ESTABLISHED ? 1 : 0; out_counts->nstat_avg_rtt = tp->t_srtt; out_counts->nstat_min_rtt = tp->t_rttbest; out_counts->nstat_var_rtt = tp->t_rttvar; if (out_counts->nstat_avg_rtt < out_counts->nstat_min_rtt) { out_counts->nstat_min_rtt = out_counts->nstat_avg_rtt; } out_counts->nstat_cell_rxbytes = os_atomic_load(&inp->inp_mstat.ms_cellular.ts_rxbytes, relaxed); out_counts->nstat_cell_txbytes = os_atomic_load(&inp->inp_mstat.ms_cellular.ts_txbytes, relaxed); out_counts->nstat_wifi_rxbytes = os_atomic_load(&inp->inp_mstat.ms_wifi_infra.ts_rxbytes, relaxed) + os_atomic_load(&inp->inp_mstat.ms_wifi_non_infra.ts_rxbytes, relaxed); out_counts->nstat_wifi_txbytes = os_atomic_load(&inp->inp_mstat.ms_wifi_infra.ts_txbytes, relaxed) + os_atomic_load(&inp->inp_mstat.ms_wifi_non_infra.ts_txbytes, relaxed); out_counts->nstat_wired_rxbytes = os_atomic_load(&inp->inp_mstat.ms_wired.ts_rxbytes, relaxed); out_counts->nstat_wired_txbytes = os_atomic_load(&inp->inp_mstat.ms_wired.ts_txbytes, relaxed); return 0; } static errno_t nstat_tcp_details( nstat_provider_cookie_t cookie, struct nstat_detailed_counts *out_details, int *out_gone) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp; bzero(out_details, sizeof(*out_details)); if (out_gone) { *out_gone = 0; } // if the pcb is in the dead state, we should stop using it if (nstat_tcp_gone(cookie)) { if (out_gone) { *out_gone = 1; } if (!(inp = sol->nsl_inp) || !intotcpcb(inp)) { return EINVAL; } } inp = sol->nsl_inp; struct tcpcb *tp = intotcpcb(inp); if (tp == NULL) { return EINVAL; } memcpy(out_details, &inp->inp_mstat, sizeof(inp->inp_mstat)); out_details->nstat_rxduplicatebytes = tp->t_stat.rxduplicatebytes; out_details->nstat_rxoutoforderbytes = tp->t_stat.rxoutoforderbytes; out_details->nstat_txretransmit = tp->t_stat.txretransmitbytes; out_details->nstat_avg_rtt = tp->t_srtt; out_details->nstat_min_rtt = tp->t_rttbest; out_details->nstat_var_rtt = tp->t_rttvar; if (out_details->nstat_avg_rtt < out_details->nstat_min_rtt) { out_details->nstat_min_rtt = out_details->nstat_avg_rtt; } return 0; } static void nstat_tcp_release( __unused nstat_provider_cookie_t cookie) { } static errno_t nstat_tcp_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { errno_t result; NSTAT_LOCK_EXCLUSIVE(); result = nstat_set_provider_filter(client, req); // Only now can nstat_tcp_new_pcb() find the client via the provider filter // so the NSTAT_LOCK_EXCLUSIVE() ensures that sockets are added once and only once if (result == 0) { OSIncrementAtomic(&nstat_tcp_watchers); struct nstat_sock_locus *sol; TAILQ_FOREACH(sol, &nstat_tcp_sock_locus_head, nsl_link) { assert(sol->nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); assert(sol->nsl_inp != NULL); assert(sol->nsl_is_tcp != 0); struct inpcb *inp = sol->nsl_inp; // Ideally all dead inpcbs should have been removed from the list, but be paranoid if (inp->inp_state == INPCB_STATE_DEAD) { NSTAT_NOTE_QUAL(nstat_add_all_tcp_skip_dead, client, 0); continue; } // The client may not be interested in all TCP flows // There's no need to check the return code from nstat_client_source_add() // which will have performed any recovery actions directly nstat_client_source_add(0, client, &nstat_tcp_provider, sol, &sol->nsl_locus); } } NSTAT_UNLOCK_EXCLUSIVE(); return result; } static void nstat_tcp_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_tcp_watchers); } __private_extern__ void nstat_tcp_new_pcb( struct inpcb *inp) { struct nstat_sock_locus *sol = NULL; inp->inp_start_timestamp = mach_continuous_time(); assert(inp->inp_nstat_locus == NULL); sol = nstat_sock_locus_alloc_locked(inp); if (sol == NULL) { return; } sol->nsl_is_tcp = 1; NSTAT_LOCK_EXCLUSIVE(); TAILQ_INSERT_HEAD(&nstat_tcp_sock_locus_head, sol, nsl_link); nstat_client *client; for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_watching & (1 << NSTAT_PROVIDER_TCP_KERNEL)) != 0) { // this client is watching tcp and may or may not be interested in this example // Metrics are gathered by nstat_client_source_add(), // There's no possible recovery action or logging needed here, no need to check return code nstat_client_source_add(0, client, &nstat_tcp_provider, sol, &sol->nsl_locus); } } NSTAT_UNLOCK_EXCLUSIVE(); } __private_extern__ void nstat_pcb_detach(struct inpcb *inp) { nstat_client *client; nstat_src *src; nstat_src *tmpsrc; tailq_head_nstat_src dead_list; struct nstat_sock_locus *sol = NULL; if (inp == NULL) { return; } TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); sol = inp->inp_nstat_locus; if (sol) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_pcb_detach_with_locus); TAILQ_FOREACH_SAFE(src, &inp->inp_nstat_locus->nsl_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(sol == (struct nstat_sock_locus *)src->nts_cookie); NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_pcb_detach_with_src); client = src->nts_client; nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } if (sol->nsl_is_tcp) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_pcb_detach_tcp); TAILQ_REMOVE(&nstat_tcp_sock_locus_head, sol, nsl_link); } else { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_pcb_detach_udp); TAILQ_REMOVE(&nstat_udp_sock_locus_head, sol, nsl_link); } inp->inp_nstat_locus = NULL; } else { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_pcb_detach_without_locus); } NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } if (sol != NULL) { nstat_sock_locus_release(sol, TRUE); } } __private_extern__ void nstat_pcb_event(struct inpcb *inp, u_int64_t event) { nstat_client *client; nstat_src *src; struct nstat_sock_locus *sol; errno_t result; nstat_provider_id_t provider_id; if (inp == NULL || (inp->inp_nstat_locus == NULL) || (nstat_tcp_watchers == 0 && nstat_udp_watchers == 0)) { return; } if (((merged_filters.mpf_filters[NSTAT_PROVIDER_TCP_KERNEL].mf_events & event) == 0) && ((merged_filters.mpf_filters[NSTAT_PROVIDER_UDP_KERNEL].mf_events & event) == 0)) { // There are clients for TCP and UDP, but none are interested in the event // This check saves taking the mutex and scanning the list return; } sol = inp->inp_nstat_locus; NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH(src, &sol->nsl_locus.ntl_src_queue, nts_locus_link) { assert(sol == (struct nstat_sock_locus *)src->nts_cookie); client = src->nts_client; provider_id = src->nts_provider->nstat_provider_id; assert((provider_id == NSTAT_PROVIDER_TCP_KERNEL) || (provider_id == NSTAT_PROVIDER_UDP_KERNEL)); struct nstat_provider_filter *filter = &client->ntc_provider_filters[provider_id]; bool isUDP = (provider_id == NSTAT_PROVIDER_UDP_KERNEL); if (((filter->npf_events & event) != 0) && (nstat_tcpudp_reporting_allowed(src->nts_cookie, filter, isUDP))) { NSTAT_NOTE_SRC(nstat_pcb_event, client, src); result = nstat_client_send_event(client, src, event); } } NSTAT_UNLOCK_EXCLUSIVE(); } __private_extern__ void nstat_udp_pcb_cache(struct inpcb *inp) { if (inp == NULL) { return; } VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); NSTAT_LOCK_EXCLUSIVE(); struct nstat_extended_sock_locus *esol = (struct nstat_extended_sock_locus *)inp->inp_nstat_locus; if (esol != NULL) { assert(esol->nesl_sock_locus.nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); if (inp->inp_vflag & INP_IPV6) { in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, inp->inp_lifscope, &esol->nesl_local.v6, sizeof(esol->nesl_local)); in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, inp->inp_fifscope, &esol->nesl_remote.v6, sizeof(esol->nesl_remote)); } else if (inp->inp_vflag & INP_IPV4) { nstat_ip_to_sockaddr(&inp->inp_laddr, inp->inp_lport, &esol->nesl_local.v4, sizeof(esol->nesl_local)); nstat_ip_to_sockaddr(&inp->inp_faddr, inp->inp_fport, &esol->nesl_remote.v4, sizeof(esol->nesl_remote)); } else { bzero(&esol->nesl_local, sizeof(esol->nesl_local)); bzero(&esol->nesl_remote, sizeof(esol->nesl_remote)); } if (inp->inp_last_outifp) { esol->nesl_if_index = inp->inp_last_outifp->if_index; } else { esol->nesl_if_index = 0; } esol->nesl_sock_locus.nsl_ifnet_properties = nstat_inpcb_to_flags(inp); esol->nesl_cached = true; } NSTAT_UNLOCK_EXCLUSIVE(); } __private_extern__ void nstat_udp_pcb_invalidate_cache(struct inpcb *inp) { if (inp == NULL) { return; } VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); struct nstat_extended_sock_locus *esol = (struct nstat_extended_sock_locus *)inp->inp_nstat_locus; if (esol != NULL) { assert(esol->nesl_sock_locus.nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); if (esol->nesl_cached) { NSTAT_LOCK_EXCLUSIVE(); esol->nesl_cached = false; NSTAT_UNLOCK_EXCLUSIVE(); } } } __private_extern__ void nstat_pcb_update_last_owner(struct inpcb *inp) { bool cause_event = false; NSTAT_LOCK_EXCLUSIVE(); struct nstat_sock_locus *sol = inp->inp_nstat_locus; if (sol != NULL) { pid_t current_pid = inp->inp_socket->last_pid; if (sol->nsl_pid != current_pid) { proc_best_name_for_pid(current_pid, sol->nsl_pname, sizeof(sol->nsl_pname)); if (sol->nsl_pname[0] != '\0') { sol->nsl_pid = current_pid; cause_event = true; NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_sck_update_last_owner); } else { // How best to recover? Just leaving things untouched seems reasonable NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_sck_fail_last_owner); } } } NSTAT_UNLOCK_EXCLUSIVE(); if (cause_event) { nstat_pcb_event(inp, NSTAT_EVENT_SRC_DID_CHANGE_OWNER); } } static errno_t nstat_tcp_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, size_t len) { if (len < sizeof(nstat_tcp_descriptor)) { return EINVAL; } if (nstat_tcp_gone(cookie)) { return EINVAL; } nstat_tcp_descriptor *desc = (nstat_tcp_descriptor*)data; struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp = sol->nsl_inp; struct tcpcb *tp = intotcpcb(inp); bzero(desc, sizeof(*desc)); if (inp->inp_vflag & INP_IPV6) { in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, inp->inp_lifscope, &desc->local.v6, sizeof(desc->local)); in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, inp->inp_fifscope, &desc->remote.v6, sizeof(desc->remote)); } else if (inp->inp_vflag & INP_IPV4) { nstat_ip_to_sockaddr(&inp->inp_laddr, inp->inp_lport, &desc->local.v4, sizeof(desc->local)); nstat_ip_to_sockaddr(&inp->inp_faddr, inp->inp_fport, &desc->remote.v4, sizeof(desc->remote)); } desc->state = intotcpcb(inp)->t_state; desc->ifindex = (inp->inp_last_outifp == NULL) ? 0 : inp->inp_last_outifp->if_index; // danger - not locked, values could be bogus desc->txunacked = tp->snd_max - tp->snd_una; desc->txwindow = tp->snd_wnd; desc->txcwindow = tp->snd_cwnd; desc->ifnet_properties = nstat_inpcb_to_flags(inp); if (CC_ALGO(tp)->name != NULL) { strbufcpy(desc->cc_algo, CC_ALGO(tp)->name); } struct socket *so = inp->inp_socket; if (so) { // TBD - take the socket lock around these to make sure // they're in sync? desc->upid = so->last_upid; desc->pid = so->last_pid; desc->traffic_class = so->so_traffic_class; if ((so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND)) { desc->traffic_mgt_flags |= TRAFFIC_MGT_SO_BACKGROUND; } if ((so->so_flags1 & SOF1_TRAFFIC_MGT_TCP_RECVBG)) { desc->traffic_mgt_flags |= TRAFFIC_MGT_TCP_RECVBG; } if (so->so_flags1 & SOF1_INBOUND) { desc->ifnet_properties |= NSTAT_SOURCE_IS_INBOUND; } else if (desc->state == TCPS_LISTEN) { desc->ifnet_properties |= NSTAT_SOURCE_IS_LISTENER; sol->nsl_ifnet_properties = NSTAT_SOURCE_IS_LISTENER; } else if (desc->state != TCPS_CLOSED) { desc->ifnet_properties |= NSTAT_SOURCE_IS_OUTBOUND; sol->nsl_ifnet_properties = NSTAT_SOURCE_IS_OUTBOUND; } else { desc->ifnet_properties |= sol->nsl_ifnet_properties; } if ((desc->pid == sol->nsl_pid) && (sol->nsl_pname[0] != '\0')) { // There should be a nicely cached name to use, for efficiency strbufcpy(desc->pname, sol->nsl_pname); } else { proc_best_name_for_pid(desc->pid, desc->pname, sizeof(desc->pname)); if (desc->pname[0] != '\0') { // This may not be fully synchronized but multiple updates should be fine desc->pname[sizeof(desc->pname) - 1] = 0; strbufcpy(sol->nsl_pname, desc->pname); sol->nsl_pid = desc->pid; NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tcp_desc_new_name); } else { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tcp_desc_fail_name); } } memcpy(desc->uuid, so->last_uuid, sizeof(so->last_uuid)); memcpy(desc->vuuid, so->so_vuuid, sizeof(so->so_vuuid)); if (so->so_flags & SOF_DELEGATED) { desc->eupid = so->e_upid; desc->epid = so->e_pid; memcpy(desc->euuid, so->e_uuid, sizeof(so->e_uuid)); } else if (!uuid_is_null(so->so_ruuid)) { memcpy(desc->euuid, so->so_ruuid, sizeof(so->so_ruuid)); } else { desc->eupid = desc->upid; desc->epid = desc->pid; memcpy(desc->euuid, desc->uuid, sizeof(desc->uuid)); } uuid_copy(desc->fuuid, inp->necp_client_uuid); desc->persona_id = so->so_persona_id; desc->uid = kauth_cred_getuid(so->so_cred); desc->sndbufsize = so->so_snd.sb_hiwat; desc->sndbufused = so->so_snd.sb_cc; desc->rcvbufsize = so->so_rcv.sb_hiwat; desc->rcvbufused = so->so_rcv.sb_cc; desc->fallback_mode = so->so_fallback_mode; if (nstat_debug) { uuid_string_t euuid_str = { 0 }; uuid_unparse(desc->euuid, euuid_str); NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: TCP - pid %d uid %d euuid %s persona id %d", desc->pid, desc->uid, euuid_str, desc->persona_id); } } tcp_get_connectivity_status(tp, &desc->connstatus); inp_get_activity_bitmap(inp, &desc->activity_bitmap); desc->start_timestamp = inp->inp_start_timestamp; desc->timestamp = mach_continuous_time(); return 0; } static bool nstat_tcpudp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter, bool is_UDP) { bool retval = true; if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS | NSTAT_FILTER_SPECIFIC_USER)) != 0) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp = sol->nsl_inp; /* Only apply interface filter if at least one is allowed. */ if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) { uint32_t interface_properties = nstat_inpcb_to_flags(inp); if ((filter->npf_flags & interface_properties) == 0) { // For UDP, we could have an undefined interface and yet transfers may have occurred. // We allow reporting if there have been transfers of the requested kind. // This is imperfect as we cannot account for the expensive attribute over wifi. // We also assume that cellular is expensive and we have no way to select for AWDL if (is_UDP) { do{ if ((filter->npf_flags & (NSTAT_FILTER_ACCEPT_CELLULAR | NSTAT_FILTER_ACCEPT_EXPENSIVE)) && (inp->inp_mstat.ms_cellular.ts_rxbytes || inp->inp_mstat.ms_cellular.ts_txbytes)) { break; } if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_WIFI) && (inp->inp_mstat.ms_wifi_infra.ts_rxbytes || inp->inp_mstat.ms_wifi_infra.ts_txbytes || inp->inp_mstat.ms_wifi_non_infra.ts_rxbytes || inp->inp_mstat.ms_wifi_non_infra.ts_txbytes)) { break; } if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_WIFI_INFRA) && (inp->inp_mstat.ms_wifi_infra.ts_rxbytes || inp->inp_mstat.ms_wifi_infra.ts_txbytes)) { break; } if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_AWDL) && (inp->inp_mstat.ms_wifi_non_infra.ts_rxbytes || inp->inp_mstat.ms_wifi_non_infra.ts_txbytes)) { break; } if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_WIRED) && (inp->inp_mstat.ms_wired.ts_rxbytes || inp->inp_mstat.ms_wired.ts_txbytes)) { break; } if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_COMPANIONLINK_BT) && (inp->inp_mstat.ms_bluetooth.ts_rxbytes || inp->inp_mstat.ms_bluetooth.ts_txbytes)) { break; } return false; } while (0); } else { return false; } } } if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) && (retval)) { struct socket *so = inp->inp_socket; retval = false; if (so) { if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) && (filter->npf_pid == so->last_pid)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) && (filter->npf_pid == (so->so_flags & SOF_DELEGATED)? so->e_upid : so->last_pid)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) && (memcmp(filter->npf_uuid, so->last_uuid, sizeof(so->last_uuid)) == 0)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) && (memcmp(filter->npf_uuid, (so->so_flags & SOF_DELEGATED)? so->e_uuid : so->last_uuid, sizeof(so->last_uuid)) == 0)) { retval = true; } } } } return retval; } static bool nstat_tcp_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { return nstat_tcpudp_reporting_allowed(cookie, filter, FALSE); } static size_t nstat_tcp_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp = sol->nsl_inp; if (nstat_tcp_gone(cookie)) { return 0; } switch (extension_id) { case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN: return nstat_inp_domain_info(inp, (nstat_domain_info *)buf, len); case NSTAT_EXTENDED_UPDATE_TYPE_BLUETOOTH_COUNTS: return nstat_inp_bluetooth_counts(inp, (nstat_interface_counts *)buf, len); case NSTAT_EXTENDED_UPDATE_TYPE_NECP_TLV: default: break; } return 0; } static void nstat_init_tcp_provider(void) { bzero(&nstat_tcp_provider, sizeof(nstat_tcp_provider)); nstat_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor); nstat_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_KERNEL; nstat_tcp_provider.nstat_lookup = nstat_tcp_lookup; nstat_tcp_provider.nstat_gone = nstat_tcp_gone; nstat_tcp_provider.nstat_counts = nstat_tcp_counts; nstat_tcp_provider.nstat_details = nstat_tcp_details; nstat_tcp_provider.nstat_release = nstat_tcp_release; nstat_tcp_provider.nstat_watcher_add = nstat_tcp_add_watcher; nstat_tcp_provider.nstat_watcher_remove = nstat_tcp_remove_watcher; nstat_tcp_provider.nstat_copy_descriptor = nstat_tcp_copy_descriptor; nstat_tcp_provider.nstat_reporting_allowed = nstat_tcp_reporting_allowed; nstat_tcp_provider.nstat_copy_extension = nstat_tcp_extensions; nstat_tcp_provider.next = nstat_providers; nstat_providers = &nstat_tcp_provider; } #pragma mark -- UDP Provider -- static nstat_provider nstat_udp_provider; static errno_t nstat_udp_lookup( __unused const void *data, __unused u_int32_t length, __unused nstat_provider_cookie_t *out_cookie) { // Looking up a specific connection is not supported. return ENOTSUP; } static int nstat_udp_gone( nstat_provider_cookie_t cookie) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp; return (!(inp = sol->nsl_inp) || inp->inp_state == INPCB_STATE_DEAD) ? 1 : 0; } static errno_t nstat_udp_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; if (out_gone) { *out_gone = 0; } // if the pcb is in the dead state, we should stop using it if (nstat_udp_gone(cookie)) { if (out_gone) { *out_gone = 1; } if (!sol->nsl_inp) { return EINVAL; } } struct inpcb *inp = sol->nsl_inp; out_counts->nstat_rxpackets = os_atomic_load(&inp->inp_mstat.ms_total.ts_rxpackets, relaxed); out_counts->nstat_rxbytes = os_atomic_load(&inp->inp_mstat.ms_total.ts_rxbytes, relaxed); out_counts->nstat_txpackets = os_atomic_load(&inp->inp_mstat.ms_total.ts_txpackets, relaxed); out_counts->nstat_txbytes = os_atomic_load(&inp->inp_mstat.ms_total.ts_txbytes, relaxed); out_counts->nstat_cell_rxbytes = os_atomic_load(&inp->inp_mstat.ms_cellular.ts_rxbytes, relaxed); out_counts->nstat_cell_txbytes = os_atomic_load(&inp->inp_mstat.ms_cellular.ts_txbytes, relaxed); out_counts->nstat_wifi_rxbytes = os_atomic_load(&inp->inp_mstat.ms_wifi_infra.ts_rxbytes, relaxed) + os_atomic_load(&inp->inp_mstat.ms_wifi_non_infra.ts_rxbytes, relaxed); out_counts->nstat_wifi_txbytes = os_atomic_load(&inp->inp_mstat.ms_wifi_infra.ts_txbytes, relaxed) + os_atomic_load(&inp->inp_mstat.ms_wifi_non_infra.ts_txbytes, relaxed); out_counts->nstat_wired_rxbytes = os_atomic_load(&inp->inp_mstat.ms_wired.ts_rxbytes, relaxed); out_counts->nstat_wired_txbytes = os_atomic_load(&inp->inp_mstat.ms_wired.ts_txbytes, relaxed); return 0; } static errno_t nstat_udp_details( nstat_provider_cookie_t cookie, struct nstat_detailed_counts *out_details, int *out_gone) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; if (out_gone) { *out_gone = 0; } // if the pcb is in the dead state, we should stop using it if (nstat_udp_gone(cookie)) { if (out_gone) { *out_gone = 1; } } struct inpcb *inp = sol->nsl_inp; // Note, other field in the out_details structure guaranteed to be zeroed memcpy(out_details, &inp->inp_mstat, sizeof(inp->inp_mstat)); return 0; } static void nstat_udp_release( __unused nstat_provider_cookie_t cookie) { } static errno_t nstat_udp_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { errno_t result; NSTAT_LOCK_EXCLUSIVE(); result = nstat_set_provider_filter(client, req); // Only now can nstat_udp_new_pcb() find the client via the provider filter // so the NSTAT_LOCK_EXCLUSIVE() ensures that sockets are added once and only once if (result == 0) { OSIncrementAtomic(&nstat_udp_watchers); struct nstat_sock_locus *sol; TAILQ_FOREACH(sol, &nstat_udp_sock_locus_head, nsl_link) { assert(sol->nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); assert(sol->nsl_inp != NULL); struct inpcb *inp = sol->nsl_inp; // Ideally all dead inpcbs should have been removed from the list, but be paranoid if (inp->inp_state == INPCB_STATE_DEAD) { NSTAT_NOTE_QUAL(nstat_add_all_udp_skip_dead, client, 0); continue; } // The client may not be interested in all TCP flows // There's no need to check the return code from nstat_client_source_add() // which will have performed any recovery actions directly nstat_client_source_add(0, client, &nstat_udp_provider, sol, &sol->nsl_locus); } } NSTAT_UNLOCK_EXCLUSIVE(); return result; } static void nstat_udp_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_udp_watchers); } __private_extern__ void nstat_udp_new_pcb( struct inpcb *inp) { struct nstat_sock_locus *sol = NULL; inp->inp_start_timestamp = mach_continuous_time(); assert(inp->inp_nstat_locus == NULL); sol = nstat_sock_locus_alloc_locked(inp); if (sol == NULL) { return; } sol->nsl_is_tcp = 0; NSTAT_LOCK_EXCLUSIVE(); TAILQ_INSERT_HEAD(&nstat_udp_sock_locus_head, sol, nsl_link); nstat_client *client; for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_watching & (1 << NSTAT_PROVIDER_UDP_KERNEL)) != 0) { // this client is watching tcp and may or may not be interested in this example // Metrics are gathered by nstat_client_source_add(), // There's no possible recovery action or logging needed here, no need to check return code nstat_client_source_add(0, client, &nstat_udp_provider, sol, &sol->nsl_locus); } } NSTAT_UNLOCK_EXCLUSIVE(); } static errno_t nstat_udp_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, size_t len) { if (len < sizeof(nstat_udp_descriptor)) { return EINVAL; } if (nstat_udp_gone(cookie)) { return EINVAL; } struct nstat_extended_sock_locus *esol = (struct nstat_extended_sock_locus *)cookie; nstat_udp_descriptor *desc = (nstat_udp_descriptor*)data; struct inpcb *inp = esol->nesl_sock_locus.nsl_inp; bzero(desc, sizeof(*desc)); if (esol->nesl_cached == false) { if (inp->inp_vflag & INP_IPV6) { in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, inp->inp_lifscope, &desc->local.v6, sizeof(desc->local.v6)); in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, inp->inp_fifscope, &desc->remote.v6, sizeof(desc->remote.v6)); } else if (inp->inp_vflag & INP_IPV4) { nstat_ip_to_sockaddr(&inp->inp_laddr, inp->inp_lport, &desc->local.v4, sizeof(desc->local.v4)); nstat_ip_to_sockaddr(&inp->inp_faddr, inp->inp_fport, &desc->remote.v4, sizeof(desc->remote.v4)); } desc->ifnet_properties = nstat_inpcb_to_flags(inp); } else { if (inp->inp_vflag & INP_IPV6) { memcpy(&desc->local.v6, &esol->nesl_local.v6, sizeof(desc->local.v6)); memcpy(&desc->remote.v6, &esol->nesl_remote.v6, sizeof(desc->remote.v6)); } else if (inp->inp_vflag & INP_IPV4) { memcpy(&desc->local.v4, &esol->nesl_local.v4, sizeof(desc->local.v4)); memcpy(&desc->remote.v4, &esol->nesl_remote.v4, sizeof(desc->remote.v4)); } desc->ifnet_properties = esol->nesl_sock_locus.nsl_ifnet_properties; } if (inp->inp_last_outifp) { desc->ifindex = inp->inp_last_outifp->if_index; } else { desc->ifindex = esol->nesl_if_index; } struct socket *so = inp->inp_socket; if (so) { // TBD - take the socket lock around these to make sure // they're in sync? desc->upid = so->last_upid; desc->pid = so->last_pid; if (so->so_flags1 & SOF1_INBOUND) { desc->ifnet_properties |= NSTAT_SOURCE_IS_INBOUND; } else { desc->ifnet_properties |= esol->nesl_sock_locus.nsl_ifnet_properties; } if (desc->pid == esol->nesl_sock_locus.nsl_pid) { // There should be a nicely cached name to use, for efficiency strbufcpy(desc->pname, esol->nesl_sock_locus.nsl_pname); } else { proc_best_name_for_pid(desc->pid, desc->pname, sizeof(desc->pname)); if (desc->pname[0] != '\0') { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_udp_desc_new_name); // This may not be fully synchronized but multiple updates should be fine desc->pname[sizeof(desc->pname) - 1] = 0; strbufcpy(esol->nesl_sock_locus.nsl_pname, desc->pname); esol->nesl_sock_locus.nsl_pid = desc->pid; } else { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_udp_desc_fail_name); } } memcpy(desc->uuid, so->last_uuid, sizeof(so->last_uuid)); memcpy(desc->vuuid, so->so_vuuid, sizeof(so->so_vuuid)); if (so->so_flags & SOF_DELEGATED) { desc->eupid = so->e_upid; desc->epid = so->e_pid; memcpy(desc->euuid, so->e_uuid, sizeof(so->e_uuid)); } else if (!uuid_is_null(so->so_ruuid)) { memcpy(desc->euuid, so->so_ruuid, sizeof(so->so_ruuid)); } else { desc->eupid = desc->upid; desc->epid = desc->pid; memcpy(desc->euuid, desc->uuid, sizeof(desc->uuid)); } uuid_copy(desc->fuuid, inp->necp_client_uuid); desc->persona_id = so->so_persona_id; desc->uid = kauth_cred_getuid(so->so_cred); desc->rcvbufsize = so->so_rcv.sb_hiwat; desc->rcvbufused = so->so_rcv.sb_cc; desc->traffic_class = so->so_traffic_class; desc->fallback_mode = so->so_fallback_mode; inp_get_activity_bitmap(inp, &desc->activity_bitmap); desc->start_timestamp = inp->inp_start_timestamp; desc->timestamp = mach_continuous_time(); if (nstat_debug) { uuid_string_t euuid_str = { 0 }; uuid_unparse(desc->euuid, euuid_str); NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: UDP - pid %d uid %d euuid %s persona id %d", desc->pid, desc->uid, euuid_str, desc->persona_id); } } return 0; } static bool nstat_udp_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { return nstat_tcpudp_reporting_allowed(cookie, filter, TRUE); } static size_t nstat_udp_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len) { struct nstat_sock_locus *sol = (struct nstat_sock_locus *)cookie; struct inpcb *inp = sol->nsl_inp; if (nstat_udp_gone(cookie)) { return 0; } switch (extension_id) { case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN: return nstat_inp_domain_info(inp, (nstat_domain_info *)buf, len); case NSTAT_EXTENDED_UPDATE_TYPE_BLUETOOTH_COUNTS: return nstat_inp_bluetooth_counts(inp, (nstat_interface_counts *)buf, len); default: break; } return 0; } static void nstat_init_udp_provider(void) { bzero(&nstat_udp_provider, sizeof(nstat_udp_provider)); nstat_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_KERNEL; nstat_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); nstat_udp_provider.nstat_lookup = nstat_udp_lookup; nstat_udp_provider.nstat_gone = nstat_udp_gone; nstat_udp_provider.nstat_counts = nstat_udp_counts; nstat_udp_provider.nstat_details = nstat_udp_details; nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher; nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher; nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor; nstat_udp_provider.nstat_release = nstat_udp_release; nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed; nstat_udp_provider.nstat_copy_extension = nstat_udp_extensions; nstat_udp_provider.next = nstat_providers; nstat_providers = &nstat_udp_provider; } #if SKYWALK #pragma mark -- TCP/UDP/QUIC Userland // Almost all of this infrastucture is common to both TCP and UDP static u_int32_t nstat_userland_quic_watchers = 0; static u_int32_t nstat_userland_udp_watchers = 0; static u_int32_t nstat_userland_tcp_watchers = 0; static u_int32_t nstat_userland_quic_shadows = 0; static u_int32_t nstat_userland_udp_shadows = 0; static u_int32_t nstat_userland_tcp_shadows = 0; static nstat_provider nstat_userland_quic_provider; static nstat_provider nstat_userland_udp_provider; static nstat_provider nstat_userland_tcp_provider; enum nstat_rnf_override { nstat_rnf_override_not_set, nstat_rnf_override_enabled, nstat_rnf_override_disabled }; struct nstat_tu_shadow { nstat_locus shad_locus; tailq_entry_tu_shadow shad_link; userland_stats_request_vals_fn *shad_getvals_fn; userland_stats_request_extension_fn *shad_get_extension_fn; userland_stats_provider_context *shad_provider_context; u_int64_t shad_properties; u_int64_t shad_start_timestamp; nstat_provider_id_t shad_provider; struct nstat_procdetails *shad_procdetails; bool shad_live; // false if defunct enum nstat_rnf_override shad_rnf_override; uint32_t shad_magic; }; // Magic number checking should remain in place until the userland provider has been fully proven #define TU_SHADOW_MAGIC 0xfeedf00d #define TU_SHADOW_UNMAGIC 0xdeaddeed static tailq_head_tu_shadow nstat_userprot_shad_head = TAILQ_HEAD_INITIALIZER(nstat_userprot_shad_head); static tailq_head_tu_shadow nstat_defunct_userprot_shad_head = TAILQ_HEAD_INITIALIZER(nstat_defunct_userprot_shad_head); static errno_t nstat_userland_tu_lookup( __unused const void *data, __unused u_int32_t length, __unused nstat_provider_cookie_t *out_cookie) { // Looking up a specific connection is not supported return ENOTSUP; } static int nstat_userland_tu_gone( __unused nstat_provider_cookie_t cookie) { // Returns non-zero if the source has gone. // We don't keep a source hanging around, so the answer is always 0 return 0; } static errno_t nstat_userland_tu_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_live); bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, out_counts, NULL, NULL); if (out_gone) { *out_gone = 0; } return (result)? 0 : EIO; } static errno_t nstat_userland_tu_details( nstat_provider_cookie_t cookie, struct nstat_detailed_counts *out_details, int *out_gone) { struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_live); bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, out_details, NULL); if (out_gone) { *out_gone = 0; } return (result)? 0 : EIO; } static errno_t nstat_userland_tu_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, __unused size_t len) { struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_live); struct nstat_procdetails *procdetails = shad->shad_procdetails; assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, NULL, data); switch (shad->shad_provider) { case NSTAT_PROVIDER_TCP_USERLAND: { nstat_tcp_descriptor *desc = (nstat_tcp_descriptor *)data; desc->pid = procdetails->pdet_pid; desc->upid = procdetails->pdet_upid; uuid_copy(desc->uuid, procdetails->pdet_uuid); strbufcpy(desc->pname, procdetails->pdet_procname); if (shad->shad_rnf_override == nstat_rnf_override_enabled) { desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_FAST; } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) { desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_NONE; } desc->ifnet_properties |= (uint32_t)shad->shad_properties; desc->start_timestamp = shad->shad_start_timestamp; desc->timestamp = mach_continuous_time(); } break; case NSTAT_PROVIDER_UDP_USERLAND: { nstat_udp_descriptor *desc = (nstat_udp_descriptor *)data; desc->pid = procdetails->pdet_pid; desc->upid = procdetails->pdet_upid; uuid_copy(desc->uuid, procdetails->pdet_uuid); strbufcpy(desc->pname, procdetails->pdet_procname); if (shad->shad_rnf_override == nstat_rnf_override_enabled) { desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_FAST; } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) { desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_NONE; } desc->ifnet_properties |= (uint32_t)shad->shad_properties; desc->start_timestamp = shad->shad_start_timestamp; desc->timestamp = mach_continuous_time(); } break; case NSTAT_PROVIDER_QUIC_USERLAND: { nstat_quic_descriptor *desc = (nstat_quic_descriptor *)data; desc->pid = procdetails->pdet_pid; desc->upid = procdetails->pdet_upid; uuid_copy(desc->uuid, procdetails->pdet_uuid); strbufcpy(desc->pname, procdetails->pdet_procname); if (shad->shad_rnf_override == nstat_rnf_override_enabled) { desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_FAST; } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) { desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK; desc->fallback_mode = SO_FALLBACK_MODE_NONE; } desc->ifnet_properties |= (uint32_t)shad->shad_properties; desc->start_timestamp = shad->shad_start_timestamp; desc->timestamp = mach_continuous_time(); } break; default: break; } return (result)? 0 : EIO; } static void nstat_userland_tu_release( __unused nstat_provider_cookie_t cookie) { // Called when a nstat_src is detached. // We don't reference count or ask for delayed release so nothing to do here. // Note that any associated nstat_tu_shadow may already have been released. } static bool check_reporting_for_user(nstat_provider_filter *filter, pid_t pid, pid_t epid, uuid_t *uuid, uuid_t *euuid) { bool retval = true; if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) { retval = false; if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) && (filter->npf_pid == pid)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) && (filter->npf_pid == epid)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) && (memcmp(filter->npf_uuid, uuid, sizeof(*uuid)) == 0)) { retval = true; } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) && (memcmp(filter->npf_uuid, euuid, sizeof(*euuid)) == 0)) { retval = true; } } return retval; } static bool nstat_userland_tcp_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { bool retval = true; struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) { u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE; if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL, NULL)) { if ((filter->npf_flags & ifflags) == 0) { return false; } } } if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) { nstat_tcp_descriptor tcp_desc; // Stack allocation - OK or pushing the limits too far? if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, NULL, &tcp_desc)) { retval = check_reporting_for_user(filter, (pid_t)tcp_desc.pid, (pid_t)tcp_desc.epid, &tcp_desc.uuid, &tcp_desc.euuid); } else { retval = false; // No further information, so might as well give up now. } } return retval; } static size_t nstat_userland_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *__sized_by(len)buf, size_t len) { struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_live); assert(shad->shad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); return shad->shad_get_extension_fn(shad->shad_provider_context, extension_id, buf, len); } static bool nstat_userland_udp_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { bool retval = true; struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) { u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE; if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL, NULL)) { if ((filter->npf_flags & ifflags) == 0) { return false; } } } if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) { nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far? if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, NULL, &udp_desc)) { retval = check_reporting_for_user(filter, (pid_t)udp_desc.pid, (pid_t)udp_desc.epid, &udp_desc.uuid, &udp_desc.euuid); } else { retval = false; // No further information, so might as well give up now. } } return retval; } static bool nstat_userland_quic_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, __unused u_int64_t suppression_flags) { bool retval = true; struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; assert(shad->shad_magic == TU_SHADOW_MAGIC); if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) { u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE; if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL, NULL)) { if ((filter->npf_flags & ifflags) == 0) { return false; } } } if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) { nstat_quic_descriptor quic_desc; // Stack allocation - OK or pushing the limits too far? if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, NULL, &quic_desc)) { retval = check_reporting_for_user(filter, (pid_t)quic_desc.pid, (pid_t)quic_desc.epid, &quic_desc.uuid, &quic_desc.euuid); } else { retval = false; // No further information, so might as well give up now. } } return retval; } static errno_t nstat_userland_protocol_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req, nstat_provider_type_t nstat_provider_type, nstat_provider *nstat_provider, u_int32_t *proto_watcher_cnt) { errno_t result; NSTAT_LOCK_EXCLUSIVE(); result = nstat_set_provider_filter(client, req); if (result == 0) { struct nstat_tu_shadow *shad; OSIncrementAtomic(proto_watcher_cnt); TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { assert(shad->shad_magic == TU_SHADOW_MAGIC); if (shad->shad_provider == nstat_provider_type) { result = nstat_client_source_add(0, client, nstat_provider, shad, &shad->shad_locus); if (result != 0) { NSTAT_LOG_ERROR("nstat_client_source_add returned %d for " "provider type: %d", result, nstat_provider_type); break; } } } } NSTAT_UNLOCK_EXCLUSIVE(); return result; } static errno_t nstat_userland_tcp_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_TCP_USERLAND, &nstat_userland_tcp_provider, &nstat_userland_tcp_watchers); } static errno_t nstat_userland_udp_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_UDP_USERLAND, &nstat_userland_udp_provider, &nstat_userland_udp_watchers); } static errno_t nstat_userland_quic_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_QUIC_USERLAND, &nstat_userland_quic_provider, &nstat_userland_quic_watchers); } static void nstat_userland_tcp_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_userland_tcp_watchers); } static void nstat_userland_udp_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_userland_udp_watchers); } static void nstat_userland_quic_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_userland_quic_watchers); } static void nstat_init_userland_tcp_provider(void) { bzero(&nstat_userland_tcp_provider, sizeof(nstat_userland_tcp_provider)); nstat_userland_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor); nstat_userland_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_USERLAND; nstat_userland_tcp_provider.nstat_lookup = nstat_userland_tu_lookup; nstat_userland_tcp_provider.nstat_gone = nstat_userland_tu_gone; nstat_userland_tcp_provider.nstat_counts = nstat_userland_tu_counts; nstat_userland_tcp_provider.nstat_details = nstat_userland_tu_details; nstat_userland_tcp_provider.nstat_release = nstat_userland_tu_release; nstat_userland_tcp_provider.nstat_watcher_add = nstat_userland_tcp_add_watcher; nstat_userland_tcp_provider.nstat_watcher_remove = nstat_userland_tcp_remove_watcher; nstat_userland_tcp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; nstat_userland_tcp_provider.nstat_reporting_allowed = nstat_userland_tcp_reporting_allowed; nstat_userland_tcp_provider.nstat_copy_extension = nstat_userland_extensions; nstat_userland_tcp_provider.next = nstat_providers; nstat_providers = &nstat_userland_tcp_provider; } static void nstat_init_userland_udp_provider(void) { bzero(&nstat_userland_udp_provider, sizeof(nstat_userland_udp_provider)); nstat_userland_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); nstat_userland_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_USERLAND; nstat_userland_udp_provider.nstat_lookup = nstat_userland_tu_lookup; nstat_userland_udp_provider.nstat_gone = nstat_userland_tu_gone; nstat_userland_udp_provider.nstat_counts = nstat_userland_tu_counts; nstat_userland_udp_provider.nstat_details = nstat_userland_tu_details; nstat_userland_udp_provider.nstat_release = nstat_userland_tu_release; nstat_userland_udp_provider.nstat_watcher_add = nstat_userland_udp_add_watcher; nstat_userland_udp_provider.nstat_watcher_remove = nstat_userland_udp_remove_watcher; nstat_userland_udp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; nstat_userland_udp_provider.nstat_reporting_allowed = nstat_userland_udp_reporting_allowed; nstat_userland_udp_provider.nstat_copy_extension = nstat_userland_extensions; nstat_userland_udp_provider.next = nstat_providers; nstat_providers = &nstat_userland_udp_provider; } static void nstat_init_userland_quic_provider(void) { bzero(&nstat_userland_quic_provider, sizeof(nstat_userland_quic_provider)); nstat_userland_quic_provider.nstat_descriptor_length = sizeof(nstat_quic_descriptor); nstat_userland_quic_provider.nstat_provider_id = NSTAT_PROVIDER_QUIC_USERLAND; nstat_userland_quic_provider.nstat_lookup = nstat_userland_tu_lookup; nstat_userland_quic_provider.nstat_gone = nstat_userland_tu_gone; nstat_userland_quic_provider.nstat_counts = nstat_userland_tu_counts; nstat_userland_quic_provider.nstat_details = nstat_userland_tu_details; nstat_userland_quic_provider.nstat_release = nstat_userland_tu_release; nstat_userland_quic_provider.nstat_watcher_add = nstat_userland_quic_add_watcher; nstat_userland_quic_provider.nstat_watcher_remove = nstat_userland_quic_remove_watcher; nstat_userland_quic_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; nstat_userland_quic_provider.nstat_reporting_allowed = nstat_userland_quic_reporting_allowed; nstat_userland_quic_provider.nstat_copy_extension = nstat_userland_extensions; nstat_userland_quic_provider.next = nstat_providers; nstat_providers = &nstat_userland_quic_provider; } // Things get started with a call to netstats to say that there’s a new connection: __private_extern__ nstat_userland_context ntstat_userland_stats_open(userland_stats_provider_context *ctx, int provider_id, u_int64_t properties, userland_stats_request_vals_fn req_fn, userland_stats_request_extension_fn req_extension_fn) { struct nstat_tu_shadow *shad; struct nstat_procdetails *procdetails; nstat_provider *provider; if ((provider_id != NSTAT_PROVIDER_TCP_USERLAND) && (provider_id != NSTAT_PROVIDER_UDP_USERLAND) && (provider_id != NSTAT_PROVIDER_QUIC_USERLAND)) { NSTAT_LOG_ERROR("incorrect provider is supplied, %d", provider_id); return NULL; } shad = kalloc_type(struct nstat_tu_shadow, Z_WAITOK | Z_NOFAIL); procdetails = nstat_retain_curprocdetails(); if (procdetails == NULL) { kfree_type(struct nstat_tu_shadow, shad); return NULL; } TAILQ_INIT(&shad->shad_locus.ntl_src_queue); shad->shad_getvals_fn = req_fn; shad->shad_get_extension_fn = req_extension_fn; shad->shad_provider_context = ctx; shad->shad_provider = provider_id; shad->shad_properties = properties; shad->shad_procdetails = procdetails; shad->shad_rnf_override = nstat_rnf_override_not_set; shad->shad_start_timestamp = mach_continuous_time(); shad->shad_live = true; shad->shad_magic = TU_SHADOW_MAGIC; NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tu_shad_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_tu_shad_current, nstat_global_tu_shad_max); NSTAT_LOCK_EXCLUSIVE(); nstat_client *client; // Even if there are no watchers, we save the shadow structure TAILQ_INSERT_HEAD(&nstat_userprot_shad_head, shad, shad_link); if (provider_id == NSTAT_PROVIDER_TCP_USERLAND) { nstat_userland_tcp_shadows++; provider = &nstat_userland_tcp_provider; } else if (provider_id == NSTAT_PROVIDER_UDP_USERLAND) { nstat_userland_udp_shadows++; provider = &nstat_userland_udp_provider; } else { nstat_userland_quic_shadows++; provider = &nstat_userland_quic_provider; } for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_watching & (1 << provider_id)) != 0) { // this client is watching tcp/udp/quic userland // Link to it. int result = nstat_client_source_add(0, client, provider, shad, &shad->shad_locus); if (result != 0) { // There should be some kind of statistics for failures like this. // The kernel ntstat component should keep some // internal counters reflecting operational state for eventual AWD reporting } } } NSTAT_UNLOCK_EXCLUSIVE(); return (nstat_userland_context)shad; } __private_extern__ void ntstat_userland_stats_close(nstat_userland_context nstat_ctx) { struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx; tailq_head_nstat_src dead_list; nstat_src *src; nstat_src *tmpsrc; nstat_client *client; errno_t result; if (shad == NULL) { return; } assert(shad->shad_magic == TU_SHADOW_MAGIC); TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH_SAFE(src, &shad->shad_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(shad == (struct nstat_tu_shadow *)src->nts_cookie); client = src->nts_client; result = nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } if (shad->shad_live) { TAILQ_REMOVE(&nstat_userprot_shad_head, shad, shad_link); if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_userland_tcp_shadows--; } else if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) { nstat_userland_udp_shadows--; } else { nstat_userland_quic_shadows--; } } else { TAILQ_REMOVE(&nstat_defunct_userprot_shad_head, shad, shad_link); } NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } nstat_release_procdetails(shad->shad_procdetails); shad->shad_magic = TU_SHADOW_UNMAGIC; NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tu_shad_current); kfree_type(struct nstat_tu_shadow, shad); } static void ntstat_userland_stats_event_locked( struct nstat_tu_shadow *shad, uint64_t event) { nstat_client *client; nstat_src *src; nstat_provider_id_t provider_id; NSTAT_ASSERT_LOCKED_EXCLUSIVE(); assert(shad->shad_magic == TU_SHADOW_MAGIC); provider_id = shad->shad_provider; TAILQ_FOREACH(src, &shad->shad_locus.ntl_src_queue, nts_locus_link) { assert(shad == (struct nstat_tu_shadow *)src->nts_cookie); client = src->nts_client; if ((client->ntc_provider_filters[provider_id].npf_events & event) != 0) { nstat_client_send_event(client, src, event); } } } __private_extern__ void ntstat_userland_stats_event( nstat_userland_context nstat_ctx, uint64_t event) { // This will need refinement for when we do genuine stats filtering // See NetworkStatistics should provide opt-in notifications // For now it deals only with events that potentially cause any traditional netstat sources to be closed struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx; tailq_head_nstat_src dead_list; nstat_src *src; nstat_src *tmpsrc; nstat_client *client; errno_t result; if (shad == NULL) { return; } assert(shad->shad_magic == TU_SHADOW_MAGIC); if (event & NECP_CLIENT_STATISTICS_EVENT_TIME_WAIT) { TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH_SAFE(src, &shad->shad_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(shad == (struct nstat_tu_shadow *)src->nts_cookie); if (!(src->nts_filter & NSTAT_FILTER_TCP_NO_EARLY_CLOSE)) { assert(src->nts_locus != NULL); client = src->nts_client; result = nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } } NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } } } __private_extern__ void nstats_userland_stats_defunct_for_process(int pid) { // Note that this can be called multiple times for the same process tailq_head_nstat_src dead_list; nstat_src *src, *tmpsrc; struct nstat_tu_shadow *shad, *tmpshad; TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH_SAFE(shad, &nstat_userprot_shad_head, shad_link, tmpshad) { assert(shad->shad_magic == TU_SHADOW_MAGIC); nstat_client *client; errno_t result; assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_live); if (shad->shad_procdetails->pdet_pid == pid) { TAILQ_FOREACH_SAFE(src, &shad->shad_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(shad == (struct nstat_tu_shadow *)src->nts_cookie); client = src->nts_client; result = nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } shad->shad_live = false; if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_userland_tcp_shadows--; } else if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) { nstat_userland_udp_shadows--; } else { nstat_userland_quic_shadows--; } TAILQ_REMOVE(&nstat_userprot_shad_head, shad, shad_link); TAILQ_INSERT_TAIL(&nstat_defunct_userprot_shad_head, shad, shad_link); } } NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } } errno_t nstat_userland_mark_rnf_override(uuid_t target_fuuid, bool rnf_override) { // Note that this can be called multiple times for the same process struct nstat_tu_shadow *shad; uuid_t fuuid; errno_t result; NSTAT_LOCK_EXCLUSIVE(); // We set the fallback state regardles of watchers as there may be future ones that need to know TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { assert(shad->shad_magic == TU_SHADOW_MAGIC); assert(shad->shad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); if (shad->shad_get_extension_fn(shad->shad_provider_context, NSTAT_EXTENDED_UPDATE_TYPE_FUUID, fuuid, sizeof(fuuid))) { if (uuid_compare(fuuid, target_fuuid) == 0) { break; } } } if (shad) { if (shad->shad_procdetails->pdet_pid != proc_selfpid()) { result = EPERM; } else { result = 0; // It would be possible but awkward to check the previous value // for RNF override, and send an event only if changed. // In practice it's fine to send an event regardless, // which "pushes" the last statistics for the previous mode shad->shad_rnf_override = rnf_override ? nstat_rnf_override_enabled : nstat_rnf_override_disabled; ntstat_userland_stats_event_locked(shad, rnf_override ? NSTAT_EVENT_SRC_ENTER_CELLFALLBACK : NSTAT_EVENT_SRC_EXIT_CELLFALLBACK); } } else { result = EEXIST; } NSTAT_UNLOCK_EXCLUSIVE(); return result; } #pragma mark -- Generic Providers -- static nstat_provider nstat_userland_conn_provider; static nstat_provider nstat_udp_subflow_provider; static u_int32_t nstat_generic_provider_watchers[NSTAT_PROVIDER_COUNT]; struct nstat_generic_shadow { nstat_locus gshad_locus; tailq_entry_generic_shadow gshad_link; nstat_provider_context gshad_provider_context; nstat_provider_request_vals_fn *gshad_getvals_fn; nstat_provider_request_extensions_fn *gshad_getextensions_fn; u_int64_t gshad_properties; u_int64_t gshad_start_timestamp; struct nstat_procdetails *gshad_procdetails; nstat_provider_id_t gshad_provider; int32_t gshad_refcnt; uint32_t gshad_magic; }; // Magic number checking should remain in place until the userland provider has been fully proven #define NSTAT_GENERIC_SHADOW_MAGIC 0xfadef00d #define NSTAT_GENERIC_SHADOW_UNMAGIC 0xfadedead static tailq_head_generic_shadow nstat_gshad_head = TAILQ_HEAD_INITIALIZER(nstat_gshad_head); static inline void nstat_retain_gshad( struct nstat_generic_shadow *gshad) { assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); OSIncrementAtomic(&gshad->gshad_refcnt); } static void nstat_release_gshad( struct nstat_generic_shadow *gshad) { assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); if (OSDecrementAtomic(&gshad->gshad_refcnt) == 1) { nstat_release_procdetails(gshad->gshad_procdetails); gshad->gshad_magic = NSTAT_GENERIC_SHADOW_UNMAGIC; NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_gshad_current); kfree_type(struct nstat_generic_shadow, gshad); } } static errno_t nstat_generic_provider_lookup( __unused const void *data, __unused u_int32_t length, __unused nstat_provider_cookie_t *out_cookie) { // Looking up a specific connection is not supported return ENOTSUP; } static int nstat_generic_provider_gone( __unused nstat_provider_cookie_t cookie) { // Returns non-zero if the source has gone. // We don't keep a source hanging around, so the answer is always 0 return 0; } static errno_t nstat_generic_provider_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); memset(out_counts, 0, sizeof(*out_counts)); bool result = (*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, out_counts, NULL, NULL); if (out_gone) { *out_gone = 0; } return (result)? 0 : EIO; } static errno_t nstat_generic_provider_details( nstat_provider_cookie_t cookie, struct nstat_detailed_counts *out_counts, int *out_gone) { struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); memset(out_counts, 0, sizeof(*out_counts)); bool result = (*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, NULL, out_counts, NULL); if (out_gone) { *out_gone = 0; } return (result)? 0 : EIO; } static errno_t nstat_generic_provider_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, __unused size_t len) { struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); struct nstat_procdetails *procdetails = gshad->gshad_procdetails; assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); bool result = (*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, NULL, NULL, data); switch (gshad->gshad_provider) { case NSTAT_PROVIDER_CONN_USERLAND: { nstat_connection_descriptor *desc = (nstat_connection_descriptor *)data; desc->pid = procdetails->pdet_pid; desc->upid = procdetails->pdet_upid; uuid_copy(desc->uuid, procdetails->pdet_uuid); strbufcpy(desc->pname, procdetails->pdet_procname); desc->start_timestamp = gshad->gshad_start_timestamp; desc->timestamp = mach_continuous_time(); break; } case NSTAT_PROVIDER_UDP_SUBFLOW: { nstat_udp_descriptor *desc = (nstat_udp_descriptor *)data; desc->pid = procdetails->pdet_pid; desc->upid = procdetails->pdet_upid; uuid_copy(desc->uuid, procdetails->pdet_uuid); strbufcpy(desc->pname, procdetails->pdet_procname); desc->start_timestamp = gshad->gshad_start_timestamp; desc->timestamp = mach_continuous_time(); break; } default: break; } return (result)? 0 : EIO; } static void nstat_generic_provider_release( __unused nstat_provider_cookie_t cookie) { // Called when a nstat_src is detached. struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; nstat_release_gshad(gshad); } static bool nstat_generic_provider_reporting_allowed( nstat_provider_cookie_t cookie, nstat_provider_filter *filter, u_int64_t suppression_flags) { struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); if ((filter->npf_flags & NSTAT_FILTER_SUPPRESS_BORING_FLAGS) != 0) { if ((filter->npf_flags & suppression_flags) != 0) { return false; } } // Filter based on interface and connection flags // If a provider doesn't support flags, a client shouldn't attempt to use filtering if ((filter->npf_flags & NSTAT_FILTER_IFNET_AND_CONN_FLAGS) != 0) { u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE; if ((*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, &ifflags, NULL, NULL, NULL)) { if ((filter->npf_flags & ifflags) == 0) { return false; } } } if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) { struct nstat_procdetails *procdetails = gshad->gshad_procdetails; assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); // Check details that we have readily to hand before asking the provider for descriptor items if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) && (filter->npf_pid == procdetails->pdet_pid)) { return true; } if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) && (memcmp(filter->npf_uuid, &procdetails->pdet_uuid, sizeof(filter->npf_uuid)) == 0)) { return true; } if ((filter->npf_flags & (NSTAT_FILTER_SPECIFIC_USER_BY_EPID | NSTAT_FILTER_SPECIFIC_USER_BY_EUUID)) != 0) { nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far? switch (gshad->gshad_provider) { case NSTAT_PROVIDER_CONN_USERLAND: // Filtering by effective uuid or effective pid is currently not supported filter->npf_flags &= ~((uint64_t)(NSTAT_FILTER_SPECIFIC_USER_BY_EPID | NSTAT_FILTER_SPECIFIC_USER_BY_EUUID)); NSTAT_LOG("attempt to filter conn provider by effective pid/uuid, not supported"); return true; case NSTAT_PROVIDER_UDP_SUBFLOW: if ((*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, NULL, NULL, &udp_desc)) { if (check_reporting_for_user(filter, procdetails->pdet_pid, (pid_t)udp_desc.epid, &procdetails->pdet_uuid, &udp_desc.euuid)) { return true; } } break; default: break; } } return false; } return true; } static size_t nstat_generic_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *__sized_by(len)buf, size_t len) { struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie; assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); assert(gshad->gshad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC); if (gshad->gshad_getextensions_fn == NULL) { return 0; } return gshad->gshad_getextensions_fn(gshad->gshad_provider_context, extension_id, buf, len); } static errno_t nstat_generic_provider_add_watcher( nstat_client *client, nstat_msg_add_all_srcs *req) { errno_t result; nstat_provider_id_t provider_id = req->provider; nstat_provider *provider; switch (provider_id) { case NSTAT_PROVIDER_CONN_USERLAND: provider = &nstat_userland_conn_provider; break; case NSTAT_PROVIDER_UDP_SUBFLOW: provider = &nstat_udp_subflow_provider; break; default: return ENOTSUP; } NSTAT_LOCK_EXCLUSIVE(); result = nstat_set_provider_filter(client, req); if (result == 0) { struct nstat_generic_shadow *gshad; nstat_provider_filter *filter = &client->ntc_provider_filters[provider_id]; OSIncrementAtomic(&nstat_generic_provider_watchers[provider_id]); TAILQ_FOREACH(gshad, &nstat_gshad_head, gshad_link) { assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); if (gshad->gshad_provider == provider_id) { if (filter->npf_flags & NSTAT_FILTER_INITIAL_PROPERTIES) { u_int64_t npf_flags = filter->npf_flags & NSTAT_FILTER_IFNET_AND_CONN_FLAGS; if ((npf_flags != 0) && ((npf_flags & gshad->gshad_properties) == 0)) { // Skip this one // Note - no filtering by pid or UUID supported at this point, for simplicity continue; } } nstat_retain_gshad(gshad); result = nstat_client_source_add(0, client, provider, gshad, &gshad->gshad_locus); if (result != 0) { NSTAT_LOG_ERROR("nstat_client_source_add returned %d for " "provider type: %d", result, provider_id); nstat_release_gshad(gshad); break; } } } } NSTAT_UNLOCK_EXCLUSIVE(); return result; } static void nstat_userland_conn_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_generic_provider_watchers[NSTAT_PROVIDER_CONN_USERLAND]); } static void nstat_udp_subflow_remove_watcher( __unused nstat_client *client) { OSDecrementAtomic(&nstat_generic_provider_watchers[NSTAT_PROVIDER_UDP_SUBFLOW]); } static void nstat_init_userland_conn_provider(void) { bzero(&nstat_userland_conn_provider, sizeof(nstat_userland_conn_provider)); nstat_userland_conn_provider.nstat_descriptor_length = sizeof(nstat_connection_descriptor); nstat_userland_conn_provider.nstat_provider_id = NSTAT_PROVIDER_CONN_USERLAND; nstat_userland_conn_provider.nstat_lookup = nstat_generic_provider_lookup; nstat_userland_conn_provider.nstat_gone = nstat_generic_provider_gone; nstat_userland_conn_provider.nstat_counts = nstat_generic_provider_counts; nstat_userland_conn_provider.nstat_details = nstat_generic_provider_details; nstat_userland_conn_provider.nstat_release = nstat_generic_provider_release; nstat_userland_conn_provider.nstat_watcher_add = nstat_generic_provider_add_watcher; nstat_userland_conn_provider.nstat_watcher_remove = nstat_userland_conn_remove_watcher; nstat_userland_conn_provider.nstat_copy_descriptor = nstat_generic_provider_copy_descriptor; nstat_userland_conn_provider.nstat_reporting_allowed = nstat_generic_provider_reporting_allowed; nstat_userland_conn_provider.nstat_copy_extension = nstat_generic_extensions; nstat_userland_conn_provider.next = nstat_providers; nstat_providers = &nstat_userland_conn_provider; } static void nstat_init_udp_subflow_provider(void) { bzero(&nstat_udp_subflow_provider, sizeof(nstat_udp_subflow_provider)); nstat_udp_subflow_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); nstat_udp_subflow_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_SUBFLOW; nstat_udp_subflow_provider.nstat_lookup = nstat_generic_provider_lookup; nstat_udp_subflow_provider.nstat_gone = nstat_generic_provider_gone; nstat_udp_subflow_provider.nstat_counts = nstat_generic_provider_counts; nstat_udp_subflow_provider.nstat_details = nstat_generic_provider_details; nstat_udp_subflow_provider.nstat_release = nstat_generic_provider_release; nstat_udp_subflow_provider.nstat_watcher_add = nstat_generic_provider_add_watcher; nstat_udp_subflow_provider.nstat_watcher_remove = nstat_udp_subflow_remove_watcher; nstat_udp_subflow_provider.nstat_copy_descriptor = nstat_generic_provider_copy_descriptor; nstat_udp_subflow_provider.nstat_reporting_allowed = nstat_generic_provider_reporting_allowed; nstat_udp_subflow_provider.nstat_copy_extension = nstat_generic_extensions; nstat_udp_subflow_provider.next = nstat_providers; nstat_providers = &nstat_udp_subflow_provider; } // Things get started with a call from the provider to netstats to say that there’s a new source __private_extern__ nstat_context nstat_provider_stats_open(nstat_provider_context ctx, int provider_id, u_int64_t properties, nstat_provider_request_vals_fn req_fn, nstat_provider_request_extensions_fn req_extensions_fn) { struct nstat_generic_shadow *gshad; struct nstat_procdetails *procdetails; nstat_provider *provider = nstat_find_provider_by_id(provider_id); gshad = kalloc_type(struct nstat_generic_shadow, Z_WAITOK | Z_NOFAIL); procdetails = nstat_retain_curprocdetails(); if (procdetails == NULL) { kfree_type(struct nstat_generic_shadow, gshad); return NULL; } TAILQ_INIT(&gshad->gshad_locus.ntl_src_queue); gshad->gshad_getvals_fn = req_fn; gshad->gshad_getextensions_fn = req_extensions_fn; gshad->gshad_provider_context = ctx; gshad->gshad_properties = properties; gshad->gshad_procdetails = procdetails; gshad->gshad_provider = provider_id; gshad->gshad_start_timestamp = mach_continuous_time(); gshad->gshad_refcnt = 0; gshad->gshad_magic = NSTAT_GENERIC_SHADOW_MAGIC; nstat_retain_gshad(gshad); NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_gshad_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_gshad_current, nstat_global_gshad_max); NSTAT_LOCK_EXCLUSIVE(); nstat_client *client; // Even if there are no watchers, we save the shadow structure TAILQ_INSERT_HEAD(&nstat_gshad_head, gshad, gshad_link); for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_watching & (1 << provider_id)) != 0) { // Does this client want an initial filtering to be made? u_int64_t npf_flags = client->ntc_provider_filters[provider->nstat_provider_id].npf_flags; if (npf_flags & NSTAT_FILTER_INITIAL_PROPERTIES) { npf_flags &= NSTAT_FILTER_IFNET_AND_CONN_FLAGS; if ((npf_flags != 0) && ((npf_flags & properties) == 0)) { // Skip this one // Note - no filtering by pid or UUID supported at this point, for simplicity continue; } } // this client is watching, so link to it. nstat_retain_gshad(gshad); int result = nstat_client_source_add(0, client, provider, gshad, &gshad->gshad_locus); if (result != 0) { // There should be some kind of statistics for failures like this. // The kernel ntstat component should keep some // internal counters reflecting operational state for eventual AWD reporting nstat_release_gshad(gshad); } } } NSTAT_UNLOCK_EXCLUSIVE(); return (nstat_context) gshad; } // When the source is closed, netstats will make one last call on the request functions to retrieve final values __private_extern__ void nstat_provider_stats_close(nstat_context nstat_ctx) { tailq_head_nstat_src dead_list; nstat_src *src; nstat_src *tmpsrc; struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)nstat_ctx; if (gshad == NULL) { NSTAT_LOG_ERROR("called with null reference"); return; } assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); if (gshad->gshad_magic != NSTAT_GENERIC_SHADOW_MAGIC) { NSTAT_LOG_ERROR("called with incorrect shadow magic 0x%x", gshad->gshad_magic); } TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH_SAFE(src, &gshad->gshad_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(gshad == (struct nstat_generic_shadow *)src->nts_cookie); nstat_client *client = src->nts_client; nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } TAILQ_REMOVE(&nstat_gshad_head, gshad, gshad_link); NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } nstat_release_gshad(gshad); } // Events that cause a significant change may be reported via a flags word void nstat_provider_stats_event(__unused nstat_context nstat_ctx, __unused uint64_t event) { nstat_src *src; struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)nstat_ctx; if (gshad == NULL) { NSTAT_LOG_ERROR("called with null reference"); return; } assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC); if (gshad->gshad_magic != NSTAT_GENERIC_SHADOW_MAGIC) { NSTAT_LOG_ERROR("called with incorrect shadow magic 0x%x", gshad->gshad_magic); } NSTAT_LOCK_EXCLUSIVE(); TAILQ_FOREACH(src, &gshad->gshad_locus.ntl_src_queue, nts_locus_link) { assert(gshad == (struct nstat_generic_shadow *)src->nts_cookie); nstat_client *client = src->nts_client; nstat_provider_id_t provider_id = gshad->gshad_provider; // Check if this client is watching and has interest in the event // or the client has requested "boring" unchanged status to be ignored if (((client->ntc_watching & (1 << provider_id)) != 0) && (((client->ntc_provider_filters[provider_id].npf_events & event) != 0) || ((client->ntc_provider_filters[provider_id].npf_flags & NSTAT_FILTER_SUPPRESS_BORING_FLAGS) != 0))) { src->nts_reported = false; if ((client->ntc_provider_filters[provider_id].npf_events & event) != 0) { nstat_client_send_event(client, src, event); // There is currently no recovery possible from failure to send, // so no need to check the return code. // rdar://28312774 (Scalability and resilience issues in ntstat.c) } } } NSTAT_UNLOCK_EXCLUSIVE(); } #endif /* SKYWALK */ #pragma mark -- ifnet Provider -- static nstat_provider nstat_ifnet_provider; /* * We store a pointer to the ifnet and the original threshold * requested by the client. */ struct nstat_ifnet_cookie { struct ifnet *ifp; uint64_t threshold; }; static errno_t nstat_ifnet_lookup( const void *data, u_int32_t length, nstat_provider_cookie_t *out_cookie) { const nstat_ifnet_add_param *param = (const nstat_ifnet_add_param *)data; struct ifnet *ifp; boolean_t changed = FALSE; nstat_client *client; nstat_src *src; struct nstat_ifnet_cookie *cookie; if (length < sizeof(*param) || param->threshold < 1024 * 1024) { return EINVAL; } if (nstat_privcheck != 0) { errno_t result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0); if (result != 0) { return result; } } cookie = kalloc_type(struct nstat_ifnet_cookie, Z_WAITOK | Z_ZERO | Z_NOFAIL); ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { if (!ifnet_get_ioref(ifp)) { continue; } ifnet_lock_exclusive(ifp); if (ifp->if_index == param->ifindex) { cookie->ifp = ifp; cookie->threshold = param->threshold; *out_cookie = cookie; if (!ifp->if_data_threshold || ifp->if_data_threshold > param->threshold) { changed = TRUE; ifp->if_data_threshold = param->threshold; } ifnet_lock_done(ifp); ifnet_reference(ifp); ifnet_decr_iorefcnt(ifp); break; } ifnet_lock_done(ifp); ifnet_decr_iorefcnt(ifp); } ifnet_head_done(); /* * When we change the threshold to something smaller, we notify * all of our clients with a description message. * We won't send a message to the client we are currently serving * because it has no `ifnet source' yet. */ if (changed) { NSTAT_LOCK_EXCLUSIVE(); for (client = nstat_clients; client; client = client->ntc_next) { TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { if (src->nts_provider != &nstat_ifnet_provider) { continue; } nstat_client_send_description(client, src, 0, 0); } } NSTAT_UNLOCK_EXCLUSIVE(); } if (cookie->ifp == NULL) { kfree_type(struct nstat_ifnet_cookie, cookie); } return ifp ? 0 : EINVAL; } static int nstat_ifnet_gone( nstat_provider_cookie_t cookie) { struct ifnet *ifp; struct nstat_ifnet_cookie *ifcookie = (struct nstat_ifnet_cookie *)cookie; ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { if (ifp == ifcookie->ifp) { break; } } ifnet_head_done(); return ifp ? 0 : 1; } static errno_t nstat_ifnet_counts( nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone) { struct nstat_ifnet_cookie *ifcookie = (struct nstat_ifnet_cookie *)cookie; struct ifnet *ifp = ifcookie->ifp; if (out_gone) { *out_gone = 0; } // if the ifnet is gone, we should stop using it if (nstat_ifnet_gone(cookie)) { if (out_gone) { *out_gone = 1; } return EINVAL; } bzero(out_counts, sizeof(*out_counts)); out_counts->nstat_rxpackets = ifp->if_ipackets; out_counts->nstat_rxbytes = ifp->if_ibytes; out_counts->nstat_txpackets = ifp->if_opackets; out_counts->nstat_txbytes = ifp->if_obytes; out_counts->nstat_cell_rxbytes = out_counts->nstat_cell_txbytes = 0; return 0; } static void nstat_ifnet_release( nstat_provider_cookie_t cookie) { struct nstat_ifnet_cookie *ifcookie; struct ifnet *ifp; nstat_client *client; nstat_src *src; uint64_t minthreshold = UINT64_MAX; /* * Find all the clients that requested a threshold * for this ifnet and re-calculate if_data_threshold. */ NSTAT_LOCK_SHARED(); for (client = nstat_clients; client; client = client->ntc_next) { TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { /* Skip the provider we are about to detach. */ if (src->nts_provider != &nstat_ifnet_provider || src->nts_cookie == cookie) { continue; } ifcookie = (struct nstat_ifnet_cookie *)src->nts_cookie; if (ifcookie->threshold < minthreshold) { minthreshold = ifcookie->threshold; } } } NSTAT_UNLOCK_SHARED(); /* * Reset if_data_threshold or disable it. */ ifcookie = (struct nstat_ifnet_cookie *)cookie; ifp = ifcookie->ifp; if (ifnet_get_ioref(ifp)) { ifnet_lock_exclusive(ifp); if (minthreshold == UINT64_MAX) { ifp->if_data_threshold = 0; } else { ifp->if_data_threshold = minthreshold; } ifnet_lock_done(ifp); ifnet_decr_iorefcnt(ifp); } ifnet_release(ifp); kfree_type(struct nstat_ifnet_cookie, ifcookie); } static void nstat_ifnet_copy_link_status( struct ifnet *ifp, struct nstat_ifnet_descriptor *desc) { struct if_link_status *ifsr = ifp->if_link_status; nstat_ifnet_desc_link_status *link_status = &desc->link_status; link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_NONE; if (ifsr == NULL) { return; } lck_rw_lock_shared(&ifp->if_link_status_lock); if (ifp->if_type == IFT_CELLULAR) { nstat_ifnet_desc_cellular_status *cell_status = &link_status->u.cellular; struct if_cellular_status_v1 *if_cell_sr = &ifsr->ifsr_u.ifsr_cell.if_cell_u.if_status_v1; if (ifsr->ifsr_version != IF_CELLULAR_STATUS_REPORT_VERSION_1) { goto done; } link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR; if (if_cell_sr->valid_bitmask & IF_CELL_LINK_QUALITY_METRIC_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_LINK_QUALITY_METRIC_VALID; cell_status->link_quality_metric = if_cell_sr->link_quality_metric; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_EFFECTIVE_BANDWIDTH_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_EFFECTIVE_BANDWIDTH_VALID; cell_status->ul_effective_bandwidth = if_cell_sr->ul_effective_bandwidth; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_BANDWIDTH_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_BANDWIDTH_VALID; cell_status->ul_max_bandwidth = if_cell_sr->ul_max_bandwidth; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MIN_LATENCY_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MIN_LATENCY_VALID; cell_status->ul_min_latency = if_cell_sr->ul_min_latency; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_EFFECTIVE_LATENCY_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_EFFECTIVE_LATENCY_VALID; cell_status->ul_effective_latency = if_cell_sr->ul_effective_latency; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_LATENCY_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_LATENCY_VALID; cell_status->ul_max_latency = if_cell_sr->ul_max_latency; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_RETXT_LEVEL_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_VALID; if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_NONE) { cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_NONE; } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_LOW) { cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_LOW; } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_MEDIUM) { cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_MEDIUM; } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_HIGH) { cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_HIGH; } else { cell_status->valid_bitmask &= ~NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_VALID; } } if (if_cell_sr->valid_bitmask & IF_CELL_UL_BYTES_LOST_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_BYTES_LOST_VALID; cell_status->ul_bytes_lost = if_cell_sr->ul_bytes_lost; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MIN_QUEUE_SIZE_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MIN_QUEUE_SIZE_VALID; cell_status->ul_min_queue_size = if_cell_sr->ul_min_queue_size; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_AVG_QUEUE_SIZE_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_AVG_QUEUE_SIZE_VALID; cell_status->ul_avg_queue_size = if_cell_sr->ul_avg_queue_size; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_QUEUE_SIZE_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_QUEUE_SIZE_VALID; cell_status->ul_max_queue_size = if_cell_sr->ul_max_queue_size; } if (if_cell_sr->valid_bitmask & IF_CELL_DL_EFFECTIVE_BANDWIDTH_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_DL_EFFECTIVE_BANDWIDTH_VALID; cell_status->dl_effective_bandwidth = if_cell_sr->dl_effective_bandwidth; } if (if_cell_sr->valid_bitmask & IF_CELL_DL_MAX_BANDWIDTH_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_DL_MAX_BANDWIDTH_VALID; cell_status->dl_max_bandwidth = if_cell_sr->dl_max_bandwidth; } if (if_cell_sr->valid_bitmask & IF_CELL_CONFIG_INACTIVITY_TIME_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_CONFIG_INACTIVITY_TIME_VALID; cell_status->config_inactivity_time = if_cell_sr->config_inactivity_time; } if (if_cell_sr->valid_bitmask & IF_CELL_CONFIG_BACKOFF_TIME_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_CONFIG_BACKOFF_TIME_VALID; cell_status->config_backoff_time = if_cell_sr->config_backoff_time; } if (if_cell_sr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) { cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_MSS_RECOMMENDED_VALID; cell_status->mss_recommended = if_cell_sr->mss_recommended; } } else if (IFNET_IS_WIFI(ifp)) { nstat_ifnet_desc_wifi_status *wifi_status = &link_status->u.wifi; struct if_wifi_status_v1 *if_wifi_sr = &ifsr->ifsr_u.ifsr_wifi.if_wifi_u.if_status_v1; if (ifsr->ifsr_version != IF_WIFI_STATUS_REPORT_VERSION_1) { goto done; } link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI; if (if_wifi_sr->valid_bitmask & IF_WIFI_LINK_QUALITY_METRIC_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_LINK_QUALITY_METRIC_VALID; wifi_status->link_quality_metric = if_wifi_sr->link_quality_metric; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_EFFECTIVE_BANDWIDTH_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_EFFECTIVE_BANDWIDTH_VALID; wifi_status->ul_effective_bandwidth = if_wifi_sr->ul_effective_bandwidth; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MAX_BANDWIDTH_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MAX_BANDWIDTH_VALID; wifi_status->ul_max_bandwidth = if_wifi_sr->ul_max_bandwidth; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MIN_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MIN_LATENCY_VALID; wifi_status->ul_min_latency = if_wifi_sr->ul_min_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_EFFECTIVE_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_EFFECTIVE_LATENCY_VALID; wifi_status->ul_effective_latency = if_wifi_sr->ul_effective_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MAX_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MAX_LATENCY_VALID; wifi_status->ul_max_latency = if_wifi_sr->ul_max_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_RETXT_LEVEL_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_VALID; if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_NONE) { wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_NONE; } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_LOW) { wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_LOW; } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_MEDIUM) { wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_MEDIUM; } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_HIGH) { wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_HIGH; } else { wifi_status->valid_bitmask &= ~NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_VALID; } } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_BYTES_LOST_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_BYTES_LOST_VALID; wifi_status->ul_bytes_lost = if_wifi_sr->ul_bytes_lost; } if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_ERROR_RATE_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_ERROR_RATE_VALID; wifi_status->ul_error_rate = if_wifi_sr->ul_error_rate; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_EFFECTIVE_BANDWIDTH_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_EFFECTIVE_BANDWIDTH_VALID; wifi_status->dl_effective_bandwidth = if_wifi_sr->dl_effective_bandwidth; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MAX_BANDWIDTH_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MAX_BANDWIDTH_VALID; wifi_status->dl_max_bandwidth = if_wifi_sr->dl_max_bandwidth; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MIN_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MIN_LATENCY_VALID; wifi_status->dl_min_latency = if_wifi_sr->dl_min_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_EFFECTIVE_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_EFFECTIVE_LATENCY_VALID; wifi_status->dl_effective_latency = if_wifi_sr->dl_effective_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MAX_LATENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MAX_LATENCY_VALID; wifi_status->dl_max_latency = if_wifi_sr->dl_max_latency; } if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_ERROR_RATE_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_ERROR_RATE_VALID; wifi_status->dl_error_rate = if_wifi_sr->dl_error_rate; } if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_FREQUENCY_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_VALID; if (if_wifi_sr->config_frequency == IF_WIFI_CONFIG_FREQUENCY_2_4_GHZ) { wifi_status->config_frequency = NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_2_4_GHZ; } else if (if_wifi_sr->config_frequency == IF_WIFI_CONFIG_FREQUENCY_5_0_GHZ) { wifi_status->config_frequency = NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_5_0_GHZ; } else { wifi_status->valid_bitmask &= ~NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_VALID; } } if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_MULTICAST_RATE_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_MULTICAST_RATE_VALID; wifi_status->config_multicast_rate = if_wifi_sr->config_multicast_rate; } if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_SCAN_COUNT_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_SCAN_COUNT_VALID; wifi_status->scan_count = if_wifi_sr->scan_count; } if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_SCAN_DURATION_VALID) { wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_SCAN_DURATION_VALID; wifi_status->scan_duration = if_wifi_sr->scan_duration; } } done: lck_rw_done(&ifp->if_link_status_lock); } /* Some thresholds to determine Low Internet mode */ #define NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD 1000000 /* 1 Mbps */ #define NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD 500000 /* 500 Kbps */ #define NSTAT_LIM_UL_MIN_RTT_THRESHOLD 1000 /* 1 second */ #define NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD (10 << 10) /* 10 percent connection timeouts */ #define NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD (2 << 10) /* 2 percent packet loss rate */ static boolean_t nstat_lim_activity_check(struct if_lim_perf_stat *st) { /* check that the current activity is enough to report stats */ if (st->lim_total_txpkts < nstat_lim_min_tx_pkts || st->lim_total_rxpkts < nstat_lim_min_rx_pkts || st->lim_conn_attempts == 0) { return FALSE; } /* * Compute percentages if there was enough activity. Use * shift-left by 10 to preserve precision. */ st->lim_packet_loss_percent = ((st->lim_total_retxpkts << 10) / st->lim_total_txpkts) * 100; st->lim_packet_ooo_percent = ((st->lim_total_oopkts << 10) / st->lim_total_rxpkts) * 100; st->lim_conn_timeout_percent = ((st->lim_conn_timeouts << 10) / st->lim_conn_attempts) * 100; /* * Is Low Internet detected? First order metrics are bandwidth * and RTT. If these metrics are below the minimum thresholds * defined then the network attachment can be classified as * having Low Internet capacity. * * High connection timeout rate also indicates Low Internet * capacity. */ if (st->lim_dl_max_bandwidth > 0 && st->lim_dl_max_bandwidth <= NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD) { st->lim_dl_detected = 1; } if ((st->lim_ul_max_bandwidth > 0 && st->lim_ul_max_bandwidth <= NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD) || st->lim_rtt_min >= NSTAT_LIM_UL_MIN_RTT_THRESHOLD) { st->lim_ul_detected = 1; } if (st->lim_conn_attempts > 20 && st->lim_conn_timeout_percent >= NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD) { st->lim_ul_detected = 1; } /* * Second order metrics: If there was high packet loss even after * using delay based algorithms then we classify it as Low Internet * again */ if (st->lim_bk_txpkts >= nstat_lim_min_tx_pkts && st->lim_packet_loss_percent >= NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD) { st->lim_ul_detected = 1; } return TRUE; } static u_int64_t nstat_lim_last_report_time = 0; static void nstat_ifnet_report_lim_stats(void) { u_int64_t uptime; struct nstat_sysinfo_data data; struct nstat_sysinfo_lim_stats *st; struct ifnet *ifp; int err; uptime = net_uptime(); if ((u_int32_t)(uptime - nstat_lim_last_report_time) < nstat_lim_interval) { return; } nstat_lim_last_report_time = uptime; data.flags = NSTAT_SYSINFO_LIM_STATS; st = &data.u.lim_stats; data.unsent_data_cnt = 0; ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { if (!ifnet_is_fully_attached(ifp)) { continue; } /* Limit reporting to Wifi, Ethernet and cellular */ if (!(IFNET_IS_ETHERNET(ifp) || IFNET_IS_CELLULAR(ifp))) { continue; } if (!nstat_lim_activity_check(&ifp->if_lim_stat)) { continue; } bzero(st, sizeof(*st)); st->ifnet_siglen = sizeof(st->ifnet_signature); err = ifnet_get_netsignature(ifp, AF_INET, (u_int8_t *)&st->ifnet_siglen, NULL, st->ifnet_signature); if (err != 0) { err = ifnet_get_netsignature(ifp, AF_INET6, (u_int8_t *)&st->ifnet_siglen, NULL, st->ifnet_signature); if (err != 0) { continue; } } ifnet_lock_shared(ifp); if (IFNET_IS_CELLULAR(ifp)) { st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR; } else if (IFNET_IS_WIFI(ifp)) { st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI; } else { st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET; } bcopy(&ifp->if_lim_stat, &st->lim_stat, sizeof(st->lim_stat)); /* Zero the stats in ifp */ bzero(&ifp->if_lim_stat, sizeof(ifp->if_lim_stat)); if (nstat_debug != 0) { NSTAT_LOG_DEBUG("Reporting LIM stats for ifindex %u", ifp->if_index); } ifnet_lock_done(ifp); nstat_sysinfo_send_data(&data); } ifnet_head_done(); } static errno_t nstat_ifnet_copy_descriptor( nstat_provider_cookie_t cookie, void *__sized_by(len)data, size_t len) { nstat_ifnet_descriptor *desc = (nstat_ifnet_descriptor *)data; struct nstat_ifnet_cookie *ifcookie = (struct nstat_ifnet_cookie *)cookie; struct ifnet *ifp = ifcookie->ifp; if (len < sizeof(nstat_ifnet_descriptor)) { return EINVAL; } if (nstat_ifnet_gone(cookie)) { return EINVAL; } bzero(desc, sizeof(*desc)); ifnet_lock_shared(ifp); strlcpy(desc->name, ifp->if_xname, sizeof(desc->name)); desc->ifindex = ifp->if_index; desc->threshold = ifp->if_data_threshold; desc->type = ifp->if_type; if (ifp->if_desc.ifd_len < sizeof(desc->description)) { memcpy(desc->description, ifp->if_desc.ifd_desc, sizeof(desc->description)); } nstat_ifnet_copy_link_status(ifp, desc); ifnet_lock_done(ifp); return 0; } static bool nstat_ifnet_cookie_equal( nstat_provider_cookie_t cookie1, nstat_provider_cookie_t cookie2) { struct nstat_ifnet_cookie *c1 = (struct nstat_ifnet_cookie *)cookie1; struct nstat_ifnet_cookie *c2 = (struct nstat_ifnet_cookie *)cookie2; return (c1->ifp->if_index == c2->ifp->if_index) ? true : false; } static void nstat_init_ifnet_provider(void) { bzero(&nstat_ifnet_provider, sizeof(nstat_ifnet_provider)); nstat_ifnet_provider.nstat_provider_id = NSTAT_PROVIDER_IFNET; nstat_ifnet_provider.nstat_descriptor_length = sizeof(nstat_ifnet_descriptor); nstat_ifnet_provider.nstat_lookup = nstat_ifnet_lookup; nstat_ifnet_provider.nstat_gone = nstat_ifnet_gone; nstat_ifnet_provider.nstat_counts = nstat_ifnet_counts; nstat_ifnet_provider.nstat_watcher_add = NULL; nstat_ifnet_provider.nstat_watcher_remove = NULL; nstat_ifnet_provider.nstat_copy_descriptor = nstat_ifnet_copy_descriptor; nstat_ifnet_provider.nstat_cookie_equal = nstat_ifnet_cookie_equal; nstat_ifnet_provider.nstat_release = nstat_ifnet_release; nstat_ifnet_provider.next = nstat_providers; nstat_providers = &nstat_ifnet_provider; } __private_extern__ void nstat_ifnet_threshold_reached(unsigned int ifindex) { nstat_client *client; nstat_src *src; struct ifnet *ifp; struct nstat_ifnet_cookie *ifcookie; NSTAT_LOCK_EXCLUSIVE(); for (client = nstat_clients; client; client = client->ntc_next) { TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { if (src->nts_provider != &nstat_ifnet_provider) { continue; } ifcookie = (struct nstat_ifnet_cookie *)src->nts_cookie; ifp = ifcookie->ifp; if (ifp->if_index != ifindex) { continue; } nstat_client_send_counts(client, src, 0, 0, NULL); } } NSTAT_UNLOCK_EXCLUSIVE(); } #pragma mark -- Sysinfo -- static void nstat_set_keyval_scalar(nstat_sysinfo_keyval *kv, int key, u_int32_t val) { kv->nstat_sysinfo_key = key; kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR; kv->u.nstat_sysinfo_scalar = val; kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar); } static void nstat_set_keyval_u64_scalar(nstat_sysinfo_keyval *kv, int key, u_int64_t val) { kv->nstat_sysinfo_key = key; kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR; kv->u.nstat_sysinfo_scalar = val; kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar); } static void nstat_set_keyval_string(nstat_sysinfo_keyval *kv, int key, u_int8_t *__counted_by(len)buf, u_int32_t len) { kv->nstat_sysinfo_key = key; kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_STRING; kv->nstat_sysinfo_valsize = len; bcopy(buf, kv->u.nstat_sysinfo_string, kv->nstat_sysinfo_valsize); } struct ntstat_sysinfo_keyval_iter { uint32_t i, nkeymax; nstat_sysinfo_keyval *__counted_by(nkeymax) kv; }; static inline nstat_sysinfo_keyval * ntstat_sysinfo_next(struct ntstat_sysinfo_keyval_iter *iter) { size_t index = iter->i; if (index < iter->nkeymax) { iter->i++; return &iter->kv[index]; } return NULL; } static void nstat_sysinfo_send_data_internal( nstat_client *client, nstat_sysinfo_data *data) { nstat_msg_sysinfo_counts *syscnt = NULL; size_t allocsize = 0, countsize = 0, finalsize = 0; uint32_t nkeyvals = 0; struct ntstat_sysinfo_keyval_iter iter = {}; errno_t result = 0; allocsize = offsetof(nstat_msg_sysinfo_counts, counts); countsize = offsetof(nstat_sysinfo_counts, nstat_sysinfo_keyvals); finalsize = allocsize; /* get number of key-vals for each kind of stat */ switch (data->flags) { case NSTAT_SYSINFO_TCP_STATS: nkeyvals = NSTAT_SYSINFO_TCP_STATS_COUNT; break; case NSTAT_SYSINFO_LIM_STATS: nkeyvals = NSTAT_LIM_STAT_KEYVAL_COUNT; break; case NSTAT_SYSINFO_NET_API_STATS: nkeyvals = NSTAT_NET_API_STAT_KEYVAL_COUNT; break; default: return; } countsize += sizeof(nstat_sysinfo_keyval) * nkeyvals; allocsize += countsize; syscnt = (nstat_msg_sysinfo_counts *) kalloc_data(allocsize, Z_WAITOK | Z_ZERO); if (syscnt == NULL) { return; } iter.i = 0; iter.kv = nstat_sysinfo_get_keyvals(syscnt); iter.nkeymax = nkeyvals; switch (data->flags) { case NSTAT_SYSINFO_TCP_STATS: { nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_IPV4_AVGRTT, data->u.tcp_stats.ipv4_avgrtt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_IPV6_AVGRTT, data->u.tcp_stats.ipv6_avgrtt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_SEND_PLR, data->u.tcp_stats.send_plr); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_RECV_PLR, data->u.tcp_stats.recv_plr); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_SEND_TLRTO, data->u.tcp_stats.send_tlrto_rate); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_KEY_SEND_REORDERRATE, data->u.tcp_stats.send_reorder_rate); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_CONNECTION_ATTEMPTS, data->u.tcp_stats.connection_attempts); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_CONNECTION_ACCEPTS, data->u.tcp_stats.connection_accepts); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CLIENT_ENABLED, data->u.tcp_stats.ecn_client_enabled); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_SERVER_ENABLED, data->u.tcp_stats.ecn_server_enabled); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CLIENT_SETUP, data->u.tcp_stats.ecn_client_setup); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_SERVER_SETUP, data->u.tcp_stats.ecn_server_setup); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CLIENT_SUCCESS, data->u.tcp_stats.ecn_client_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_SERVER_SUCCESS, data->u.tcp_stats.ecn_server_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_NOT_SUPPORTED, data->u.tcp_stats.ecn_not_supported); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_LOST_SYN, data->u.tcp_stats.ecn_lost_syn); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_LOST_SYNACK, data->u.tcp_stats.ecn_lost_synack); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_RECV_CE, data->u.tcp_stats.ecn_recv_ce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_RECV_ECE, data->u.tcp_stats.ecn_recv_ece); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_SENT_ECE, data->u.tcp_stats.ecn_sent_ece); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CONN_RECV_CE, data->u.tcp_stats.ecn_conn_recv_ce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CONN_RECV_ECE, data->u.tcp_stats.ecn_conn_recv_ece); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CONN_PLNOCE, data->u.tcp_stats.ecn_conn_plnoce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CONN_PL_CE, data->u.tcp_stats.ecn_conn_pl_ce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_CONN_NOPL_CE, data->u.tcp_stats.ecn_conn_nopl_ce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_FALLBACK_SYNLOSS, data->u.tcp_stats.ecn_fallback_synloss); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_FALLBACK_REORDER, data->u.tcp_stats.ecn_fallback_reorder); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_ECN_FALLBACK_CE, data->u.tcp_stats.ecn_fallback_ce); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_SYN_DATA_RCV, data->u.tcp_stats.tfo_syn_data_rcv); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_REQ_RCV, data->u.tcp_stats.tfo_cookie_req_rcv); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_SENT, data->u.tcp_stats.tfo_cookie_sent); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_INVALID, data->u.tcp_stats.tfo_cookie_invalid); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_REQ, data->u.tcp_stats.tfo_cookie_req); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_RCV, data->u.tcp_stats.tfo_cookie_rcv); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_SYN_DATA_SENT, data->u.tcp_stats.tfo_syn_data_sent); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_SYN_DATA_ACKED, data->u.tcp_stats.tfo_syn_data_acked); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_SYN_LOSS, data->u.tcp_stats.tfo_syn_loss); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_BLACKHOLE, data->u.tcp_stats.tfo_blackhole); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_COOKIE_WRONG, data->u.tcp_stats.tfo_cookie_wrong); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_NO_COOKIE_RCV, data->u.tcp_stats.tfo_no_cookie_rcv); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_HEURISTICS_DISABLE, data->u.tcp_stats.tfo_heuristics_disable); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_TFO_SEND_BLACKHOLE, data->u.tcp_stats.tfo_sndblackhole); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT, data->u.tcp_stats.mptcp_handover_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT, data->u.tcp_stats.mptcp_interactive_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT, data->u.tcp_stats.mptcp_aggregate_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT, data->u.tcp_stats.mptcp_fp_handover_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT, data->u.tcp_stats.mptcp_fp_interactive_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT, data->u.tcp_stats.mptcp_fp_aggregate_attempt); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK, data->u.tcp_stats.mptcp_heuristic_fallback); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK, data->u.tcp_stats.mptcp_fp_heuristic_fallback); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI, data->u.tcp_stats.mptcp_handover_success_wifi); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL, data->u.tcp_stats.mptcp_handover_success_cell); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS, data->u.tcp_stats.mptcp_interactive_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS, data->u.tcp_stats.mptcp_aggregate_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI, data->u.tcp_stats.mptcp_fp_handover_success_wifi); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL, data->u.tcp_stats.mptcp_fp_handover_success_cell); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS, data->u.tcp_stats.mptcp_fp_interactive_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS, data->u.tcp_stats.mptcp_fp_aggregate_success); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI, data->u.tcp_stats.mptcp_handover_cell_from_wifi); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL, data->u.tcp_stats.mptcp_handover_wifi_from_cell); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI, data->u.tcp_stats.mptcp_interactive_cell_from_wifi); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES, data->u.tcp_stats.mptcp_handover_cell_bytes); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES, data->u.tcp_stats.mptcp_interactive_cell_bytes); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES, data->u.tcp_stats.mptcp_aggregate_cell_bytes); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES, data->u.tcp_stats.mptcp_handover_all_bytes); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES, data->u.tcp_stats.mptcp_interactive_all_bytes); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES, data->u.tcp_stats.mptcp_aggregate_all_bytes); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI, data->u.tcp_stats.mptcp_back_to_wifi); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_WIFI_PROXY, data->u.tcp_stats.mptcp_wifi_proxy); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_CELL_PROXY, data->u.tcp_stats.mptcp_cell_proxy); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_MPTCP_TRIGGERED_CELL, data->u.tcp_stats.mptcp_triggered_cell); VERIFY(iter.i == nkeyvals); break; } case NSTAT_SYSINFO_LIM_STATS: { nstat_set_keyval_string(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_SIGNATURE, data->u.lim_stats.ifnet_signature, min(data->u.lim_stats.ifnet_siglen, NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE)); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH, data->u.lim_stats.lim_stat.lim_dl_max_bandwidth); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH, data->u.lim_stats.lim_stat.lim_ul_max_bandwidth); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT, data->u.lim_stats.lim_stat.lim_packet_loss_percent); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT, data->u.lim_stats.lim_stat.lim_packet_ooo_percent); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE, data->u.lim_stats.lim_stat.lim_rtt_variance); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_RTT_MIN, data->u.lim_stats.lim_stat.lim_rtt_min); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_RTT_AVG, data->u.lim_stats.lim_stat.lim_rtt_average); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT, data->u.lim_stats.lim_stat.lim_conn_timeout_percent); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED, data->u.lim_stats.lim_stat.lim_dl_detected); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED, data->u.lim_stats.lim_stat.lim_ul_detected); nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_LIM_IFNET_TYPE, data->u.lim_stats.ifnet_type); break; } case NSTAT_SYSINFO_NET_API_STATS: { nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IF_FLTR_ATTACH, data->u.net_api_stats.net_api_stats.nas_iflt_attach_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS, data->u.net_api_stats.net_api_stats.nas_iflt_attach_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IP_FLTR_ADD, data->u.net_api_stats.net_api_stats.nas_ipf_add_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IP_FLTR_ADD_OS, data->u.net_api_stats.net_api_stats.nas_ipf_add_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH, data->u.net_api_stats.net_api_stats.nas_sfltr_register_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS, data->u.net_api_stats.net_api_stats.nas_sfltr_register_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL, data->u.net_api_stats.net_api_stats.nas_socket_alloc_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL, data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS, data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID, data->u.net_api_stats.net_api_stats.nas_socket_necp_clientuuid_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL, data->u.net_api_stats.net_api_stats.nas_socket_domain_local_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE, data->u.net_api_stats.net_api_stats.nas_socket_domain_route_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_INET, data->u.net_api_stats.net_api_stats.nas_socket_domain_inet_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6, data->u.net_api_stats.net_api_stats.nas_socket_domain_inet6_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM, data->u.net_api_stats.net_api_stats.nas_socket_domain_system_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH, data->u.net_api_stats.net_api_stats.nas_socket_domain_multipath_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY, data->u.net_api_stats.net_api_stats.nas_socket_domain_key_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV, data->u.net_api_stats.net_api_stats.nas_socket_domain_ndrv_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER, data->u.net_api_stats.net_api_stats.nas_socket_domain_other_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_STREAM, data->u.net_api_stats.net_api_stats.nas_socket_inet_stream_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_DGRAM, data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED, data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_connected); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS, data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_dns); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA, data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_no_data); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET6_STREAM, data->u.net_api_stats.net_api_stats.nas_socket_inet6_stream_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET6_DGRAM, data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED, data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_connected); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS, data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_dns); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA, data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_no_data); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN, data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS, data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM, data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_stream_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM, data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_dgram_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM, data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_stream_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM, data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_dgram_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IFNET_ALLOC, data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IFNET_ALLOC_OS, data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_os_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_PF_ADDRULE, data->u.net_api_stats.net_api_stats.nas_pf_addrule_total); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_PF_ADDRULE_OS, data->u.net_api_stats.net_api_stats.nas_pf_addrule_os); nstat_set_keyval_u64_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_VMNET_START, data->u.net_api_stats.net_api_stats.nas_vmnet_total); #if SKYWALK nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_IF_NETAGENT_ENABLED, if_is_fsw_transport_netagent_enabled()); #endif /* SKYWALK */ nstat_set_keyval_scalar(ntstat_sysinfo_next(&iter), NSTAT_SYSINFO_API_REPORT_INTERVAL, data->u.net_api_stats.report_interval); break; } } if (syscnt != NULL) { VERIFY(iter.i > 0); countsize = offsetof(nstat_sysinfo_counts, nstat_sysinfo_keyvals[iter.i]); finalsize += countsize; syscnt->hdr.type = NSTAT_MSG_TYPE_SYSINFO_COUNTS; assert(finalsize <= MAX_NSTAT_MSG_HDR_LENGTH); syscnt->hdr.length = (u_int16_t)finalsize; syscnt->counts.nstat_sysinfo_len = (u_int32_t)countsize; result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, syscnt, finalsize, CTL_DATA_EOR); if (result != 0) { nstat_stats.nstat_sysinfofailures += 1; } kfree_data(syscnt, allocsize); } return; } __private_extern__ void nstat_sysinfo_send_data( nstat_sysinfo_data *data) { if (nstat_debug != 0) { NSTAT_LOG_DEBUG("Sending stats report for %x", data->flags); } nstat_client *client; NSTAT_LOCK_EXCLUSIVE(); for (client = nstat_clients; client; client = client->ntc_next) { if ((client->ntc_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) { nstat_sysinfo_send_data_internal(client, data); } } NSTAT_UNLOCK_EXCLUSIVE(); } static void nstat_sysinfo_generate_report(void) { tcp_report_stats(); nstat_ifnet_report_lim_stats(); nstat_net_api_report_stats(); } #pragma mark -- net_api -- static struct net_api_stats net_api_stats_before; static u_int64_t net_api_stats_last_report_time; static void nstat_net_api_report_stats(void) { struct nstat_sysinfo_data data; struct nstat_sysinfo_net_api_stats *st = &data.u.net_api_stats; u_int64_t uptime; uptime = net_uptime(); if ((u_int32_t)(uptime - net_api_stats_last_report_time) < net_api_stats_report_interval) { return; } st->report_interval = (u_int32_t)(uptime - net_api_stats_last_report_time); net_api_stats_last_report_time = uptime; data.flags = NSTAT_SYSINFO_NET_API_STATS; data.unsent_data_cnt = 0; /* * Some of the fields in the report are the current value and * other fields are the delta from the last report: * - Report difference for the per flow counters as they increase * with time * - Report current value for other counters as they tend not to change * much with time */ #define STATCOPY(f) \ (st->net_api_stats.f = net_api_stats.f) #define STATDIFF(f) \ (st->net_api_stats.f = net_api_stats.f - net_api_stats_before.f) STATCOPY(nas_iflt_attach_count); STATCOPY(nas_iflt_attach_total); STATCOPY(nas_iflt_attach_os_total); STATCOPY(nas_ipf_add_count); STATCOPY(nas_ipf_add_total); STATCOPY(nas_ipf_add_os_total); STATCOPY(nas_sfltr_register_count); STATCOPY(nas_sfltr_register_total); STATCOPY(nas_sfltr_register_os_total); STATDIFF(nas_socket_alloc_total); STATDIFF(nas_socket_in_kernel_total); STATDIFF(nas_socket_in_kernel_os_total); STATDIFF(nas_socket_necp_clientuuid_total); STATDIFF(nas_socket_domain_local_total); STATDIFF(nas_socket_domain_route_total); STATDIFF(nas_socket_domain_inet_total); STATDIFF(nas_socket_domain_inet6_total); STATDIFF(nas_socket_domain_system_total); STATDIFF(nas_socket_domain_multipath_total); STATDIFF(nas_socket_domain_key_total); STATDIFF(nas_socket_domain_ndrv_total); STATDIFF(nas_socket_domain_other_total); STATDIFF(nas_socket_inet_stream_total); STATDIFF(nas_socket_inet_dgram_total); STATDIFF(nas_socket_inet_dgram_connected); STATDIFF(nas_socket_inet_dgram_dns); STATDIFF(nas_socket_inet_dgram_no_data); STATDIFF(nas_socket_inet6_stream_total); STATDIFF(nas_socket_inet6_dgram_total); STATDIFF(nas_socket_inet6_dgram_connected); STATDIFF(nas_socket_inet6_dgram_dns); STATDIFF(nas_socket_inet6_dgram_no_data); STATDIFF(nas_socket_mcast_join_total); STATDIFF(nas_socket_mcast_join_os_total); STATDIFF(nas_sock_inet6_stream_exthdr_in); STATDIFF(nas_sock_inet6_stream_exthdr_out); STATDIFF(nas_sock_inet6_dgram_exthdr_in); STATDIFF(nas_sock_inet6_dgram_exthdr_out); STATDIFF(nas_nx_flow_inet_stream_total); STATDIFF(nas_nx_flow_inet_dgram_total); STATDIFF(nas_nx_flow_inet6_stream_total); STATDIFF(nas_nx_flow_inet6_dgram_total); STATCOPY(nas_ifnet_alloc_count); STATCOPY(nas_ifnet_alloc_total); STATCOPY(nas_ifnet_alloc_os_count); STATCOPY(nas_ifnet_alloc_os_total); STATCOPY(nas_pf_addrule_total); STATCOPY(nas_pf_addrule_os); STATCOPY(nas_vmnet_total); #undef STATCOPY #undef STATDIFF nstat_sysinfo_send_data(&data); /* * Save a copy of the current fields so we can diff them the next time */ memcpy(&net_api_stats_before, &net_api_stats, sizeof(struct net_api_stats)); static_assert(sizeof(net_api_stats_before) == sizeof(net_api_stats)); } #pragma mark -- Kernel Control Socket -- static kern_ctl_ref nstat_ctlref = NULL; static errno_t nstat_client_connect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo); static errno_t nstat_client_disconnect(kern_ctl_ref kctl, u_int32_t unit, void *uinfo); static errno_t nstat_client_send(kern_ctl_ref kctl, u_int32_t unit, void *uinfo, mbuf_t m, int flags); static errno_t nstat_enqueue_success( uint64_t context, nstat_client *client, u_int16_t flags) { nstat_msg_hdr success; errno_t result; bzero(&success, sizeof(success)); success.context = context; success.type = NSTAT_MSG_TYPE_SUCCESS; success.length = sizeof(success); success.flags = flags; result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &success, sizeof(success), CTL_DATA_EOR | CTL_DATA_CRIT); if (result != 0) { if (nstat_debug != 0) { NSTAT_LOG_ERROR("could not enqueue success message %d", result); } nstat_stats.nstat_successmsgfailures += 1; } return result; } static errno_t nstat_client_send_event( nstat_client *client, nstat_src *src, u_int64_t event) { errno_t result = ENOTSUP; if (nstat_client_reporting_allowed(client, src, 0)) { if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_DETAILS) != 0) { result = nstat_client_send_details(client, src, 0, event, 0, NULL); if (result != 0) { if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_event() %d", result); } } } else if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_UPDATES) != 0) { result = nstat_client_send_update(client, src, 0, event, 0, NULL); if (result == 0) { NSTAT_NOTE_SRC(nstat_send_event, client, src); } else { NSTAT_NOTE_SRC(nstat_send_event_fail, client, src); if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_event() %d", result); } } } else { NSTAT_NOTE_SRC(nstat_send_event_notsup, client, src); if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_event() used when updates not supported"); } } } return result; } static errno_t nstat_client_send_goodbye( nstat_client *client, nstat_src *src) { errno_t result = 0; int failed = 0; u_int16_t hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_FILTER; if (nstat_client_reporting_allowed(client, src, (src->nts_reported)? NSTAT_FILTER_SUPPRESS_BORING_CLOSE: 0)) { hdr_flags = 0; if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_DETAILS) != 0) { result = nstat_client_send_details(client, src, 0, 0, NSTAT_MSG_HDR_FLAG_CLOSING, NULL); if (result != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_failed_details, client, src); failed = 1; hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_details() %d", result); } } else { NSTAT_NOTE_SRC(nstat_src_goodbye_sent_details, client, src); } } else if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_UPDATES) != 0) { result = nstat_client_send_update(client, src, 0, 0, NSTAT_MSG_HDR_FLAG_CLOSING, NULL); if (result != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_failed_update, client, src); failed = 1; hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_update() %d", result); } } else { NSTAT_NOTE_SRC(nstat_src_goodbye_sent_update, client, src); } } else { // send one last counts notification result = nstat_client_send_counts(client, src, 0, NSTAT_MSG_HDR_FLAG_CLOSING, NULL); if (result != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_failed_counts, client, src); failed = 1; hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_counts() %d", result); } } else { NSTAT_NOTE_SRC(nstat_src_goodbye_sent_counts, client, src); } // send a last description result = nstat_client_send_description(client, src, 0, NSTAT_MSG_HDR_FLAG_CLOSING); if (result != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_failed_description, client, src); failed = 1; hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_description() %d", result); } } else { NSTAT_NOTE_SRC(nstat_src_goodbye_sent_description, client, src); } } } else { if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_DETAILS) != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_filtered_details, client, src); } else if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_UPDATES) != 0) { NSTAT_NOTE_SRC(nstat_src_goodbye_filtered_update, client, src); } else { NSTAT_NOTE_SRC(nstat_src_goodbye_filtered_counts, client, src); } } // send the source removed notification result = nstat_client_send_removed(client, src, hdr_flags); if (result != 0 && nstat_debug) { NSTAT_NOTE_SRC(nstat_src_goodbye_failed_removed, client, src); failed = 1; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_removed() %d", result); } } else { NSTAT_NOTE_SRC(nstat_src_goodbye_sent_removed, client, src); } if (failed != 0) { nstat_stats.nstat_control_send_goodbye_failures++; } return result; } static errno_t nstat_flush_accumulated_msgs( nstat_client *client) { errno_t result = 0; if (client->ntc_accumulated != NULL && mbuf_len(client->ntc_accumulated) > 0) { mbuf_pkthdr_setlen(client->ntc_accumulated, mbuf_len(client->ntc_accumulated)); result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, client->ntc_accumulated, CTL_DATA_EOR); if (result != 0) { nstat_stats.nstat_flush_accumulated_msgs_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("ctl_enqueuembuf failed: %d", result); } mbuf_freem(client->ntc_accumulated); } client->ntc_accumulated = NULL; } return result; } static errno_t nstat_accumulate_msg( nstat_client *client, uint8_t *__sized_by(length) msg, size_t length) { assert(length <= MAX_NSTAT_MSG_HDR_LENGTH); if (client->ntc_accumulated && mbuf_trailingspace(client->ntc_accumulated) < length) { // Will send the current mbuf nstat_flush_accumulated_msgs(client); } errno_t result = 0; if (client->ntc_accumulated == NULL) { unsigned int one = 1; if (mbuf_allocpacket(MBUF_DONTWAIT, NSTAT_MAX_MSG_SIZE, &one, &client->ntc_accumulated) != 0) { if (nstat_debug != 0) { NSTAT_LOG_ERROR("mbuf_allocpacket failed"); } result = ENOMEM; } else { mbuf_setlen(client->ntc_accumulated, 0); } } if (result == 0) { result = mbuf_copyback(client->ntc_accumulated, mbuf_len(client->ntc_accumulated), length, msg, MBUF_DONTWAIT); } if (result != 0) { nstat_flush_accumulated_msgs(client); if (nstat_debug != 0) { NSTAT_LOG_ERROR("resorting to ctl_enqueuedata"); } result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, msg, length, CTL_DATA_EOR); } if (result != 0) { nstat_stats.nstat_accumulate_msg_failures++; } return result; } static void nstat_idle_check( __unused thread_call_param_t p0, __unused thread_call_param_t p1) { nstat_client *client; nstat_src *src, *tmpsrc; struct nstat_sock_locus *sol, *tmpsol; tailq_head_nstat_src dead_list; TAILQ_INIT(&dead_list); NSTAT_LOCK_EXCLUSIVE(); nstat_idle_time = 0; TAILQ_FOREACH_SAFE(sol, &nstat_tcp_sock_locus_head, nsl_link, tmpsol) { assert(sol->nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); assert(sol->nsl_inp != NULL); assert(sol->nsl_is_tcp != 0); struct inpcb *inp = sol->nsl_inp; assert(inp->inp_nstat_locus == sol); // Ideally all dead inpcbs should have been removed from the list, but be paranoid if (inp->inp_state == INPCB_STATE_DEAD) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_idlecheck_tcp_gone); TAILQ_FOREACH_SAFE(src, &inp->inp_nstat_locus->nsl_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(sol == (struct nstat_sock_locus *)src->nts_cookie); client = src->nts_client; NSTAT_NOTE_QUAL(nstat_add_all_tcp_skip_dead, client, 0); nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } TAILQ_REMOVE(&nstat_tcp_sock_locus_head, sol, nsl_link); inp->inp_nstat_locus = NULL; nstat_sock_locus_release(sol, FALSE); } } // Give others a chance to run NSTAT_LOCK_YIELD_EXCLUSIVE(); TAILQ_FOREACH_SAFE(sol, &nstat_udp_sock_locus_head, nsl_link, tmpsol) { assert(sol->nsl_magic == NSTAT_SOCK_LOCUS_MAGIC); assert(sol->nsl_inp != NULL); assert(sol->nsl_is_tcp == 0); struct inpcb *inp = sol->nsl_inp; // Ideally all dead inpcbs should have been removed from the list, but be paranoid if (inp->inp_state == INPCB_STATE_DEAD) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_idlecheck_udp_gone); TAILQ_FOREACH_SAFE(src, &inp->inp_nstat_locus->nsl_locus.ntl_src_queue, nts_locus_link, tmpsrc) { assert(sol == (struct nstat_sock_locus *)src->nts_cookie); client = src->nts_client; NSTAT_NOTE_QUAL(nstat_add_all_tcp_skip_dead, client, 0); nstat_client_send_goodbye(client, src); nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); } TAILQ_REMOVE(&nstat_tcp_sock_locus_head, sol, nsl_link); inp->inp_nstat_locus = NULL; nstat_sock_locus_release(sol, FALSE); } } // Give others a chance to run NSTAT_LOCK_YIELD_EXCLUSIVE(); // Only routes and interfaces are left for the "gone" check. // Routes should at some point be moved to the locus mechanism // and interface functionality provided by alternative means to the // NSTAT_PROVIDER_IFNET, which will remove the need to scan all possible sources. // In the meantime, they should be a minority interest and // it is expected that most clients will be skipped in the sequence below for (client = nstat_clients; client; client = client->ntc_next) { if (((client->ntc_watching & (1 << NSTAT_PROVIDER_ROUTE)) != 0) || ((client->ntc_added_src & (1 << NSTAT_PROVIDER_ROUTE)) != 0) || ((client->ntc_added_src & (1 << NSTAT_PROVIDER_IFNET)) != 0)) { // this client is watching routes TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc) { if (((src->nts_provider->nstat_provider_id == NSTAT_PROVIDER_ROUTE) || (src->nts_provider->nstat_provider_id == NSTAT_PROVIDER_IFNET)) && (src->nts_provider->nstat_gone(src->nts_cookie))) { errno_t result; // Pull it off the list NSTAT_NOTE_SRC(nstat_route_src_gone_idlecheck, client, src); nstat_src_remove_linkages(client, src); result = nstat_client_send_goodbye(client, src); // Put this on the list to release later TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_idlecheck_route_src_gone); } } } } if (nstat_clients) { clock_interval_to_deadline(60, NSEC_PER_SEC, &nstat_idle_time); thread_call_func_delayed((thread_call_func_t)nstat_idle_check, NULL, nstat_idle_time); } NSTAT_UNLOCK_EXCLUSIVE(); /* Generate any system level reports, if needed */ nstat_sysinfo_generate_report(); // Release the sources now that we aren't holding locks while ((src = TAILQ_FIRST(&dead_list))) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_src_idlecheck_gone); TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } nstat_prune_procdetails(); } static void nstat_client_register(void) { // Register the control struct kern_ctl_reg nstat_control; bzero(&nstat_control, sizeof(nstat_control)); strlcpy(nstat_control.ctl_name, NET_STAT_CONTROL_NAME, sizeof(nstat_control.ctl_name)); nstat_control.ctl_flags = CTL_FLAG_REG_EXTENDED | CTL_FLAG_REG_CRIT; nstat_control.ctl_sendsize = nstat_sendspace; nstat_control.ctl_recvsize = nstat_recvspace; nstat_control.ctl_connect = nstat_client_connect; nstat_control.ctl_disconnect = nstat_client_disconnect; nstat_control.ctl_send = nstat_client_send; ctl_register(&nstat_control, &nstat_ctlref); } static void nstat_client_cleanup_source( nstat_client *client, struct nstat_src *src) { errno_t result; if (client) { result = nstat_client_send_removed(client, src, 0); if (result != 0) { nstat_stats.nstat_control_cleanup_source_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_removed() %d", result); } } } // Cleanup the source if we found it. src->nts_provider->nstat_release(src->nts_cookie); NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_src_current); kfree_type(struct nstat_src, src); } static bool nstat_client_reporting_allowed( nstat_client *client, nstat_src *src, u_int64_t suppression_flags) { if (src->nts_provider->nstat_reporting_allowed == NULL) { return TRUE; } return src->nts_provider->nstat_reporting_allowed(src->nts_cookie, &client->ntc_provider_filters[src->nts_provider->nstat_provider_id], suppression_flags); } static errno_t nstat_client_connect( kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo) { nstat_client *client = kalloc_type(nstat_client, Z_WAITOK | Z_ZERO); if (client == NULL) { NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_client_alloc_fails); return ENOMEM; } lck_mtx_init(&client->ntc_user_mtx, &nstat_lck_grp, NULL); client->ntc_kctl = kctl; client->ntc_unit = sac->sc_unit; client->ntc_flags = NSTAT_FLAG_REQCOUNTS; client->ntc_procdetails = nstat_retain_curprocdetails(); *uinfo = client; NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_client_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_client_current, nstat_global_client_max); #if NSTAT_TRACE_ENABLED client->ntc_trace = (nstat_cyclic_trace *)kalloc_data(sizeof(nstat_cyclic_trace), Z_WAITOK | Z_ZERO); #endif NSTAT_LOCK_EXCLUSIVE(); client->ntc_client_id = nstat_next_client_id++; client->ntc_next = nstat_clients; nstat_clients = client; if (nstat_idle_time == 0) { clock_interval_to_deadline(60, NSEC_PER_SEC, &nstat_idle_time); thread_call_func_delayed((thread_call_func_t)nstat_idle_check, NULL, nstat_idle_time); } merge_current_event_filters(); NSTAT_UNLOCK_EXCLUSIVE(); return 0; } static errno_t nstat_client_disconnect( __unused kern_ctl_ref kctl, __unused u_int32_t unit, void *uinfo) { u_int32_t watching; nstat_client *client = (nstat_client*)uinfo; tailq_head_nstat_src cleanup_list; nstat_src *src; TAILQ_INIT(&cleanup_list); // pull it out of the global list of clients NSTAT_LOCK_EXCLUSIVE(); nstat_client **clientpp; for (clientpp = &nstat_clients; *clientpp; clientpp = &(*clientpp)->ntc_next) { if (*clientpp == client) { *clientpp = client->ntc_next; break; } } merge_current_event_filters(); // Stop watching for sources nstat_provider *provider; watching = client->ntc_watching; client->ntc_watching = 0; for (provider = nstat_providers; provider && watching; provider = provider->next) { if ((watching & (1 << provider->nstat_provider_id)) != 0) { watching &= ~(1 << provider->nstat_provider_id); provider->nstat_watcher_remove(client); } } // set cleanup flags client->ntc_flags |= NSTAT_FLAG_CLEANUP; if (client->ntc_accumulated) { mbuf_freem(client->ntc_accumulated); client->ntc_accumulated = NULL; } // Remove any source links to associated locus TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { if (src->nts_locus != NULL) { TAILQ_REMOVE(&src->nts_locus->ntl_src_queue, src, nts_locus_link); src->nts_locus = NULL; } } // Copy out the list of sources TAILQ_CONCAT(&cleanup_list, &client->ntc_src_queue, nts_client_link); NSTAT_UNLOCK_EXCLUSIVE(); while ((src = TAILQ_FIRST(&cleanup_list))) { TAILQ_REMOVE(&cleanup_list, src, nts_client_link); nstat_client_cleanup_source(NULL, src); } lck_mtx_destroy(&client->ntc_user_mtx, &nstat_lck_grp); nstat_release_procdetails(client->ntc_procdetails); nstat_accumulate_client_metrics(&nstat_metrics, client); #if NSTAT_TRACE_ENABLED if (client->ntc_trace != NULL) { kfree_data(client->ntc_trace, sizeof(nstat_cyclic_trace)); } #endif NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_client_current); kfree_type(struct nstat_client, client); return 0; } static nstat_src_ref_t nstat_client_next_src_ref( nstat_client *client) { return ++client->ntc_next_srcref; } static errno_t nstat_client_send_counts( nstat_client *client, nstat_src *src, unsigned long long context, u_int16_t hdr_flags, int *gone) { nstat_msg_src_counts counts; errno_t result = 0; /* Some providers may not have any counts to send */ if (src->nts_provider->nstat_counts == NULL) { return 0; } bzero(&counts, sizeof(counts)); counts.hdr.type = NSTAT_MSG_TYPE_SRC_COUNTS; counts.hdr.length = sizeof(counts); counts.hdr.flags = hdr_flags; counts.hdr.context = context; counts.srcref = src->nts_srcref; counts.event_flags = 0; if (src->nts_provider->nstat_counts(src->nts_cookie, &counts.counts, gone) == 0) { if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) && counts.counts.nstat_rxbytes == 0 && counts.counts.nstat_txbytes == 0) { result = EAGAIN; } else { result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &counts, sizeof(counts), CTL_DATA_EOR); if (result != 0) { nstat_stats.nstat_sendcountfailures += 1; } } } return result; } static errno_t nstat_client_append_counts( nstat_client *client, nstat_src *src, int *gone) { /* Some providers may not have any counts to send */ if (!src->nts_provider->nstat_counts) { return 0; } nstat_msg_src_counts counts; bzero(&counts, sizeof(counts)); counts.hdr.type = NSTAT_MSG_TYPE_SRC_COUNTS; counts.hdr.length = sizeof(counts); counts.srcref = src->nts_srcref; counts.event_flags = 0; errno_t result = 0; result = src->nts_provider->nstat_counts(src->nts_cookie, &counts.counts, gone); if (result != 0) { return result; } if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES && counts.counts.nstat_rxbytes == 0 && counts.counts.nstat_txbytes == 0) { return EAGAIN; } return nstat_accumulate_msg(client, (uint8_t *)&counts, sizeof(counts)); } static int nstat_client_send_description( nstat_client *client, nstat_src *src, u_int64_t context, u_int16_t hdr_flags) { // Provider doesn't support getting the descriptor? Done. if (src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) { return EOPNOTSUPP; } // Allocate storage for the descriptor message mbuf_ref_t msg; unsigned int one = 1; size_t size = offsetof(nstat_msg_src_description, data) + src->nts_provider->nstat_descriptor_length; assert(size <= MAX_NSTAT_MSG_HDR_LENGTH); if (mbuf_allocpacket(MBUF_DONTWAIT, size, &one, &msg) != 0) { return ENOMEM; } bzero(m_mtod_current(msg), size); mbuf_setlen(msg, size); mbuf_pkthdr_setlen(msg, mbuf_len(msg)); // Query the provider for the provider specific bits nstat_msg_src_description *desc = mtod(msg, nstat_msg_src_description *); desc->hdr.context = context; desc->hdr.type = NSTAT_MSG_TYPE_SRC_DESC; desc->hdr.length = (u_int16_t)size; desc->hdr.flags = hdr_flags; desc->srcref = src->nts_srcref; desc->event_flags = 0; desc->provider = src->nts_provider->nstat_provider_id; uint8_t *desc_data_ptr = nstat_get_data(desc); errno_t result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { mbuf_freem(msg); return result; } result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR); if (result != 0) { nstat_stats.nstat_descriptionfailures += 1; mbuf_freem(msg); } return result; } static errno_t nstat_client_append_description( nstat_client *client, nstat_src *src) { size_t size = offsetof(nstat_msg_src_description, data) + src->nts_provider->nstat_descriptor_length; if (size > 512 || src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) { return EOPNOTSUPP; } // Fill out a buffer on the stack, we will copy to the mbuf later u_int64_t buffer[size / sizeof(u_int64_t) + 1]; // u_int64_t to ensure alignment bzero(buffer, size); nstat_msg_src_description *desc = (nstat_msg_src_description*)buffer; desc->hdr.type = NSTAT_MSG_TYPE_SRC_DESC; desc->hdr.length = (u_int16_t)size; desc->srcref = src->nts_srcref; desc->event_flags = 0; desc->provider = src->nts_provider->nstat_provider_id; errno_t result = 0; // Fill in the description // Query the provider for the provider specific bits uint8_t *desc_data_ptr = nstat_get_data(desc); result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { return result; } return nstat_accumulate_msg(client, (uint8_t *)buffer, size); } static uint64_t nstat_extension_flags_for_source( nstat_client *client, nstat_src *src) { VERIFY(client != NULL & src != NULL); nstat_provider_id_t provider_id = src->nts_provider->nstat_provider_id; return client->ntc_provider_filters[provider_id].npf_extensions; } static int nstat_client_send_update( nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone) { // Provider doesn't support getting the descriptor or counts? Done. if ((src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) && src->nts_provider->nstat_counts == NULL) { return EOPNOTSUPP; } // Allocate storage for the descriptor message mbuf_ref_t msg; unsigned int one = 1; size_t size = offsetof(nstat_msg_src_update, data) + src->nts_provider->nstat_descriptor_length; size_t total_extension_size = 0; u_int32_t num_extensions = 0; u_int64_t extension_mask = nstat_extension_flags_for_source(client, src); if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) { uint32_t extension_id = 0; for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0); if (extension_size == 0) { extension_mask &= ~(1ull << extension_id); } else { num_extensions++; total_extension_size += ROUNDUP64(extension_size); } } } size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions); } assert(size <= MAX_NSTAT_MSG_HDR_LENGTH); /* * XXX Would be interesting to see how extended updates affect mbuf * allocations, given the max segments defined as 1, one may get * allocations with higher fragmentation. */ if (mbuf_allocpacket(MBUF_DONTWAIT, size, &one, &msg) != 0) { return ENOMEM; } /* zero out for nstat_msg_src_update */ bzero(m_mtod_current(msg), size); nstat_msg_src_update *desc = mtod(msg, nstat_msg_src_update *); desc->hdr.context = context; desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_UPDATE : NSTAT_MSG_TYPE_SRC_EXTENDED_UPDATE; desc->hdr.length = (u_int16_t)size; desc->hdr.flags = hdr_flags; desc->srcref = src->nts_srcref; desc->event_flags = event; desc->provider = src->nts_provider->nstat_provider_id; /* * XXX The following two lines are only valid when max-segments is passed * as one. * Other computations with offset also depend on that being true. * Be aware of that before making any modifications that changes that * behavior. */ mbuf_setlen(msg, size); mbuf_pkthdr_setlen(msg, mbuf_len(msg)); errno_t result = 0; if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) { // Query the provider for the provider specific bits u_int8_t *desc_data_ptr = nstat_get_data(desc); result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { mbuf_freem(msg); return result; } } if (num_extensions > 0) { nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)mtodo(msg, sizeof(nstat_msg_src_update_hdr) + src->nts_provider->nstat_descriptor_length); uint32_t extension_id = 0; bzero(p_extension_hdr, total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions)); for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { void *buf = (void *)(p_extension_hdr + 1); size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size); if ((extension_size == 0) || (extension_size > total_extension_size)) { // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN; p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1)); break; } else { // The extension may be of any size alignment, reported as such in the extension header, // but we pad to ensure that whatever comes next is suitably aligned p_extension_hdr->type = extension_id; p_extension_hdr->length = extension_size; extension_size = ROUNDUP64(extension_size); total_extension_size -= extension_size; p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size); num_extensions--; } } } } if (src->nts_provider->nstat_counts) { result = src->nts_provider->nstat_counts(src->nts_cookie, &desc->counts, gone); if (result == 0) { if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES && desc->counts.nstat_rxbytes == 0 && desc->counts.nstat_txbytes == 0) { result = EAGAIN; } else { result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR); } } } if (result != 0) { nstat_stats.nstat_srcupatefailures += 1; mbuf_freem(msg); } else { src->nts_reported = true; } return result; } static int nstat_client_send_details( nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone) { // Provider doesn't support getting the descriptor or counts? Done. if ((src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) || src->nts_provider->nstat_details == NULL) { return EOPNOTSUPP; } // Allocate storage for the descriptor message mbuf_ref_t msg; unsigned int one = 1; size_t size = offsetof(nstat_msg_src_details, data) + src->nts_provider->nstat_descriptor_length; size_t total_extension_size = 0; u_int32_t num_extensions = 0; u_int64_t extension_mask = nstat_extension_flags_for_source(client, src); if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) { uint32_t extension_id = 0; for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0); if (extension_size == 0) { extension_mask &= ~(1ull << extension_id); } else { num_extensions++; total_extension_size += ROUNDUP64(extension_size); } } } size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions); } assert(size <= MAX_NSTAT_MSG_HDR_LENGTH); /* * XXX Would be interesting to see how extended details affect mbuf * allocations, given the max segments defined as 1, one may get * allocations with higher fragmentation. */ if (mbuf_allocpacket(MBUF_DONTWAIT, size, &one, &msg) != 0) { return ENOMEM; } /* zero out for nstat_msg_src_details */ bzero(m_mtod_current(msg), size); nstat_msg_src_details *desc = mtod(msg, nstat_msg_src_details *); desc->hdr.context = context; desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_DETAILS : NSTAT_MSG_TYPE_SRC_EXTENDED_DETAILS; desc->hdr.length = (u_int16_t)size; desc->hdr.flags = hdr_flags; desc->srcref = src->nts_srcref; desc->event_flags = event; desc->provider = src->nts_provider->nstat_provider_id; /* * XXX The following two lines are only valid when max-segments is passed * as one. * Other computations with offset also depend on that being true. * Be aware of that before making any modifications that changes that * behavior. */ mbuf_setlen(msg, size); mbuf_pkthdr_setlen(msg, mbuf_len(msg)); errno_t result = 0; if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) { // Query the provider for the provider specific bits u_int8_t *desc_data_ptr = nstat_get_data(desc); result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { mbuf_freem(msg); return result; } } if (num_extensions > 0) { nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)mtodo(msg, sizeof(nstat_msg_src_details_hdr) + src->nts_provider->nstat_descriptor_length); uint32_t extension_id = 0; for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { void *buf = (void *)(p_extension_hdr + 1); size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size); if ((extension_size == 0) || (extension_size > total_extension_size)) { // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN; p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1)); break; } else { // The extension may be of any size alignment, reported as such in the extension header, // but we pad to ensure that whatever comes next is suitably aligned p_extension_hdr->type = extension_id; p_extension_hdr->length = extension_size; extension_size = ROUNDUP64(extension_size); total_extension_size -= extension_size; p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size); num_extensions--; } } } } if (src->nts_provider->nstat_details) { result = src->nts_provider->nstat_details(src->nts_cookie, &desc->detailed_counts, gone); if (result == 0) { if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES && desc->detailed_counts.nstat_media_stats.ms_total.ts_rxbytes == 0 && desc->detailed_counts.nstat_media_stats.ms_total.ts_txbytes == 0) { result = EAGAIN; } else { result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR); } } } if (result != 0) { nstat_stats.nstat_srcupatefailures += 1; mbuf_freem(msg); } else { src->nts_reported = true; } return result; } static errno_t nstat_client_append_update( nstat_client *client, nstat_src *src, int *gone) { if ((src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) && src->nts_provider->nstat_counts == NULL) { return EOPNOTSUPP; } size_t size = offsetof(nstat_msg_src_update, data) + src->nts_provider->nstat_descriptor_length; size_t total_extension_size = 0; u_int32_t num_extensions = 0; u_int64_t extension_mask = nstat_extension_flags_for_source(client, src); if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) { uint32_t extension_id = 0; for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0); if (extension_size == 0) { extension_mask &= ~(1ull << extension_id); } else { num_extensions++; total_extension_size += ROUNDUP64(extension_size); } } } size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions); } /* * This kind of limits extensions. * The optimization is around being able to deliver multiple * of updates bundled together. * Increasing the size runs the risk of too much stack usage. * One could potentially changed the allocation below to be on heap. * For now limiting it to half of NSTAT_MAX_MSG_SIZE. */ if (size > (NSTAT_MAX_MSG_SIZE >> 1)) { return EOPNOTSUPP; } // Fill out a buffer on the stack, we will copy to the mbuf later u_int64_t buffer[size / sizeof(u_int64_t) + 1]; // u_int64_t to ensure alignment bzero(buffer, size); nstat_msg_src_update *desc = (nstat_msg_src_update*)buffer; desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_UPDATE : NSTAT_MSG_TYPE_SRC_EXTENDED_UPDATE; desc->hdr.length = (u_int16_t)size; desc->srcref = src->nts_srcref; desc->event_flags = 0; desc->provider = src->nts_provider->nstat_provider_id; errno_t result = 0; // Fill in the description if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) { // Query the provider for the provider specific bits u_int8_t *desc_data_ptr = nstat_get_data(desc); result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { nstat_stats.nstat_copy_descriptor_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("src->nts_provider->nstat_copy_descriptor: %d", result); } return result; } } if (num_extensions > 0) { nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buffer + sizeof(nstat_msg_src_update_hdr) + src->nts_provider->nstat_descriptor_length); uint32_t extension_id = 0; bzero(p_extension_hdr, total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions)); for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { void *buf = (void *)(p_extension_hdr + 1); size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size); if ((extension_size == 0) || (extension_size > total_extension_size)) { // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN; p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1)); break; } else { extension_size = ROUNDUP64(extension_size); p_extension_hdr->type = extension_id; p_extension_hdr->length = extension_size; total_extension_size -= extension_size; p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size); num_extensions--; } } } } if (src->nts_provider->nstat_counts) { result = src->nts_provider->nstat_counts(src->nts_cookie, &desc->counts, gone); if (result != 0) { nstat_stats.nstat_provider_counts_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("src->nts_provider->nstat_counts: %d", result); } return result; } if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES && desc->counts.nstat_rxbytes == 0 && desc->counts.nstat_txbytes == 0) { return EAGAIN; } } result = nstat_accumulate_msg(client, (uint8_t *)buffer, size); if (result == 0) { src->nts_reported = true; } return result; } static errno_t nstat_client_append_details( nstat_client *client, nstat_src *src, int *gone) { if ((src->nts_provider->nstat_descriptor_length == 0 || src->nts_provider->nstat_copy_descriptor == NULL) && src->nts_provider->nstat_details == NULL) { return EOPNOTSUPP; } size_t size = offsetof(nstat_msg_src_details, data) + src->nts_provider->nstat_descriptor_length; size_t total_extension_size = 0; u_int32_t num_extensions = 0; u_int64_t extension_mask = nstat_extension_flags_for_source(client, src); if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) { uint32_t extension_id = 0; for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0); if (extension_size == 0) { extension_mask &= ~(1ull << extension_id); } else { num_extensions++; total_extension_size += ROUNDUP64(extension_size); } } } size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions); } /* * This kind of limits extensions. * The optimization is around being able to deliver multiple * numbers of details bundled together. * Increasing the size runs the risk of too much stack usage. * One could potentially changed the allocation below to be on heap. * For now limiting it to half of NSTAT_MAX_MSG_SIZE. */ if (size > (NSTAT_MAX_MSG_SIZE >> 1)) { return EOPNOTSUPP; } // Fill out a buffer on the stack, we will copy to the mbuf later u_int64_t buffer[size / sizeof(u_int64_t) + 1]; // u_int64_t to ensure alignment bzero(buffer, size); nstat_msg_src_details *desc = (nstat_msg_src_details *)buffer; desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_DETAILS : NSTAT_MSG_TYPE_SRC_EXTENDED_DETAILS; desc->hdr.length = (u_int16_t)size; desc->srcref = src->nts_srcref; desc->event_flags = 0; desc->provider = src->nts_provider->nstat_provider_id; errno_t result = 0; // Fill in the description if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) { // Query the provider for the provider specific bits u_int8_t *desc_data_ptr = nstat_get_data(desc); result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length); if (result != 0) { nstat_stats.nstat_copy_descriptor_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("src->nts_provider->nstat_copy_descriptor: %d", result); } return result; } } if (num_extensions > 0) { nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buffer + sizeof(nstat_msg_src_details_hdr) + src->nts_provider->nstat_descriptor_length); uint32_t extension_id = 0; bzero(p_extension_hdr, total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions)); for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) { if ((extension_mask & (1ull << extension_id)) != 0) { void *buf = (void *)(p_extension_hdr + 1); size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size); if ((extension_size == 0) || (extension_size > total_extension_size)) { // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN; p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1)); break; } else { extension_size = ROUNDUP64(extension_size); p_extension_hdr->type = extension_id; p_extension_hdr->length = extension_size; total_extension_size -= extension_size; p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size); num_extensions--; } } } } if (src->nts_provider->nstat_details) { result = src->nts_provider->nstat_details(src->nts_cookie, &desc->detailed_counts, gone); if (result != 0) { nstat_stats.nstat_provider_counts_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("src->nts_provider->nstat_counts: %d", result); } return result; } if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES && desc->detailed_counts.nstat_media_stats.ms_total.ts_rxbytes == 0 && desc->detailed_counts.nstat_media_stats.ms_total.ts_txbytes == 0) { return EAGAIN; } } result = nstat_accumulate_msg(client, (uint8_t *)buffer, size); if (result == 0) { src->nts_reported = true; } return result; } static errno_t nstat_client_send_removed( nstat_client *client, nstat_src *src, u_int16_t hdr_flags) { nstat_msg_src_removed removed; errno_t result; bzero(&removed, sizeof(removed)); removed.hdr.type = NSTAT_MSG_TYPE_SRC_REMOVED; removed.hdr.length = sizeof(removed); removed.hdr.context = 0; removed.hdr.flags = hdr_flags; removed.srcref = src->nts_srcref; result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &removed, sizeof(removed), CTL_DATA_EOR | CTL_DATA_CRIT); if (result != 0) { nstat_stats.nstat_msgremovedfailures += 1; } return result; } static errno_t nstat_client_handle_add_request( nstat_client *client, mbuf_t m) { errno_t result; // Verify the header fits in the first mbuf if (mbuf_len(m) < offsetof(nstat_msg_add_src_req, param)) { return EINVAL; } // Calculate the length of the parameter field ssize_t paramlength = mbuf_pkthdr_len(m) - offsetof(nstat_msg_add_src_req, param); if (paramlength < 0 || paramlength > 2 * 1024) { return EINVAL; } nstat_provider *__single provider = NULL; nstat_provider_cookie_t __single cookie = NULL; nstat_msg_add_src_req *req = mtod(m, nstat_msg_add_src_req *); if (mbuf_pkthdr_len(m) > mbuf_len(m)) { // parameter is too large, we need to make a contiguous copy void *data = (void *) kalloc_data(paramlength, Z_WAITOK); if (!data) { return ENOMEM; } result = mbuf_copydata(m, offsetof(nstat_msg_add_src_req, param), paramlength, data); if (result == 0) { result = nstat_lookup_entry(req->provider, data, paramlength, &provider, &cookie); } kfree_data(data, paramlength); } else { uint8_t *req_param_ptr = nstat_get_data(req); result = nstat_lookup_entry(req->provider, req_param_ptr, paramlength, &provider, &cookie); } if (result != 0) { return result; } // sanitize cookie nstat_client_sanitize_cookie(client, provider->nstat_provider_id, cookie); NSTAT_LOCK_EXCLUSIVE(); result = nstat_client_source_add(req->hdr.context, client, provider, cookie, NULL); NSTAT_UNLOCK_EXCLUSIVE(); if (result != 0) { provider->nstat_release(cookie); } // Set the flag if a provider added a single source os_atomic_or(&client->ntc_added_src, (1 << provider->nstat_provider_id), relaxed); return result; } static errno_t nstat_set_provider_filter( nstat_client *client, nstat_msg_add_all_srcs *req) { nstat_provider_id_t provider_id = req->provider; u_int32_t prev_ntc_watching = os_atomic_or_orig(&client->ntc_watching, (1 << provider_id), relaxed); // Reject it if the client is already watching all the sources. if ((prev_ntc_watching & (1 << provider_id)) != 0) { return EALREADY; } // Reject it if any single source has already been added. u_int32_t ntc_added_src = os_atomic_load(&client->ntc_added_src, relaxed); if ((ntc_added_src & (1 << provider_id)) != 0) { return EALREADY; } client->ntc_watching |= (1 << provider_id); client->ntc_provider_filters[provider_id].npf_events = req->events; client->ntc_provider_filters[provider_id].npf_flags = req->filter; client->ntc_provider_filters[provider_id].npf_pid = req->target_pid; uuid_copy(client->ntc_provider_filters[provider_id].npf_uuid, req->target_uuid); // The extensions should be populated by a more direct mechanism // Using the top 32 bits of the filter flags reduces the namespace of both, // but is a convenient workaround that avoids ntstat.h changes that would require rebuild of all clients // Extensions give away additional privacy information and are subject to unconditional privilege check, // unconstrained by the value of nstat_privcheck if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) == 0) { client->ntc_provider_filters[provider_id].npf_extensions = (req->filter >> NSTAT_FILTER_ALLOWED_EXTENSIONS_SHIFT) & NSTAT_EXTENDED_UPDATE_FLAG_MASK; } return 0; } static errno_t nstat_client_handle_add_all( nstat_client *client, mbuf_t m) { errno_t result = 0; // Verify the header fits in the first mbuf if (mbuf_len(m) < sizeof(nstat_msg_add_all_srcs)) { return EINVAL; } nstat_msg_add_all_srcs *req = mtod(m, nstat_msg_add_all_srcs *); if (req->provider > NSTAT_PROVIDER_LAST) { return ENOENT; } nstat_provider *provider = nstat_find_provider_by_id(req->provider); if (!provider) { return ENOENT; } if (provider->nstat_watcher_add == NULL) { return ENOTSUP; } // Traditionally the nstat_privcheck value allowed for easy access to ntstat on the Mac. // Keep backwards compatibility while being more stringent with recent providers if ((nstat_privcheck != 0) || (req->provider == NSTAT_PROVIDER_UDP_SUBFLOW) || (req->provider == NSTAT_PROVIDER_CONN_USERLAND)) { result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0); if (result != 0) { return result; } } NSTAT_LOCK_EXCLUSIVE(); if (req->filter & NSTAT_FILTER_SUPPRESS_SRC_ADDED) { // Suppression of source messages implicitly requires the use of update messages client->ntc_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; } NSTAT_UNLOCK_EXCLUSIVE(); // rdar://problem/30301300 Different providers require different synchronization // to ensure that a new entry does not get double counted due to being added prior // to all current provider entries being added. Hence pass the provider the details // in the original request for this to be applied atomically result = provider->nstat_watcher_add(client, req); if (result == 0) { nstat_enqueue_success(req->hdr.context, client, 0); } return result; } static errno_t nstat_client_source_add( u_int64_t context, nstat_client *client, nstat_provider *provider, nstat_provider_cookie_t cookie, nstat_locus *locus) { // It is a condition of this function that the lock be held in exclusive mode NSTAT_ASSERT_LOCKED_EXCLUSIVE(); // Fill out source added message if appropriate errno_t result = 0; mbuf_ref_t msg = NULL; nstat_src_ref_t *srcrefp = NULL; u_int64_t provider_filter_flags = client->ntc_provider_filters[provider->nstat_provider_id].npf_flags; boolean_t tell_user = ((provider_filter_flags & NSTAT_FILTER_SUPPRESS_SRC_ADDED) == 0); u_int32_t src_filter = (provider_filter_flags & NSTAT_FILTER_PROVIDER_NOZEROBYTES)? NSTAT_FILTER_NOZEROBYTES : 0; if (provider_filter_flags & NSTAT_FILTER_TCP_NO_EARLY_CLOSE) { src_filter |= NSTAT_FILTER_TCP_NO_EARLY_CLOSE; } do { if (tell_user) { unsigned int one = 1; if (mbuf_allocpacket(MBUF_DONTWAIT, sizeof(nstat_msg_src_added), &one, &msg) != 0) { NSTAT_NOTE_QUAL(nstat_src_add_no_buf, client, 0); result = ENOMEM; break; } mbuf_setlen(msg, sizeof(nstat_msg_src_added)); mbuf_pkthdr_setlen(msg, mbuf_len(msg)); bzero(m_mtod_current(msg), sizeof(nstat_msg_src_added)); nstat_msg_src_added *add = mtod(msg, nstat_msg_src_added *); add->hdr.type = NSTAT_MSG_TYPE_SRC_ADDED; assert(mbuf_len(msg) <= MAX_NSTAT_MSG_HDR_LENGTH); add->hdr.length = (u_int16_t)mbuf_len(msg); add->hdr.context = context; add->provider = provider->nstat_provider_id; srcrefp = &add->srcref; } // Allocate storage for the source nstat_src *src = kalloc_type(struct nstat_src, Z_WAITOK); if (src == NULL) { NSTAT_NOTE_QUAL(nstat_src_add_no_src_mem, client, 0); if (msg) { mbuf_freem(msg); } result = ENOMEM; break; } // Fill in the source, including picking an unused source ref src->nts_srcref = nstat_client_next_src_ref(client); if (srcrefp) { *srcrefp = src->nts_srcref; } if (client->ntc_flags & NSTAT_FLAG_CLEANUP || src->nts_srcref == NSTAT_SRC_REF_INVALID) { NSTAT_NOTE_QUAL(nstat_src_add_while_cleanup, client, 0); kfree_type(struct nstat_src, src); if (msg) { mbuf_freem(msg); } result = EINVAL; break; } src->nts_locus = locus; src->nts_provider = provider; src->nts_cookie = cookie; src->nts_filter = src_filter; src->nts_seq = 0; if (msg) { // send the source added message if appropriate result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR); if (result != 0) { NSTAT_NOTE_SRC(nstat_src_add_send_err, client, src); nstat_stats.nstat_srcaddedfailures += 1; kfree_type(struct nstat_src, src); mbuf_freem(msg); break; } } // Put the source in the list TAILQ_INSERT_HEAD(&client->ntc_src_queue, src, nts_client_link); src->nts_client = client; if (locus != NULL) { TAILQ_INSERT_HEAD(&locus->ntl_src_queue, src, nts_locus_link); } NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_src_allocs); NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_src_current, nstat_global_src_max); client->ntc_metrics.nstat_src_current++; if (client->ntc_metrics.nstat_src_current > client->ntc_metrics.nstat_src_max) { client->ntc_metrics.nstat_src_max = client->ntc_metrics.nstat_src_current; } NSTAT_NOTE_SRC(nstat_src_add_success, client, src); } while (0); return result; } static errno_t nstat_client_handle_remove_request( nstat_client *client, mbuf_t m) { nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID; nstat_src *src; if (mbuf_copydata(m, offsetof(nstat_msg_rem_src_req, srcref), sizeof(srcref), &srcref) != 0) { return EINVAL; } NSTAT_LOCK_EXCLUSIVE(); // Remove this source as we look for it TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { if (src->nts_srcref == srcref) { break; } } if (src) { nstat_src_remove_linkages(client, src); } NSTAT_UNLOCK_EXCLUSIVE(); if (src) { nstat_client_cleanup_source(client, src); NSTAT_NOTE_QUAL(nstat_remove_src_found, client, srcref); } else { NSTAT_NOTE_QUAL(nstat_remove_src_missed, client, srcref); } return src ? 0 : ENOENT; } static errno_t nstat_client_handle_query_request( nstat_client *client, mbuf_t m) { // TBD: handle this from another thread so we can enqueue a lot of data // As written, if a client requests query all, this function will be // called from their send of the request message. We will attempt to write // responses and succeed until the buffer fills up. Since the clients thread // is blocked on send, it won't be reading unless the client has two threads // using this socket, one for read and one for write. Two threads probably // won't work with this code anyhow since we don't have proper locking in // place yet. tailq_head_nstat_src dead_list; errno_t result = ENOENT; nstat_msg_query_src_req req; if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) { return EINVAL; } TAILQ_INIT(&dead_list); const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); if (all_srcs) { NSTAT_NOTE_QUAL(nstat_query_request_all, client, 0); } else { NSTAT_NOTE_QUAL(nstat_query_request_one, client, req.srcref); } NSTAT_LOCK_SHARED(); if (all_srcs) { client->ntc_flags |= NSTAT_FLAG_REQCOUNTS; } nstat_src *src, *tmpsrc; u_int64_t src_count = 0; boolean_t partial = FALSE; /* * Error handling policy and sequence number generation is folded into * nstat_client_begin_query. */ partial = nstat_client_begin_query(client, &req.hdr); TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc) { int gone = 0; // XXX ignore IFACE types? if (all_srcs || src->nts_srcref == req.srcref) { if (nstat_client_reporting_allowed(client, src, 0) && (!partial || !all_srcs || src->nts_seq != client->ntc_seq)) { if (all_srcs) { result = nstat_client_append_counts(client, src, &gone); } else { result = nstat_client_send_counts(client, src, req.hdr.context, 0, &gone); } if (ENOMEM == result || ENOBUFS == result) { /* * If the counts message failed to * enqueue then we should clear our flag so * that a client doesn't miss anything on * idle cleanup. We skip the "gone" * processing in the hope that we may * catch it another time. */ NSTAT_NOTE_SRC(nstat_query_request_nobuf, client, src); client->ntc_flags &= ~NSTAT_FLAG_REQCOUNTS; break; } if (partial) { /* * We skip over hard errors and * filtered sources. */ src->nts_seq = client->ntc_seq; src_count++; } } } if (gone) { // send one last descriptor message so client may see last state // If we can't send the notification now, it // will be sent in the idle cleanup. if (NSTAT_LOCK_SHARED_TO_EXCLUSIVE()) { NSTAT_NOTE_SRC(nstat_query_request_upgrade, client, src); // Successfully upgraded the lock, now we can remove the source from the client result = nstat_client_send_description(client, src, 0, 0); if (result != 0) { nstat_stats.nstat_control_send_description_failures++; if (nstat_debug != 0) { NSTAT_LOG_ERROR("nstat_client_send_description() %d", result); } client->ntc_flags &= ~NSTAT_FLAG_REQCOUNTS; NSTAT_NOTE_SRC(nstat_query_request_nodesc, client, src); NSTAT_LOCK_EXCLUSIVE_TO_SHARED(); break; } // pull src out of the list nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); NSTAT_LOCK_EXCLUSIVE_TO_SHARED(); } else { // The upgrade failed and the shared lock has been dropped // This should be rare. Simply drop out here and have user level retry // the poll, have the idle cleanup catch the "gone" source NSTAT_NOTE_SRC(nstat_query_request_noupgrade, client, src); NSTAT_LOCK_SHARED(); break; } } if (all_srcs) { if (src_count >= QUERY_CONTINUATION_SRC_COUNT) { NSTAT_NOTE_SRC(nstat_query_request_limit, client, src); break; } if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) && (NSTAT_LOCK_WOULD_YIELD())) { // A possibly higher priority thread is waiting // Exit from here and have user level initiate the next fragment NSTAT_NOTE_SRC(nstat_query_request_yield, client, src); break; } } else if (req.srcref == src->nts_srcref) { break; } } nstat_flush_accumulated_msgs(client); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) { flags = nstat_client_end_query(client, src, partial); } NSTAT_UNLOCK_SHARED(); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_client_send. This way, the error is sent to * user-level. */ if (all_srcs && ENOMEM != result && ENOBUFS != result) { nstat_enqueue_success(req.hdr.context, client, flags); result = 0; } while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); nstat_client_cleanup_source(client, src); } return result; } static errno_t nstat_client_handle_get_src_description( nstat_client *client, mbuf_t m) { nstat_msg_get_src_description req; errno_t result = ENOENT; nstat_src *src; if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) { return EINVAL; } NSTAT_LOCK_SHARED(); u_int64_t src_count = 0; boolean_t partial = FALSE; const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); if (all_srcs) { NSTAT_NOTE_QUAL(nstat_query_description_all, client, 0); } else { NSTAT_NOTE_QUAL(nstat_query_description_one, client, req.srcref); } /* * Error handling policy and sequence number generation is folded into * nstat_client_begin_query. */ partial = nstat_client_begin_query(client, &req.hdr); TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) { if (all_srcs || src->nts_srcref == req.srcref) { if (nstat_client_reporting_allowed(client, src, 0) && (!all_srcs || !partial || src->nts_seq != client->ntc_seq)) { if (all_srcs) { result = nstat_client_append_description(client, src); } else { result = nstat_client_send_description(client, src, req.hdr.context, 0); } if (ENOMEM == result || ENOBUFS == result) { /* * If the description message failed to * enqueue then we give up for now. */ break; } if (partial) { /* * Note, we skip over hard errors and * filtered sources. */ src->nts_seq = client->ntc_seq; src_count++; if (src_count >= QUERY_CONTINUATION_SRC_COUNT) { NSTAT_NOTE_SRC(nstat_query_description_limit, client, src); break; } if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) && (NSTAT_LOCK_WOULD_YIELD())) { // A possibly higher priority thread is waiting // Exit from here and have user level initiate the next fragment NSTAT_NOTE_SRC(nstat_query_description_yield, client, src); break; } } } if (!all_srcs) { break; } } } nstat_flush_accumulated_msgs(client); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) { flags = nstat_client_end_query(client, src, partial); } NSTAT_UNLOCK_SHARED(); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_client_send. This way, the error is sent to * user-level. */ if (all_srcs && ENOMEM != result && ENOBUFS != result) { nstat_enqueue_success(req.hdr.context, client, flags); result = 0; } return result; } static void nstat_send_error( nstat_client *client, u_int64_t context, u_int32_t error) { errno_t result; struct nstat_msg_error err; bzero(&err, sizeof(err)); err.hdr.type = NSTAT_MSG_TYPE_ERROR; err.hdr.length = sizeof(err); err.hdr.context = context; err.error = error; result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &err, sizeof(err), CTL_DATA_EOR | CTL_DATA_CRIT); if (result != 0) { nstat_stats.nstat_msgerrorfailures++; } } static boolean_t nstat_client_begin_query( nstat_client *client, const nstat_msg_hdr *hdrp) { boolean_t partial = FALSE; if (hdrp->flags & NSTAT_MSG_HDR_FLAG_CONTINUATION) { /* A partial query all has been requested. */ partial = TRUE; if (client->ntc_context != hdrp->context) { if (client->ntc_context != 0) { nstat_send_error(client, client->ntc_context, EAGAIN); } /* Initialize client for a partial query all. */ client->ntc_context = hdrp->context; client->ntc_seq++; } } return partial; } static u_int16_t nstat_client_end_query( nstat_client *client, nstat_src *last_src, boolean_t partial) { u_int16_t flags = 0; if (last_src == NULL || !partial) { /* * We iterated through the entire srcs list or exited early * from the loop when a partial update was not requested (an * error occurred), so clear context to indicate internally * that the query is finished. */ client->ntc_context = 0; } else { /* * Indicate to userlevel to make another partial request as * there are still sources left to be reported. */ flags |= NSTAT_MSG_HDR_FLAG_CONTINUATION; } return flags; } static errno_t nstat_client_handle_get_update( nstat_client *client, mbuf_t m) { nstat_msg_query_src_req req; if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) { return EINVAL; } NSTAT_LOCK_SHARED(); client->ntc_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; errno_t result = ENOENT; nstat_src *src, *tmpsrc; tailq_head_nstat_src dead_list; u_int64_t src_count = 0; boolean_t partial = FALSE; const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); TAILQ_INIT(&dead_list); if (all_srcs) { NSTAT_NOTE_QUAL(nstat_query_update_all, client, 0); } else { NSTAT_NOTE_QUAL(nstat_query_update_one, client, req.srcref); } /* * Error handling policy and sequence number generation is folded into * nstat_client_begin_query. */ partial = nstat_client_begin_query(client, &req.hdr); TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc) { int gone = 0; if (all_srcs) { // Check to see if we should handle this source or if we're still skipping to find where to continue if ((FALSE == partial || src->nts_seq != client->ntc_seq)) { u_int64_t suppression_flags = (src->nts_reported)? NSTAT_FILTER_SUPPRESS_BORING_POLL: 0; if (nstat_client_reporting_allowed(client, src, suppression_flags)) { result = nstat_client_append_update(client, src, &gone); if (ENOMEM == result || ENOBUFS == result) { /* * If the update message failed to * enqueue then give up. */ NSTAT_NOTE_SRC(nstat_query_update_nobuf, client, src); break; } if (partial) { /* * We skip over hard errors and * filtered sources. */ src->nts_seq = client->ntc_seq; src_count++; } } } } else if (src->nts_srcref == req.srcref) { if (nstat_client_reporting_allowed(client, src, 0)) { result = nstat_client_send_update(client, src, req.hdr.context, 0, 0, &gone); } } if (gone) { if (NSTAT_LOCK_SHARED_TO_EXCLUSIVE()) { // Successfully upgraded the lock, now we can remove the source from the client // pull src out of the list nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); NSTAT_LOCK_EXCLUSIVE_TO_SHARED(); NSTAT_NOTE_SRC(nstat_query_update_upgrade, client, src); } else { // The upgrade failed and the shared lock has been dropped // This should be rare. Simply drop out here and have user level retry // the poll, have the idle cleanup catch the "gone" source NSTAT_NOTE_SRC(nstat_query_update_noupgrade, client, src); NSTAT_LOCK_SHARED(); break; } } if (!all_srcs && req.srcref == src->nts_srcref) { break; } if (src_count >= QUERY_CONTINUATION_SRC_COUNT) { NSTAT_NOTE_SRC(nstat_query_update_limit, client, src); break; } if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) && (NSTAT_LOCK_WOULD_YIELD())) { // A possibly higher priority thread is waiting // Exit from here and have user level initiate the next fragment NSTAT_NOTE_SRC(nstat_query_update_yield, client, src); break; } } nstat_flush_accumulated_msgs(client); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) { flags = nstat_client_end_query(client, src, partial); } NSTAT_UNLOCK_SHARED(); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_client_send. This way, the error is sent to * user-level. */ if (all_srcs && ENOMEM != result && ENOBUFS != result) { nstat_enqueue_success(req.hdr.context, client, flags); result = 0; } while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); // release src and send notification nstat_client_cleanup_source(client, src); } return result; } static errno_t nstat_client_handle_get_details( nstat_client *client, mbuf_t m) { nstat_msg_query_src_req req; if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) { return EINVAL; } NSTAT_LOCK_SHARED(); client->ntc_flags |= NSTAT_FLAG_SUPPORTS_DETAILS; errno_t result = ENOENT; nstat_src *src, *tmpsrc; tailq_head_nstat_src dead_list; u_int64_t src_count = 0; boolean_t partial = FALSE; const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); TAILQ_INIT(&dead_list); if (all_srcs) { NSTAT_NOTE_QUAL(nstat_query_details_all, client, 0); } else { NSTAT_NOTE_QUAL(nstat_query_details_one, client, req.srcref); } /* * Error handling policy and sequence number generation is folded into * nstat_client_begin_query. */ partial = nstat_client_begin_query(client, &req.hdr); TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc) { int gone = 0; if (all_srcs) { // Check to see if we should handle this source or if we're still skipping to find where to continue if ((FALSE == partial || src->nts_seq != client->ntc_seq)) { u_int64_t suppression_flags = (src->nts_reported)? NSTAT_FILTER_SUPPRESS_BORING_POLL: 0; if (nstat_client_reporting_allowed(client, src, suppression_flags)) { result = nstat_client_append_details(client, src, &gone); if (ENOMEM == result || ENOBUFS == result) { /* * If the details message failed to * enqueue then give up. */ NSTAT_NOTE_SRC(nstat_query_details_nobuf, client, src); break; } if (partial) { /* * We skip over hard errors and * filtered sources. */ src->nts_seq = client->ntc_seq; src_count++; } } } } else if (src->nts_srcref == req.srcref) { if (nstat_client_reporting_allowed(client, src, 0)) { result = nstat_client_send_details(client, src, req.hdr.context, 0, 0, &gone); } } if (gone) { if (NSTAT_LOCK_SHARED_TO_EXCLUSIVE()) { // Successfully upgraded the lock, now we can remove the source from the client // pull src out of the list nstat_src_remove_linkages(client, src); TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link); NSTAT_LOCK_EXCLUSIVE_TO_SHARED(); NSTAT_NOTE_SRC(nstat_query_details_upgrade, client, src); } else { // The upgrade failed and the shared lock has been dropped // This should be rare. Simply drop out here and have user level retry // the poll, have the idle cleanup catch the "gone" source NSTAT_NOTE_SRC(nstat_query_details_noupgrade, client, src); NSTAT_LOCK_SHARED(); break; } } if (!all_srcs && req.srcref == src->nts_srcref) { break; } if (src_count >= QUERY_CONTINUATION_SRC_COUNT) { NSTAT_NOTE_SRC(nstat_query_details_limit, client, src); break; } if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) && (NSTAT_LOCK_WOULD_YIELD())) { // A possibly higher priority thread is waiting // Exit from here and have user level initiate the next fragment NSTAT_NOTE_SRC(nstat_query_details_yield, client, src); break; } } nstat_flush_accumulated_msgs(client); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) { flags = nstat_client_end_query(client, src, partial); } NSTAT_UNLOCK_SHARED(); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_client_send. This way, the error is sent to * user-level. */ if (all_srcs && ENOMEM != result && ENOBUFS != result) { nstat_enqueue_success(req.hdr.context, client, flags); result = 0; } while ((src = TAILQ_FIRST(&dead_list))) { TAILQ_REMOVE(&dead_list, src, nts_client_link); // release src and send notification nstat_client_cleanup_source(client, src); } return result; } static errno_t nstat_client_handle_subscribe_sysinfo( nstat_client *client) { errno_t result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0); if (result != 0) { return result; } NSTAT_LOCK_EXCLUSIVE(); client->ntc_flags |= NSTAT_FLAG_SYSINFO_SUBSCRIBED; NSTAT_UNLOCK_EXCLUSIVE(); return 0; } static errno_t nstat_client_send( kern_ctl_ref kctl, u_int32_t unit, void *uinfo, mbuf_t m, __unused int flags) { nstat_client *client = (nstat_client*)uinfo; struct nstat_msg_hdr *hdr; struct nstat_msg_hdr storage; errno_t result = 0; if (mbuf_pkthdr_len(m) < sizeof(*hdr)) { // Is this the right thing to do? mbuf_freem(m); return EINVAL; } if (mbuf_len(m) >= sizeof(*hdr)) { hdr = mtod(m, struct nstat_msg_hdr *); } else { mbuf_copydata(m, 0, sizeof(storage), &storage); hdr = &storage; } // Legacy clients may not set the length // Those clients are likely not setting the flags either // Fix everything up so old clients continue to work if (hdr->length != mbuf_pkthdr_len(m)) { hdr->flags = 0; assert(mbuf_pkthdr_len(m) <= MAX_NSTAT_MSG_HDR_LENGTH); hdr->length = (u_int16_t)mbuf_pkthdr_len(m); if (hdr == &storage) { mbuf_copyback(m, 0, sizeof(*hdr), hdr, MBUF_DONTWAIT); } } lck_mtx_lock(&client->ntc_user_mtx); /* Prevent misbehaving clients from overlapping calls */ switch (hdr->type) { case NSTAT_MSG_TYPE_ADD_SRC: result = nstat_client_handle_add_request(client, m); break; case NSTAT_MSG_TYPE_ADD_ALL_SRCS: result = nstat_client_handle_add_all(client, m); break; case NSTAT_MSG_TYPE_REM_SRC: result = nstat_client_handle_remove_request(client, m); break; case NSTAT_MSG_TYPE_QUERY_SRC: result = nstat_client_handle_query_request(client, m); break; case NSTAT_MSG_TYPE_GET_SRC_DESC: result = nstat_client_handle_get_src_description(client, m); break; case NSTAT_MSG_TYPE_GET_UPDATE: result = nstat_client_handle_get_update(client, m); break; case NSTAT_MSG_TYPE_GET_DETAILS: result = nstat_client_handle_get_details(client, m); break; case NSTAT_MSG_TYPE_SUBSCRIBE_SYSINFO: result = nstat_client_handle_subscribe_sysinfo(client); break; default: result = EINVAL; break; } if (result != 0) { struct nstat_msg_error err; bzero(&err, sizeof(err)); err.hdr.type = NSTAT_MSG_TYPE_ERROR; err.hdr.length = (u_int16_t)(sizeof(err) + mbuf_pkthdr_len(m)); err.hdr.context = hdr->context; err.error = result; if (mbuf_prepend(&m, sizeof(err), MBUF_DONTWAIT) == 0 && mbuf_copyback(m, 0, sizeof(err), &err, MBUF_DONTWAIT) == 0) { result = ctl_enqueuembuf(kctl, unit, m, CTL_DATA_EOR | CTL_DATA_CRIT); if (result != 0) { mbuf_freem(m); } m = NULL; } if (result != 0) { // Unable to prepend the error to the request - just send the error err.hdr.length = sizeof(err); result = ctl_enqueuedata(kctl, unit, &err, sizeof(err), CTL_DATA_EOR | CTL_DATA_CRIT); if (result != 0) { nstat_stats.nstat_msgerrorfailures += 1; } } nstat_stats.nstat_handle_msg_failures += 1; } lck_mtx_unlock(&client->ntc_user_mtx); if (m) { mbuf_freem(m); } return result; } /* Performs interface matching based on NSTAT_IFNET_IS… filter flags provided by an external caller */ static bool nstat_interface_matches_filter_flag(uint32_t filter_flags, struct ifnet *ifp) { bool result = false; if (ifp) { uint32_t flag_mask = (NSTAT_FILTER_IFNET_FLAGS & ~(NSTAT_IFNET_IS_NON_LOCAL | NSTAT_IFNET_IS_LOCAL)); filter_flags &= flag_mask; uint32_t flags = nstat_ifnet_to_flags(ifp); if (filter_flags & flags) { result = true; } } return result; } static int progress_indicators_for_interface(unsigned int ifindex, uint64_t recentflow_maxduration, uint32_t filter_flags, uint64_t transport_protocol_mask, struct nstat_progress_indicators *indicators) { int error = 0; struct inpcb *inp; uint64_t min_recent_start_time; bool update_tcp_indicators = ((transport_protocol_mask & PR_PROTO_TCP) == PR_PROTO_TCP) || (transport_protocol_mask == 0); bool update_quic_indicators = (transport_protocol_mask & PR_PROTO_QUIC) == PR_PROTO_QUIC; #if SKYWALK struct nstat_tu_shadow *shad; #endif /* SKYWALK */ min_recent_start_time = mach_continuous_time() - recentflow_maxduration; bzero(indicators, sizeof(*indicators)); if (nstat_debug != 0) { /* interface index -1 may be passed in to only match against the filters specified in the flags */ if (ifindex < UINT_MAX) { NSTAT_LOG_DEBUG("for interface index %u with flags %x", ifindex, filter_flags); } else { NSTAT_LOG_DEBUG("for matching interface with flags %x", filter_flags); } } if (update_tcp_indicators) { lck_rw_lock_shared(&tcbinfo.ipi_lock); /* * For progress indicators we don't need to special case TCP to collect time wait connections */ LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) { struct tcpcb *tp = intotcpcb(inp); /* radar://57100452 * The conditional logic implemented below performs an *inclusive* match based on the desired interface index in addition to any filter values. * While the general expectation is that only one criteria normally is used for queries, the capability exists satisfy any eccentric future needs. */ if (tp && inp->inp_state != INPCB_STATE_DEAD && inp->inp_last_outifp && /* matches the given interface index, or against any provided filter flags */ (((inp->inp_last_outifp->if_index == ifindex) || nstat_interface_matches_filter_flag(filter_flags, inp->inp_last_outifp)) && /* perform flow state matching based any provided filter flags */ (((filter_flags & (NSTAT_IFNET_IS_NON_LOCAL | NSTAT_IFNET_IS_LOCAL)) == 0) || ((filter_flags & NSTAT_IFNET_IS_NON_LOCAL) && !(tp->t_flags & TF_LOCAL)) || ((filter_flags & NSTAT_IFNET_IS_LOCAL) && (tp->t_flags & TF_LOCAL))))) { struct tcp_conn_status connstatus; if (nstat_debug != 0) { NSTAT_LOG_DEBUG("*matched non-Skywalk* [filter match: %d]", nstat_interface_matches_filter_flag(filter_flags, inp->inp_last_outifp)); } indicators->np_numflows++; tcp_get_connectivity_status(tp, &connstatus); if (connstatus.write_probe_failed) { indicators->np_write_probe_fails++; } if (connstatus.read_probe_failed) { indicators->np_read_probe_fails++; } if (connstatus.conn_probe_failed) { indicators->np_conn_probe_fails++; } if (inp->inp_start_timestamp > min_recent_start_time) { uint64_t flow_count; indicators->np_recentflows++; flow_count = os_atomic_load(&inp->inp_mstat.ms_total.ts_rxbytes, relaxed); indicators->np_recentflows_rxbytes += flow_count; flow_count = os_atomic_load(&inp->inp_mstat.ms_total.ts_txbytes, relaxed); indicators->np_recentflows_txbytes += flow_count; indicators->np_recentflows_rxooo += tp->t_stat.rxoutoforderbytes; indicators->np_recentflows_rxdup += tp->t_stat.rxduplicatebytes; indicators->np_recentflows_retx += tp->t_stat.txretransmitbytes; if (tp->snd_max - tp->snd_una) { indicators->np_recentflows_unacked++; } } } } lck_rw_done(&tcbinfo.ipi_lock); } #if SKYWALK u_int32_t locality_flags = (filter_flags & (NSTAT_IFNET_IS_LOCAL | NSTAT_IFNET_IS_NON_LOCAL)); u_int32_t flag_mask = (NSTAT_FILTER_IFNET_FLAGS & ~(NSTAT_IFNET_IS_NON_LOCAL | NSTAT_IFNET_IS_LOCAL)); u_int32_t interface_flags = (filter_flags & flag_mask); NSTAT_LOCK_SHARED(); TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { assert(shad->shad_magic == TU_SHADOW_MAGIC); bool consider_shad = false; if (shad->shad_provider == NSTAT_PROVIDER_QUIC_USERLAND) { consider_shad = update_quic_indicators; } else if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) { consider_shad = update_tcp_indicators; } if (consider_shad) { u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE; nstat_progress_digest digest; bzero(&digest, sizeof(digest)); // fetch ifflags and digest from necp_client bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, &digest, NULL, NULL, NULL); error = (result)? 0 : EIO; if (error) { NSTAT_LOG_ERROR("nstat get ifflags and progressdigest returned %d", error); continue; } if (ifflags & (NSTAT_IFNET_FLOWSWITCH_VALUE_UNOBTAINABLE | NSTAT_IFNET_ROUTE_VALUE_UNOBTAINABLE)) { NSTAT_LOG_ERROR("nstat get ifflags and progressdigest resulted in Skywalk error, ifflags=%x", ifflags); continue; } if ((digest.ifindex == (u_int32_t)ifindex) || ((ifindex == UINT_MAX) && ((ifflags & interface_flags) != 0))) { // a match on either a requested interface index or on interface type if ((locality_flags != 0) && ((locality_flags & ifflags) == 0)) { continue; } if (nstat_debug != 0) { NSTAT_LOG_DEBUG("*matched Skywalk* [ifindex=%u digest.ifindex=%u filter_flags=%x ifflags=%x]", ifindex, digest.ifindex, filter_flags, ifflags); } indicators->np_numflows++; if (digest.connstatus.write_probe_failed) { indicators->np_write_probe_fails++; } if (digest.connstatus.read_probe_failed) { indicators->np_read_probe_fails++; } if (digest.connstatus.conn_probe_failed) { indicators->np_conn_probe_fails++; } if (shad->shad_start_timestamp > min_recent_start_time) { indicators->np_recentflows++; indicators->np_recentflows_rxbytes += digest.rxbytes; indicators->np_recentflows_txbytes += digest.txbytes; indicators->np_recentflows_rxooo += digest.rxoutoforderbytes; indicators->np_recentflows_rxdup += digest.rxduplicatebytes; indicators->np_recentflows_retx += digest.txretransmit; if (digest.txunacked) { indicators->np_recentflows_unacked++; } } } } } NSTAT_UNLOCK_SHARED(); #endif /* SKYWALK */ return error; } static int tcp_progress_probe_enable_for_interface(unsigned int ifindex, uint32_t filter_flags, uint32_t enable_flags) { int error = 0; struct ifnet *ifp; if (nstat_debug != 0) { NSTAT_LOG_DEBUG("for interface index %u with flags %d", ifindex, filter_flags); } ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { if ((ifp->if_index == ifindex) || nstat_interface_matches_filter_flag(filter_flags, ifp)) { if (nstat_debug != 0) { NSTAT_LOG_DEBUG("*matched* interface index %d, enable: %d", ifp->if_index, enable_flags); } error = if_probe_connectivity(ifp, enable_flags); if (error) { NSTAT_LOG_ERROR("(%d) - nstat set tcp probe %d for interface index %d", error, enable_flags, ifp->if_index); } } } ifnet_head_done(); return error; } static int ntstat_progress_indicators(struct sysctl_req *req) { struct nstat_progress_indicators indicators = {}; int error = 0; struct nstat_progress_req requested; if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { return EACCES; } if (req->newptr == USER_ADDR_NULL) { return EINVAL; } if (req->newlen < sizeof(requested)) { return EINVAL; } error = SYSCTL_IN(req, &requested, sizeof(requested)); if (error != 0) { return error; } error = progress_indicators_for_interface((unsigned int)requested.np_ifindex, requested.np_recentflow_maxduration, (uint32_t)requested.np_filter_flags, requested.np_transport_protocol_mask, &indicators); if (error != 0) { return error; } error = SYSCTL_OUT(req, &indicators, sizeof(indicators)); return error; } __private_extern__ int ntstat_tcp_progress_enable(struct sysctl_req *req) { int error = 0; struct tcpprobereq requested; if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { return EACCES; } if (req->newptr == USER_ADDR_NULL) { return EINVAL; } if (req->newlen < sizeof(requested)) { return EINVAL; } error = SYSCTL_IN(req, &requested, sizeof(requested)); if (error != 0) { return error; } error = tcp_progress_probe_enable_for_interface((unsigned int)requested.ifindex, (uint32_t)requested.filter_flags, (uint32_t)requested.enable); return error; } #if SKYWALK #pragma mark -- netstat support for user level providers -- static int nstat_gather_flow_data(nstat_provider_id_t provider, nstat_flow_data *__counted_by(n)flow_data_start, int n) { nstat_flow_data * flow_data = flow_data_start; struct nstat_tu_shadow *shad; int prepared = 0; errno_t err; NSTAT_ASSERT_LOCKED(); TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { assert(shad->shad_magic == TU_SHADOW_MAGIC); if (shad->shad_provider == provider) { if (prepared >= n) { break; } err = nstat_userland_tu_copy_descriptor((nstat_provider_cookie_t) shad, &flow_data->flow_descriptor, sizeof(flow_data->flow_descriptor)); if (err != 0) { NSTAT_LOG_ERROR("nstat_userland_tu_copy_descriptor returned %d", err); } err = nstat_userland_tu_counts((nstat_provider_cookie_t) shad, &flow_data->counts, NULL); if (err != 0) { NSTAT_LOG_ERROR("nstat_userland_tu_counts returned %d", err); } flow_data++; prepared++; } } return prepared; } static void nstat_userland_to_xinpcb_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xinpcb_n *xinp) { xinp->xi_len = sizeof(struct xinpcb_n); xinp->xi_kind = XSO_INPCB; if (provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor; struct sockaddr_in *sa = &desc->local.v4; if (sa->sin_family == AF_INET) { xinp->inp_vflag = INP_IPV4; xinp->inp_laddr = desc->local.v4.sin_addr; xinp->inp_lport = desc->local.v4.sin_port; xinp->inp_faddr = desc->remote.v4.sin_addr; xinp->inp_fport = desc->remote.v4.sin_port; } else if (sa->sin_family == AF_INET6) { xinp->inp_vflag = INP_IPV6; xinp->in6p_laddr = desc->local.v6.sin6_addr; xinp->in6p_lport = desc->local.v6.sin6_port; xinp->in6p_faddr = desc->remote.v6.sin6_addr; xinp->in6p_fport = desc->remote.v6.sin6_port; } } else if (provider == NSTAT_PROVIDER_UDP_USERLAND) { nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor; struct sockaddr_in *sa = &desc->local.v4; if (sa->sin_family == AF_INET) { xinp->inp_vflag = INP_IPV4; xinp->inp_laddr = desc->local.v4.sin_addr; xinp->inp_lport = desc->local.v4.sin_port; xinp->inp_faddr = desc->remote.v4.sin_addr; xinp->inp_fport = desc->remote.v4.sin_port; } else if (sa->sin_family == AF_INET6) { xinp->inp_vflag = INP_IPV6; xinp->in6p_laddr = desc->local.v6.sin6_addr; xinp->in6p_lport = desc->local.v6.sin6_port; xinp->in6p_faddr = desc->remote.v6.sin6_addr; xinp->in6p_fport = desc->remote.v6.sin6_port; } } } static void nstat_userland_to_xsocket_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsocket_n *xso) { xso->xso_len = sizeof(struct xsocket_n); xso->xso_kind = XSO_SOCKET; if (provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor; xso->xso_protocol = IPPROTO_TCP; xso->so_e_pid = desc->epid; xso->so_last_pid = desc->pid; } else { nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor; xso->xso_protocol = IPPROTO_UDP; xso->so_e_pid = desc->epid; xso->so_last_pid = desc->pid; } } static void nstat_userland_to_rcv_xsockbuf_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsockbuf_n *xsbrcv) { xsbrcv->xsb_len = sizeof(struct xsockbuf_n); xsbrcv->xsb_kind = XSO_RCVBUF; if (provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor; xsbrcv->sb_hiwat = desc->rcvbufsize; xsbrcv->sb_cc = desc->rcvbufused; } else { nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor; xsbrcv->sb_hiwat = desc->rcvbufsize; xsbrcv->sb_cc = desc->rcvbufused; } } static void nstat_userland_to_snd_xsockbuf_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsockbuf_n *xsbsnd) { xsbsnd->xsb_len = sizeof(struct xsockbuf_n); xsbsnd->xsb_kind = XSO_SNDBUF; if (provider == NSTAT_PROVIDER_TCP_USERLAND) { nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor; xsbsnd->sb_hiwat = desc->sndbufsize; xsbsnd->sb_cc = desc->sndbufused; } else { } } static void nstat_userland_to_xsockstat_n(nstat_flow_data *flow_data, struct xsockstat_n *xst) { xst->xst_len = sizeof(struct xsockstat_n); xst->xst_kind = XSO_STATS; // The kernel version supports an array of counts, here we only support one and map to first entry xst->xst_tc_stats[0].rxpackets = flow_data->counts.nstat_rxpackets; xst->xst_tc_stats[0].rxbytes = flow_data->counts.nstat_rxbytes; xst->xst_tc_stats[0].txpackets = flow_data->counts.nstat_txpackets; xst->xst_tc_stats[0].txbytes = flow_data->counts.nstat_txbytes; } static void nstat_userland_to_xtcpcb_n(nstat_flow_data *flow_data, struct xtcpcb_n *xt) { nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor; xt->xt_len = sizeof(struct xtcpcb_n); xt->xt_kind = XSO_TCPCB; xt->t_state = desc->state; xt->snd_wnd = desc->txwindow; xt->snd_cwnd = desc->txcwindow; } __private_extern__ int ntstat_userland_count(short proto) { int n = 0; if (proto == IPPROTO_TCP) { n = nstat_userland_tcp_shadows; } else if (proto == IPPROTO_UDP) { n = nstat_userland_udp_shadows; } return n; } __private_extern__ int nstat_userland_get_snapshot(short proto, void *__sized_by(*snapshot_size) * snapshotp, size_t *snapshot_size, int *countp) { int error = 0; int n = 0; size_t data_size = 0; nstat_provider_id_t provider; nstat_flow_data *flow_data = NULL; NSTAT_LOCK_SHARED(); if (proto == IPPROTO_TCP) { n = nstat_userland_tcp_shadows; provider = NSTAT_PROVIDER_TCP_USERLAND; } else if (proto == IPPROTO_UDP) { n = nstat_userland_udp_shadows; provider = NSTAT_PROVIDER_UDP_USERLAND; } if (n == 0) { goto done; } data_size = n * sizeof(*flow_data); flow_data = (nstat_flow_data *) kalloc_data(data_size, Z_WAITOK | Z_ZERO); if (flow_data) { n = nstat_gather_flow_data(provider, flow_data, n); } else { error = ENOMEM; } done: NSTAT_UNLOCK_SHARED(); *snapshotp = flow_data; *snapshot_size = data_size; *countp = n; return error; } // nstat_userland_list_snapshot() does most of the work for a sysctl that uses a return format // as per get_pcblist_n() even though the vast majority of fields are unused. // Additional items are required in the sysctl output before and after the data added // by this function. __private_extern__ int nstat_userland_list_snapshot(short proto, struct sysctl_req *req, void *__sized_by(n * sizeof(nstat_flow_data))userlandsnapshot, int n) { int error = 0; int i; nstat_provider_id_t provider; void *buf = NULL; nstat_flow_data *flow_data, *flow_data_array = NULL; size_t item_size = ROUNDUP64(sizeof(struct xinpcb_n)) + ROUNDUP64(sizeof(struct xsocket_n)) + 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) + ROUNDUP64(sizeof(struct xsockstat_n)); if ((n == 0) || (userlandsnapshot == NULL)) { goto done; } if (proto == IPPROTO_TCP) { item_size += ROUNDUP64(sizeof(struct xtcpcb_n)); provider = NSTAT_PROVIDER_TCP_USERLAND; } else if (proto == IPPROTO_UDP) { provider = NSTAT_PROVIDER_UDP_USERLAND; } else { error = EINVAL; goto done; } buf = (void *) kalloc_data(item_size, Z_WAITOK); if (buf) { struct xinpcb_n *xi = (struct xinpcb_n *)buf; struct xsocket_n *xso = (struct xsocket_n *) ADVANCE64(xi, sizeof(*xi)); struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *) ADVANCE64(xso, sizeof(*xso)); struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *) ADVANCE64(xsbrcv, sizeof(*xsbrcv)); struct xsockstat_n *xsostats = (struct xsockstat_n *) ADVANCE64(xsbsnd, sizeof(*xsbsnd)); struct xtcpcb_n *xt = (struct xtcpcb_n *) ADVANCE64(xsostats, sizeof(*xsostats)); flow_data_array = (nstat_flow_data *)userlandsnapshot; for (i = 0; i < n; i++) { flow_data = &flow_data_array[i]; bzero(buf, item_size); nstat_userland_to_xinpcb_n(provider, flow_data, xi); nstat_userland_to_xsocket_n(provider, flow_data, xso); nstat_userland_to_rcv_xsockbuf_n(provider, flow_data, xsbrcv); nstat_userland_to_snd_xsockbuf_n(provider, flow_data, xsbsnd); nstat_userland_to_xsockstat_n(flow_data, xsostats); if (proto == IPPROTO_TCP) { nstat_userland_to_xtcpcb_n(flow_data, xt); } error = SYSCTL_OUT(req, buf, item_size); if (error) { break; } } kfree_data(buf, item_size); } else { error = ENOMEM; } done: return error; } __private_extern__ void nstat_userland_release_snapshot(void *snapshot, int nuserland) { if (snapshot != NULL) { kfree_data(snapshot, nuserland * sizeof(nstat_flow_data)); } } #if NTSTAT_SUPPORTS_STANDALONE_SYSCTL __private_extern__ int ntstat_userland_list_n(short proto, struct sysctl_req *req) { int error = 0; int n; struct xinpgen xig; void *snapshot = NULL; size_t item_size = ROUNDUP64(sizeof(struct xinpcb_n)) + ROUNDUP64(sizeof(struct xsocket_n)) + 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) + ROUNDUP64(sizeof(struct xsockstat_n)); if (proto == IPPROTO_TCP) { item_size += ROUNDUP64(sizeof(struct xtcpcb_n)); } if (req->oldptr == USER_ADDR_NULL) { n = ntstat_userland_count(proto); req->oldidx = 2 * (sizeof(xig)) + (n + 1 + n / 8) * item_size; goto done; } if (req->newptr != USER_ADDR_NULL) { error = EPERM; goto done; } error = nstat_userland_get_snapshot(proto, &snapshot, &n); if (error) { goto done; } bzero(&xig, sizeof(xig)); xig.xig_len = sizeof(xig); xig.xig_gen = 0; xig.xig_sogen = 0; xig.xig_count = n; error = SYSCTL_OUT(req, &xig, sizeof(xig)); if (error) { goto done; } /* * We are done if there are no flows */ if (n == 0) { goto done; } error = nstat_userland_list_snapshot(proto, req, snapshot, n); if (!error) { /* * Give the user an updated idea of our state, * which is unchanged */ error = SYSCTL_OUT(req, &xig, sizeof(xig)); } done: nstat_userland_release_snapshot(snapshot, n); return error; } #endif /* NTSTAT_SUPPORTS_STANDALONE_SYSCTL */ #endif /* SKYWALK */