1 /*
2 * Copyright (c) 2010-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/param.h>
30 #include <sys/types.h>
31 #include <sys/kpi_mbuf.h>
32 #include <sys/socket.h>
33 #include <sys/kern_control.h>
34 #include <sys/mcache.h>
35 #include <sys/socketvar.h>
36 #include <sys/sysctl.h>
37 #include <sys/queue.h>
38 #include <sys/priv.h>
39 #include <sys/protosw.h>
40 #include <sys/persona.h>
41 #include <sys/random.h>
42 #include <kern/clock.h>
43 #include <kern/debug.h>
44
45 #include <libkern/libkern.h>
46 #include <libkern/OSAtomic.h>
47 #include <libkern/locks.h>
48
49 #include <net/if.h>
50 #include <net/if_var.h>
51 #include <net/if_types.h>
52 #include <net/route.h>
53 #include <net/route_private.h>
54 #include <net/dlil.h>
55
56 // These includes appear in ntstat.h but we include them here first so they won't trigger
57 // any clang diagnostic errors.
58 #include <netinet/in.h>
59 #include <netinet/in_stat.h>
60 #include <netinet/tcp.h>
61
62 #pragma clang diagnostic push
63 #pragma clang diagnostic error "-Wpadded"
64 #pragma clang diagnostic error "-Wpacked"
65 // This header defines structures shared with user space, so we need to ensure there is
66 // no compiler inserted padding in case the user space process isn't using the same
67 // architecture as the kernel (example: i386 process with x86_64 kernel).
68 #include <net/ntstat.h>
69 #pragma clang diagnostic pop
70
71 #include <netinet/ip_var.h>
72 #include <netinet/in_pcb.h>
73 #include <netinet/in_var.h>
74 #include <netinet/tcp_var.h>
75 #include <netinet/tcp_fsm.h>
76 #include <netinet/tcp_cc.h>
77 #include <netinet/udp.h>
78 #include <netinet/udp_var.h>
79 #include <netinet6/in6_pcb.h>
80 #include <netinet6/in6_var.h>
81
82 #include <net/sockaddr_utils.h>
83
84 __private_extern__ int nstat_collect = 1;
85
86 #define NSTAT_TRACE_ENABLED 0
87 #define NSTAT_FUZZ_TIMING 0
88
89
90 #if (DEBUG || DEVELOPMENT)
91 SYSCTL_INT(_net, OID_AUTO, statistics, CTLFLAG_RW | CTLFLAG_LOCKED,
92 &nstat_collect, 0, "Collect detailed statistics");
93 #endif /* (DEBUG || DEVELOPMENT) */
94
95 #if !XNU_TARGET_OS_OSX
96 static int nstat_privcheck = 1;
97 #else /* XNU_TARGET_OS_OSX */
98 static int nstat_privcheck = 0;
99 #endif /* XNU_TARGET_OS_OSX */
100 SYSCTL_INT(_net, OID_AUTO, statistics_privcheck, CTLFLAG_RW | CTLFLAG_LOCKED,
101 &nstat_privcheck, 0, "Entitlement check");
102
103 SYSCTL_NODE(_net, OID_AUTO, stats,
104 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "network statistics");
105
106 static int nstat_debug = 0;
107 SYSCTL_INT(_net_stats, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
108 &nstat_debug, 0, "");
109
110 static int nstat_debug_pid = 0; // Only log socket level debug for specified pid
111 SYSCTL_INT(_net_stats, OID_AUTO, debug_pid, CTLFLAG_RW | CTLFLAG_LOCKED,
112 &nstat_debug_pid, 0, "");
113
114 static int nstat_sendspace = 2048;
115 SYSCTL_INT(_net_stats, OID_AUTO, sendspace, CTLFLAG_RW | CTLFLAG_LOCKED,
116 &nstat_sendspace, 0, "");
117
118 static int nstat_recvspace = 8192;
119 SYSCTL_INT(_net_stats, OID_AUTO, recvspace, CTLFLAG_RW | CTLFLAG_LOCKED,
120 &nstat_recvspace, 0, "");
121
122 static struct nstat_stats nstat_stats;
123 SYSCTL_STRUCT(_net_stats, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_LOCKED,
124 &nstat_stats, nstat_stats, "");
125
126 static u_int32_t nstat_lim_interval = 30 * 60; /* Report interval, seconds */
127 static u_int32_t nstat_lim_min_tx_pkts = 100;
128 static u_int32_t nstat_lim_min_rx_pkts = 100;
129 #if (DEBUG || DEVELOPMENT)
130 SYSCTL_INT(_net_stats, OID_AUTO, lim_report_interval,
131 CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_interval, 0,
132 "Low internet stat report interval");
133
134 SYSCTL_INT(_net_stats, OID_AUTO, lim_min_tx_pkts,
135 CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_tx_pkts, 0,
136 "Low Internet, min transmit packets threshold");
137
138 SYSCTL_INT(_net_stats, OID_AUTO, lim_min_rx_pkts,
139 CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_rx_pkts, 0,
140 "Low Internet, min receive packets threshold");
141 #endif /* DEBUG || DEVELOPMENT */
142
143 static int ntstat_progress_indicators(struct sysctl_req *);
144 static struct net_api_stats net_api_stats_before;
145 static u_int64_t net_api_stats_last_report_time;
146 #define NET_API_STATS_REPORT_INTERVAL (12 * 60 * 60) /* 12 hours, in seconds */
147 static u_int32_t net_api_stats_report_interval = NET_API_STATS_REPORT_INTERVAL;
148
149 #if (DEBUG || DEVELOPMENT)
150 SYSCTL_UINT(_net_stats, OID_AUTO, api_report_interval,
151 CTLFLAG_RW | CTLFLAG_LOCKED, &net_api_stats_report_interval, 0, "");
152 #endif /* DEBUG || DEVELOPMENT */
153
154 static os_log_t nstat_log_handle = NULL;
155 #define _NSTAT_LOG(handle, type, fmt, ...) do { \
156 os_log_with_type(handle, type, "%s - " fmt, __func__, ##__VA_ARGS__); \
157 } while (0)
158
159 #define NSTAT_LOG(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_DEFAULT, fmt, ##__VA_ARGS__)
160 #define NSTAT_LOG_DEBUG(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_DEBUG, fmt, ##__VA_ARGS__)
161 #define NSTAT_LOG_INFO(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_INFO, fmt, ##__VA_ARGS__)
162 #define NSTAT_LOG_ERROR(fmt, ...) _NSTAT_LOG(nstat_log_handle, OS_LOG_TYPE_ERROR, fmt, ##__VA_ARGS__)
163
164 #define NSTAT_DEBUG_SOCKET_PID_MATCHED(so) \
165 (so && (nstat_debug_pid == (so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid)))
166
167 #define NSTAT_DEBUG_SOCKET_ON(so) \
168 ((nstat_debug && (!nstat_debug_pid || NSTAT_DEBUG_SOCKET_PID_MATCHED(so))) ? nstat_debug : 0)
169
170 #define NSTAT_DEBUG_SOCKET_LOG(so, fmt, ...) \
171 if (NSTAT_DEBUG_SOCKET_ON(so)) { \
172 NSTAT_LOG_DEBUG("NSTAT_DEBUG_SOCKET <pid %d>: " fmt, (so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid), ##__VA_ARGS__); \
173 }
174
175 enum{
176 NSTAT_FLAG_CLEANUP = (1 << 0),
177 NSTAT_FLAG_REQCOUNTS = (1 << 1),
178 NSTAT_FLAG_SUPPORTS_UPDATES = (1 << 2),
179 NSTAT_FLAG_SYSINFO_SUBSCRIBED = (1 << 3),
180 };
181
182 static int
183 progress_indicators SYSCTL_HANDLER_ARGS
184 {
185 return ntstat_progress_indicators(req);
186 }
187
188 SYSCTL_PROC(_net_stats, OID_AUTO, progress,
189 CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, 0, 0,
190 progress_indicators, "S", "Various items that indicate the current state of progress on the link");
191
192 #if !XNU_TARGET_OS_OSX
193 #define QUERY_CONTINUATION_SRC_COUNT 50
194 #else /* XNU_TARGET_OS_OSX */
195 #define QUERY_CONTINUATION_SRC_COUNT 100
196 #endif /* XNU_TARGET_OS_OSX */
197 #define QUERY_CONTINUATION_MIN_SRC_COUNT 5
198
199 #ifndef ROUNDUP64
200 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
201 #endif
202
203 #ifndef ADVANCE64
204 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
205 #endif
206
207 typedef TAILQ_HEAD(, nstat_src) tailq_head_nstat_src;
208 typedef TAILQ_ENTRY(nstat_src) tailq_entry_nstat_src;
209
210 typedef TAILQ_HEAD(, nstat_tu_shadow) tailq_head_tu_shadow;
211 typedef TAILQ_ENTRY(nstat_tu_shadow) tailq_entry_tu_shadow;
212
213 typedef TAILQ_HEAD(, nstat_generic_shadow) tailq_head_generic_shadow;
214 typedef TAILQ_ENTRY(nstat_generic_shadow) tailq_entry_generic_shadow;
215
216 typedef TAILQ_HEAD(, nstat_procdetails) tailq_head_procdetails;
217 typedef TAILQ_ENTRY(nstat_procdetails) tailq_entry_procdetails;
218
219
220 typedef void *nstat_provider_cookie_t;
221
222 struct nstat_procdetails {
223 tailq_entry_procdetails pdet_link;
224 int pdet_pid;
225 u_int64_t pdet_upid;
226 char pdet_procname[64];
227 uuid_t pdet_uuid;
228 u_int32_t pdet_refcnt;
229 u_int32_t pdet_magic;
230 };
231
232 typedef struct nstat_provider_filter {
233 u_int64_t npf_flags;
234 u_int64_t npf_events;
235 u_int64_t npf_extensions;
236 pid_t npf_pid;
237 uuid_t npf_uuid;
238 } nstat_provider_filter;
239
240
241 struct nstat_global_counts {
242 uint64_t nstat_global_client_current; // current number of clients overall
243 uint64_t nstat_global_client_max; // max number of clients overall
244 uint64_t nstat_global_client_allocs; // total number of clients allocated
245
246 uint64_t nstat_global_src_current; // current number of srcs overall
247 uint64_t nstat_global_src_max; // max number of srcs overall
248 uint64_t nstat_global_src_allocs; // total number of sources allocated
249 uint64_t nstat_global_src_idlecheck_gone;// total number of sources discovered "gone" in idle check
250
251 uint64_t nstat_global_tucookie_current; // current number of tucookies overall
252 uint64_t nstat_global_tucookie_max; // max number of tucookies overall
253 uint64_t nstat_global_tucookie_allocs; // total number of tucookies allocated
254 // Details for tucookie lifecycle
255 uint64_t nstat_global_tucookie_skip_dead; // When adding a watcher, pcb with "dead" state skipped over
256 uint64_t nstat_global_tucookie_skip_stopusing; // When adding a watcher, pcb marked as stop using
257 uint64_t nstat_global_tucookie_alloc_fail; // Allocation failure for a tucookie
258
259 uint64_t nstat_global_tu_shad_current; // current number of nstat_tu_shadow objects overall
260 uint64_t nstat_global_tu_shad_max; // max number of tu_shadows overall
261 uint64_t nstat_global_tu_shad_allocs; // total number of tu_shadows allocated
262
263 uint64_t nstat_global_gshad_current; // current number of generic shadow objects overall
264 uint64_t nstat_global_gshad_max; // max number of srcs overall
265 uint64_t nstat_global_gshad_allocs; // total number of sources allocated
266
267 uint64_t nstat_global_procdetails_current;// current number of procdetails objects overall
268 uint64_t nstat_global_procdetails_max; // max number of procdetails overall
269 uint64_t nstat_global_procdetails_allocs;// total number of procdetails allocated
270 };
271
272 struct nstat_metrics {
273 uint32_t nstat_src_current; // current number of srcs for client
274 uint32_t nstat_src_max; // max number of srcs for client
275 uint32_t nstat_first_uint32_count; // Subsequent fields must be uint32_t values that, if kept per-client,
276 // should simply added to the global counts when the client exit
277
278 // Tracking client requests
279 uint32_t nstat_query_request_all; // Client requests for all counts
280 uint32_t nstat_query_request_one; // Client request for counts on a single source
281 uint32_t nstat_query_description_all; // Client requests for all descriptors
282 uint32_t nstat_query_description_one; // Client requests for descriptor on a single source
283 uint32_t nstat_query_update_all; // Client requests for all updates
284 uint32_t nstat_query_update_one; // Client requests for update on a single source
285 uint32_t nstat_remove_src_found; // Client request to remove a source which is still in existence
286 uint32_t nstat_remove_src_missed; // Client request to remove a source which is no longer there
287
288 // Details for nstat_query_request all/one
289 uint32_t nstat_query_request_nobuf; // No buffers for message send
290 uint32_t nstat_query_request_upgrade; // Successful lock upgrade to handle "gone" source
291 uint32_t nstat_query_request_noupgrade; // Unsuccessful lock upgrade to handle "gone" source
292 uint32_t nstat_query_request_nodesc; // Can't send a descriptor for "gone" source
293 uint32_t nstat_query_request_yield; // Client yields lock due to possibly higher priority processing
294 uint32_t nstat_query_request_limit; // Client requests for all counts
295
296 // Details for nstat_query_description all/one
297 uint32_t nstat_query_description_nobuf; // No buffers for message send
298 uint32_t nstat_query_description_yield; // Client yields lock due to possibly higher priority processing
299 uint32_t nstat_query_description_limit; // Client requests for all counts
300
301 // Details for nstat_query_update all/one
302 uint32_t nstat_query_update_nobuf; // No buffers for message send
303 uint32_t nstat_query_update_upgrade; // Successful lock upgrade to handle "gone" source
304 uint32_t nstat_query_update_noupgrade; // Unsuccessful lock upgrade to handle "gone" source
305 uint32_t nstat_query_update_nodesc; // Can't send a descriptor for "gone" source
306 uint32_t nstat_query_update_yield; // Client yields lock due to possibly higher priority processing
307 uint32_t nstat_query_update_limit; // Client requests for all counts
308
309 // Details for adding a source
310 uint32_t nstat_src_add_success; // successful src_add
311 uint32_t nstat_src_add_no_buf; // fail to get buffer for initial src-added
312 uint32_t nstat_src_add_no_src_mem; // fail to get memory for nstat_src structure
313 uint32_t nstat_src_add_send_err; // fail to send initial src-added
314 uint32_t nstat_src_add_while_cleanup; // fail to add because client is in clean up state
315
316 uint32_t nstat_src_gone_idlecheck; // src gone noted during periodic idle check
317
318 uint32_t nstat_last_uint32_count; // Must be the last uint32_t count in the structure
319 uint32_t nstat_stats_pad;
320 };
321
322 #define NUM_NSTAT_METRICS_UINT32_COUNTS ((__builtin_offsetof(struct nstat_metrics, nstat_last_uint32_count) - \
323 __builtin_offsetof(struct nstat_metrics, nstat_first_uint32_count)) / sizeof(uint32_t))
324
325
326 typedef struct nstat_trace_entry {
327 u_int32_t nte_seqno;
328 u_int32_t nte_event;
329 u_int64_t nte_qualifier;
330 } nstat_trace_entry;
331
332 #define NSTAT_TRACE_ENTRIES_PER_CLIENT (16 * 1024)
333
334 typedef struct nstat_cyclic_trace {
335 uint32_t ntt_next_trace_id;
336 int32_t ntt_pad;
337 nstat_trace_entry ntt_entry[NSTAT_TRACE_ENTRIES_PER_CLIENT];
338 } nstat_cyclic_trace;
339
340
341 typedef struct nstat_client {
342 struct nstat_client *ntc_next;
343 /* A bitmask to indicate whether a provider ever done NSTAT_MSG_TYPE_ADD_ALL_SRCS */
344 u_int32_t ntc_watching;
345 /* A bitmask to indicate whether a provider ever done NSTAT_MSG_TYPE_ADD_SRC */
346 u_int32_t ntc_added_src;
347 decl_lck_mtx_data(, ntc_user_mtx); // Mutual exclusion for user level requests on this ntc_client
348 kern_ctl_ref ntc_kctl;
349 u_int32_t ntc_unit;
350 nstat_src_ref_t ntc_next_srcref;
351 tailq_head_nstat_src ntc_src_queue;
352 mbuf_t ntc_accumulated;
353 u_int32_t ntc_flags;
354 nstat_provider_filter ntc_provider_filters[NSTAT_PROVIDER_COUNT];
355 /* state maintained for partial query requests */
356 u_int64_t ntc_context;
357 u_int64_t ntc_seq;
358 /* For ease of debugging with lldb macros */
359 struct nstat_procdetails *ntc_procdetails;
360 struct nstat_metrics ntc_metrics;
361 #if NSTAT_TRACE_ENABLED
362 nstat_cyclic_trace *ntc_trace;
363 #endif
364 } nstat_client;
365
366 typedef struct nstat_provider {
367 struct nstat_provider *next;
368 nstat_provider_id_t nstat_provider_id;
369 size_t nstat_descriptor_length;
370 errno_t (*nstat_lookup)(const void *__sized_by (length)data, u_int32_t length, nstat_provider_cookie_t *out_cookie);
371 int (*nstat_gone)(nstat_provider_cookie_t cookie);
372 errno_t (*nstat_counts)(nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone);
373 errno_t (*nstat_watcher_add)(nstat_client *client, nstat_msg_add_all_srcs *req);
374 void (*nstat_watcher_remove)(nstat_client *client);
375 errno_t (*nstat_copy_descriptor)(nstat_provider_cookie_t cookie, void *__sized_by (len)data, size_t len);
376 void (*nstat_release)(nstat_provider_cookie_t cookie, boolean_t locked);
377 bool (*nstat_reporting_allowed)(nstat_provider_cookie_t cookie, nstat_provider_filter *filter, u_int64_t suppression_flags);
378 bool (*nstat_cookie_equal)(nstat_provider_cookie_t cookie1, nstat_provider_cookie_t cookie2);
379 size_t (*nstat_copy_extension)(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len);
380 } nstat_provider;
381
382 typedef struct nstat_src {
383 tailq_entry_nstat_src nts_client_link; // All sources for the nstat_client, for iterating over.
384 nstat_client *nts_client; // The nstat_client that this is a source for
385 nstat_src_ref_t nts_srcref; // The reference quoted in any messages
386 nstat_provider *nts_provider; // The "provider" for the source, e.g. for kernel TCP sockets
387 nstat_provider_cookie_t nts_cookie; // Provider-specific futher information,
388 uint32_t nts_filter;
389 bool nts_reported; // At least one update/counts/desc message has been sent
390 uint64_t nts_seq; // Keeping track when getting partial poll results
391 } nstat_src;
392
393 // The merge structures are intended to give a global picture of what may be asked for by the current set of clients
394 // This is to avoid taking locks to check them all individually
395 typedef struct nstat_merged_provider_filter {
396 u_int64_t mf_events; // So far we only merge the events portion of any filters
397 } nstat_merged_provider_filter;
398
399 typedef struct nstat_merged_provider_filters {
400 nstat_merged_provider_filter mpf_filters[NSTAT_PROVIDER_COUNT];
401 } nstat_merged_provider_filters;
402
403 static errno_t nstat_client_send_counts(nstat_client *client, nstat_src *src, unsigned long long context, u_int16_t hdr_flags, int *gone);
404 static int nstat_client_send_description(nstat_client *client, nstat_src *src, u_int64_t context, u_int16_t hdr_flags);
405 static int nstat_client_send_update(nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone);
406 static errno_t nstat_client_send_removed(nstat_client *client, nstat_src *src, u_int16_t hdr_flags);
407 static errno_t nstat_client_send_goodbye(nstat_client *client, nstat_src *src);
408 static void nstat_client_cleanup_source(nstat_client *client, nstat_src *src, boolean_t);
409 static bool nstat_client_reporting_allowed(nstat_client *client, nstat_src *src, u_int64_t suppression_flags);
410 static boolean_t nstat_client_begin_query(nstat_client *client, const nstat_msg_hdr *hdrp);
411 static u_int16_t nstat_client_end_query(nstat_client *client, nstat_src *last_src, boolean_t partial);
412 static void nstat_ifnet_report_ecn_stats(void);
413 static void nstat_ifnet_report_lim_stats(void);
414 static void nstat_net_api_report_stats(void);
415 static errno_t nstat_set_provider_filter( nstat_client *client, nstat_msg_add_all_srcs *req);
416 static errno_t nstat_client_send_event(nstat_client *client, nstat_src *src, u_int64_t event);
417
418 static u_int32_t nstat_udp_watchers = 0;
419 static u_int32_t nstat_tcp_watchers = 0;
420 static nstat_merged_provider_filters merged_filters = {};
421
422 static nstat_client *nstat_clients = NULL;
423 static uint64_t nstat_idle_time = 0;
424
425 #if NSTAT_FUZZ_TIMING
426 static uint32_t nstat_random_delay_insert_modulo = 5000;
427 static uint32_t nstat_max_nsec_delay = (NSEC_PER_SEC / 1000);
428 #endif // NSTAT_FUZZ_TIMING
429
430 static struct nstat_metrics nstat_metrics;
431 static struct nstat_global_counts nstat_global_counts;
432
433 // For lldb macro usage
434 static __unused const size_t nstat_trace_entries_per_client = NSTAT_TRACE_ENTRIES_PER_CLIENT;
435
436 #if NSTAT_TRACE_ENABLED
437 static nstat_cyclic_trace nstat_global_trace;
438 #endif
439
440 #if NSTAT_TRACE_ENABLED
441 #define NSTAT_TRACE(field, client, qual) \
442 { \
443 nstat_trace(offsetof(struct nstat_metrics, field), client, (uint64_t)qual); \
444 }
445 #else // NSTAT_TRACE_ENABLED
446 #define NSTAT_TRACE(field, client, qual)
447 #endif
448
449 #if NSTAT_FUZZ_TIMING
450 #define NSTAT_RANDOM_DELAY(client) \
451 { \
452 nstat_random_delay(client); \
453 }
454 #else
455 #define NSTAT_RANDOM_DELAY(client)
456 #endif
457
458 #define NSTAT_NOTE_QUAL(field, client, qual) \
459 { \
460 NSTAT_TRACE(field, client, qual); \
461 NSTAT_RANDOM_DELAY(client); \
462 client->ntc_metrics.field++; \
463 }
464
465 #define NSTAT_NOTE_SRC(field, client, src) \
466 { \
467 NSTAT_TRACE(field, client, (src != NULL)? src->nts_srcref: 0); \
468 NSTAT_RANDOM_DELAY(client); \
469 client->ntc_metrics.field++; \
470 }
471
472 #define NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(_currentfield, _maxfield) \
473 { \
474 uint64_t _prev_count = os_atomic_inc_orig(&nstat_global_counts._currentfield, relaxed); \
475 if (_prev_count >= os_atomic_load(&nstat_global_counts._maxfield, relaxed)) { \
476 os_atomic_store(&nstat_global_counts._maxfield, _prev_count + 1, relaxed); \
477 } \
478 }
479
480 #define NSTAT_GLOBAL_COUNT_INCREMENT(_field) \
481 { \
482 os_atomic_inc_orig(&nstat_global_counts._field, relaxed); \
483 }
484
485 #define NSTAT_GLOBAL_COUNT_DECREMENT(_field) \
486 { \
487 os_atomic_dec_orig(&nstat_global_counts._field, relaxed); \
488 }
489
490 static errno_t nstat_client_send_counts(nstat_client *client, nstat_src *src, unsigned long long context, u_int16_t hdr_flags, int *gone);
491 static int nstat_client_send_description(nstat_client *client, nstat_src *src, u_int64_t context, u_int16_t hdr_flags);
492 static int nstat_client_send_update(nstat_client *client, nstat_src *src, u_int64_t context, u_int64_t event, u_int16_t hdr_flags, int *gone);
493 static errno_t nstat_client_send_removed(nstat_client *client, nstat_src *src, u_int16_t hdr_flags);
494 static errno_t nstat_client_send_goodbye(nstat_client *client, nstat_src *src);
495 static void nstat_client_cleanup_source(nstat_client *client, nstat_src *src, boolean_t);
496 static bool nstat_client_reporting_allowed(nstat_client *client, nstat_src *src, u_int64_t suppression_flags);
497 static boolean_t nstat_client_begin_query(nstat_client *client, const nstat_msg_hdr *hdrp);
498 static u_int16_t nstat_client_end_query(nstat_client *client, nstat_src *last_src, boolean_t partial);
499 static void nstat_ifnet_report_ecn_stats(void);
500 static void nstat_ifnet_report_lim_stats(void);
501 static void nstat_net_api_report_stats(void);
502 static errno_t nstat_set_provider_filter( nstat_client *client, nstat_msg_add_all_srcs *req);
503 static errno_t nstat_client_send_event(nstat_client *client, nstat_src *src, u_int64_t event);
504 static void nstat_client_register(void);
505
506 /*
507 * The lock order is as follows:
508 *
509 * nstat_rwlock
510 *
511 * Or:
512 *
513 * client->ntc_user_mtx
514 * nstat_rwlock
515 *
516 *
517 * The nstat_rwlock may be held in exclusive (writer) or shared (reader) mode.
518 *
519 * In general, events from the kernel, such as flow creation and deletion,
520 * require that the lock be held in exclusive mode while "nstat_src" structures
521 * are created or destroyed and appropriate linkages made or broken.
522 *
523 * In contrast, things that are driven from ntstat user level clients,
524 * most obviously polls, typically only require shared mode locking.
525 * There are some rare exceptions where a need to manipulate linkages
526 * that go across multiple clients will require an upgrade to an exlusive lock.
527 *
528 * There is one other lock to consider. Calls from NetworkStatistics clients will first
529 * obtain the per-client ntc_user_mtx. This ensures that only one such client thread
530 * can be in progress at a time, making it easier to reason about correctness.
531 */
532 static LCK_ATTR_DECLARE(nstat_lck_attr, 0, 0);
533 static LCK_GRP_DECLARE(nstat_lck_grp, "network statistics kctl");
534 static LCK_RW_DECLARE_ATTR(nstat_rwlock, &nstat_lck_grp, &nstat_lck_attr);
535
536 #define NSTAT_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&nstat_rwlock)
537 #define NSTAT_LOCK_SHARED() lck_rw_lock_shared(&nstat_rwlock)
538 #define NSTAT_LOCK_SHARED_TO_EXCLUSIVE() lck_rw_lock_shared_to_exclusive(&nstat_rwlock)
539 #define NSTAT_LOCK_EXCLUSIVE_TO_SHARED() lck_rw_lock_exclusive_to_shared(&nstat_rwlock)
540 #define NSTAT_UNLOCK() lck_rw_done(&nstat_rwlock)
541 #define NSTAT_UNLOCK_EXCLUSIVE() lck_rw_unlock_exclusive(&nstat_rwlock)
542 #define NSTAT_UNLOCK_SHARED() lck_rw_unlock_shared(&nstat_rwlock)
543 #define NSTAT_LOCK_WOULD_YIELD() lck_rw_lock_would_yield_shared(&nstat_rwlock)
544 #define NSTAT_LOCK_YIELD() lck_rw_lock_yield_shared(&nstat_rwlock, FALSE)
545 #define NSTAT_ASSERT_LOCKED_EXCLUSIVE() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_EXCLUSIVE)
546 #define NSTAT_ASSERT_LOCKED_SHARED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_SHARED)
547 #define NSTAT_ASSERT_LOCKED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_HELD)
548 #define NSTAT_ASSERT_UNLOCKED() LCK_RW_ASSERT(&nstat_rwlock, LCK_RW_ASSERT_NOTHELD)
549
550 typedef enum {
551 NSTAT_LOCK_NOTHELD = 0,
552 NSTAT_LOCK_HELD = 1,
553 } nstat_lock_status;
554
555 /* some extern definitions */
556 extern void tcp_report_stats(void);
557
558 #if NSTAT_TRACE_ENABLED
559 static void
nstat_trace(int32_t trace_id,nstat_client * client,uint64_t qual)560 nstat_trace(int32_t trace_id, nstat_client *client, uint64_t qual)
561 {
562 nstat_cyclic_trace *trace;
563 nstat_trace_entry *entry;
564 uint32_t seqno;
565 if (client == NULL) {
566 trace = &nstat_global_trace;
567 } else {
568 trace = client->ntc_trace;
569 }
570 if (trace != NULL) {
571 seqno = OSIncrementAtomic(&trace->ntt_next_trace_id);
572 if (seqno > (NSTAT_TRACE_ENTRIES_PER_CLIENT - 2)) {
573 seqno = NSTAT_TRACE_ENTRIES_PER_CLIENT - 2;
574 }
575 int32_t index = seqno % NSTAT_TRACE_ENTRIES_PER_CLIENT;
576 entry = &trace->ntt_entry[index];
577 entry->nte_seqno = seqno;
578 entry->nte_event = trace_id;
579 entry->nte_qualifier = qual;
580 }
581 }
582 #endif
583
584 #if NSTAT_FUZZ_TIMING
585 static void
nstat_random_delay(nstat_client * client)586 nstat_random_delay(nstat_client *client)
587 {
588 if (nstat_random_delay_insert_modulo) {
589 uint64_t random;
590 read_random(&random, sizeof(random));
591 if ((random % nstat_random_delay_insert_modulo) == 0) {
592 #if NSTAT_TRACE_ENABLED
593 nstat_trace(1, client, 0);
594 #endif
595 read_random(&random, sizeof(random));
596 uint64_t nsec_delay = (random % nstat_max_nsec_delay);
597 delay_for_interval(nsec_delay, 1);
598 #if NSTAT_TRACE_ENABLED
599 nstat_trace(2, client, 0);
600 #endif
601 }
602 }
603 }
604 #endif
605
606 static void
nstat_accumulate_client_metrics(nstat_client * client)607 nstat_accumulate_client_metrics(nstat_client *client)
608 {
609 if (nstat_metrics.nstat_src_max < client->ntc_metrics.nstat_src_max) {
610 nstat_metrics.nstat_src_max = client->ntc_metrics.nstat_src_max;
611 }
612 // Most of the counts happen to be consecutive uint32_t values that can be picked up via pointer iteration rather than name
613 uint32_t *srcptr = __unsafe_forge_bidi_indexable(uint32_t *,
614 (uint32_t *)(void *)&client->ntc_metrics.nstat_first_uint32_count,
615 (NUM_NSTAT_METRICS_UINT32_COUNTS * sizeof(uint32_t)));
616 uint32_t *destptr = __unsafe_forge_bidi_indexable(uint32_t *,
617 (uint32_t *)(void *)&nstat_metrics.nstat_first_uint32_count,
618 (NUM_NSTAT_METRICS_UINT32_COUNTS * sizeof(uint32_t)));
619
620 for (int i = 0; i < NUM_NSTAT_METRICS_UINT32_COUNTS; i++) {
621 destptr[i] += srcptr[i];
622 }
623 }
624
625 static void
nstat_copy_sa_out(const struct sockaddr * src,struct sockaddr * dst,int maxlen)626 nstat_copy_sa_out(
627 const struct sockaddr *src,
628 struct sockaddr *dst,
629 int maxlen)
630 {
631 if (src->sa_len > maxlen) {
632 return;
633 }
634
635 SOCKADDR_COPY(src, dst, src->sa_len);
636 if (src->sa_family == AF_INET6 &&
637 src->sa_len >= sizeof(struct sockaddr_in6)) {
638 struct sockaddr_in6 *sin6 = SIN6(dst);
639 if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) {
640 sin6->sin6_scope_id = (SIN6(src))->sin6_scope_id;
641 if (in6_embedded_scope) {
642 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
643 sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]);
644 sin6->sin6_addr.s6_addr16[1] = 0;
645 }
646 }
647 }
648 }
649
650 static void
nstat_ip_to_sockaddr(const struct in_addr * ip,u_int16_t port,struct sockaddr_in * sin,u_int32_t maxlen)651 nstat_ip_to_sockaddr(
652 const struct in_addr *ip,
653 u_int16_t port,
654 struct sockaddr_in *sin,
655 u_int32_t maxlen)
656 {
657 if (maxlen < sizeof(struct sockaddr_in)) {
658 return;
659 }
660
661 sin->sin_family = AF_INET;
662 sin->sin_len = sizeof(*sin);
663 sin->sin_port = port;
664 sin->sin_addr = *ip;
665 }
666
667 u_int32_t
nstat_ifnet_to_flags(struct ifnet * ifp)668 nstat_ifnet_to_flags(
669 struct ifnet *ifp)
670 {
671 u_int32_t flags = 0;
672 u_int32_t functional_type = if_functional_type(ifp, FALSE);
673 u_int32_t peer_egress_functional_type = IFRTYPE_FUNCTIONAL_UNKNOWN;
674
675 // If known VPN, check the delegate interface to see if it has peer
676 // egress interface type set to cell.
677 peer_egress_functional_type = if_peer_egress_functional_type(ifp, !(IFNET_IS_VPN(ifp)));
678
679 /* Panic if someone adds a functional type without updating ntstat. */
680 VERIFY(0 <= functional_type && functional_type <= IFRTYPE_FUNCTIONAL_LAST);
681
682 switch (functional_type) {
683 case IFRTYPE_FUNCTIONAL_UNKNOWN:
684 flags |= NSTAT_IFNET_IS_UNKNOWN_TYPE;
685 break;
686 case IFRTYPE_FUNCTIONAL_LOOPBACK:
687 flags |= NSTAT_IFNET_IS_LOOPBACK;
688 break;
689 case IFRTYPE_FUNCTIONAL_WIRED:
690 case IFRTYPE_FUNCTIONAL_INTCOPROC:
691 case IFRTYPE_FUNCTIONAL_MANAGEMENT:
692 flags |= NSTAT_IFNET_IS_WIRED;
693 break;
694 case IFRTYPE_FUNCTIONAL_WIFI_INFRA:
695 flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_WIFI_INFRA;
696 break;
697 case IFRTYPE_FUNCTIONAL_WIFI_AWDL:
698 flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_AWDL;
699 break;
700 case IFRTYPE_FUNCTIONAL_CELLULAR:
701 flags |= NSTAT_IFNET_IS_CELLULAR;
702 break;
703 case IFRTYPE_FUNCTIONAL_COMPANIONLINK:
704 flags |= NSTAT_IFNET_IS_COMPANIONLINK;
705 if (IFNET_IS_COMPANION_LINK_BLUETOOTH(ifp)) {
706 flags |= NSTAT_IFNET_IS_COMPANIONLINK_BT;
707 }
708 break;
709 }
710
711 switch (peer_egress_functional_type) {
712 case IFRTYPE_FUNCTIONAL_CELLULAR:
713 flags |= NSTAT_IFNET_PEEREGRESSINTERFACE_IS_CELLULAR;
714 break;
715 }
716
717 if (IFNET_IS_EXPENSIVE(ifp)) {
718 flags |= NSTAT_IFNET_IS_EXPENSIVE;
719 }
720 if (IFNET_IS_CONSTRAINED(ifp)) {
721 flags |= NSTAT_IFNET_IS_CONSTRAINED;
722 }
723 if (ifnet_is_low_latency(ifp)) {
724 flags |= NSTAT_IFNET_IS_WIFI | NSTAT_IFNET_IS_LLW;
725 }
726 if (IFNET_IS_VPN(ifp)) {
727 flags |= NSTAT_IFNET_IS_VPN;
728 }
729
730 return flags;
731 }
732
733 static void
nstat_update_local_flag_from_inpcb_route(const struct inpcb * inp,u_int32_t * flags)734 nstat_update_local_flag_from_inpcb_route(const struct inpcb *inp,
735 u_int32_t *flags)
736 {
737 if (inp != NULL &&
738 ((inp->inp_route.ro_rt != NULL &&
739 IS_LOCALNET_ROUTE(inp->inp_route.ro_rt)) ||
740 (inp->inp_flags2 & INP2_LAST_ROUTE_LOCAL))) {
741 *flags |= NSTAT_IFNET_IS_LOCAL;
742 } else {
743 *flags |= NSTAT_IFNET_IS_NON_LOCAL;
744 }
745 }
746
747 static u_int32_t
nstat_inpcb_to_flags(const struct inpcb * inp)748 nstat_inpcb_to_flags(
749 const struct inpcb *inp)
750 {
751 u_int32_t flags = 0;
752
753 if (inp != NULL) {
754 if (inp->inp_last_outifp != NULL) {
755 struct ifnet *ifp = inp->inp_last_outifp;
756 flags = nstat_ifnet_to_flags(ifp);
757
758 struct tcpcb *tp = intotcpcb(inp);
759 if (tp) {
760 if (tp->t_flags & TF_LOCAL) {
761 flags |= NSTAT_IFNET_IS_LOCAL;
762 } else {
763 flags |= NSTAT_IFNET_IS_NON_LOCAL;
764 }
765 } else {
766 nstat_update_local_flag_from_inpcb_route(inp, &flags);
767 }
768 } else {
769 flags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
770 nstat_update_local_flag_from_inpcb_route(inp, &flags);
771 }
772 if (inp->inp_socket != NULL &&
773 (inp->inp_socket->so_flags1 & SOF1_CELLFALLBACK)) {
774 flags |= NSTAT_IFNET_VIA_CELLFALLBACK;
775 }
776 }
777 return flags;
778 }
779
780 static void
merge_current_event_filters(void)781 merge_current_event_filters(void)
782 {
783 // The nstat_rwlock is assumed locked
784 NSTAT_ASSERT_LOCKED();
785
786 nstat_merged_provider_filters new_merge = {};
787 nstat_provider_type_t provider;
788 nstat_client *client;
789
790 for (client = nstat_clients; client; client = client->ntc_next) {
791 for (provider = NSTAT_PROVIDER_NONE; provider <= NSTAT_PROVIDER_LAST; provider++) {
792 new_merge.mpf_filters[provider].mf_events |= client->ntc_provider_filters[provider].npf_events;
793 }
794 }
795 for (provider = NSTAT_PROVIDER_NONE; provider <= NSTAT_PROVIDER_LAST; provider++) {
796 // This should do atomic updates of the 64 bit words, where memcpy would be undefined
797 merged_filters.mpf_filters[provider].mf_events = new_merge.mpf_filters[provider].mf_events;
798 }
799 }
800
801
802 #pragma mark -- Network Statistic Providers --
803
804 static errno_t nstat_client_source_add(u_int64_t context, nstat_client *client, nstat_provider *provider,
805 nstat_provider_cookie_t cookie, nstat_lock_status lock_status);
806 struct nstat_provider *nstat_providers = NULL;
807
808 static struct nstat_provider*
nstat_find_provider_by_id(nstat_provider_id_t id)809 nstat_find_provider_by_id(
810 nstat_provider_id_t id)
811 {
812 struct nstat_provider *provider;
813
814 for (provider = nstat_providers; provider != NULL; provider = provider->next) {
815 if (provider->nstat_provider_id == id) {
816 break;
817 }
818 }
819
820 return provider;
821 }
822
823 static errno_t
nstat_lookup_entry(nstat_provider_id_t id,const void * __sized_by (length)data,u_int32_t length,nstat_provider ** out_provider,nstat_provider_cookie_t * out_cookie)824 nstat_lookup_entry(
825 nstat_provider_id_t id,
826 const void *__sized_by(length)data,
827 u_int32_t length,
828 nstat_provider **out_provider,
829 nstat_provider_cookie_t *out_cookie)
830 {
831 *out_provider = nstat_find_provider_by_id(id);
832 if (*out_provider == NULL) {
833 return ENOENT;
834 }
835
836 return (*out_provider)->nstat_lookup(data, length, out_cookie);
837 }
838
839 static void
nstat_client_sanitize_cookie(nstat_client * client,nstat_provider_id_t id,nstat_provider_cookie_t cookie)840 nstat_client_sanitize_cookie(
841 nstat_client *client,
842 nstat_provider_id_t id,
843 nstat_provider_cookie_t cookie)
844 {
845 nstat_src *src = NULL;
846
847 // Scan the source list to find any duplicate entry and remove it.
848 NSTAT_LOCK_EXCLUSIVE();
849 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
850 {
851 nstat_provider *sp = src->nts_provider;
852 if (sp->nstat_provider_id == id &&
853 sp->nstat_cookie_equal != NULL &&
854 sp->nstat_cookie_equal(src->nts_cookie, cookie)) {
855 break;
856 }
857 }
858 if (src) {
859 nstat_client_send_goodbye(client, src);
860 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
861 }
862 NSTAT_UNLOCK_EXCLUSIVE();
863
864 if (src) {
865 nstat_client_cleanup_source(NULL, src, TRUE);
866 }
867 }
868
869 static void nstat_init_route_provider(void);
870 static void nstat_init_tcp_provider(void);
871 static void nstat_init_udp_provider(void);
872 #if SKYWALK
873 static void nstat_init_userland_tcp_provider(void);
874 static void nstat_init_userland_udp_provider(void);
875 static void nstat_init_userland_quic_provider(void);
876 #endif /* SKYWALK */
877 static void nstat_init_userland_conn_provider(void);
878 static void nstat_init_udp_subflow_provider(void);
879 static void nstat_init_ifnet_provider(void);
880
881 __private_extern__ void
nstat_init(void)882 nstat_init(void)
883 {
884 nstat_log_handle = os_log_create("com.apple.xnu.net", "ntstat");
885 nstat_init_route_provider();
886 nstat_init_tcp_provider();
887 nstat_init_udp_provider();
888 #if SKYWALK
889 nstat_init_userland_tcp_provider();
890 nstat_init_userland_udp_provider();
891 nstat_init_userland_quic_provider();
892 #endif /* SKYWALK */
893 nstat_init_userland_conn_provider();
894 nstat_init_udp_subflow_provider();
895 nstat_init_ifnet_provider();
896 nstat_client_register();
897 }
898
899 #pragma mark -- Utilities --
900
901 #define NSTAT_PROCDETAILS_MAGIC 0xfeedc001
902 #define NSTAT_PROCDETAILS_UNMAGIC 0xdeadc001
903
904 static tailq_head_procdetails nstat_procdetails_head = TAILQ_HEAD_INITIALIZER(nstat_procdetails_head);
905
906 static struct nstat_procdetails *
nstat_retain_curprocdetails(void)907 nstat_retain_curprocdetails(void)
908 {
909 struct nstat_procdetails *procdetails = NULL;
910 uint64_t upid = proc_uniqueid(current_proc());
911
912 NSTAT_LOCK_SHARED();
913 TAILQ_FOREACH(procdetails, &nstat_procdetails_head, pdet_link) {
914 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
915
916 if (procdetails->pdet_upid == upid) {
917 OSIncrementAtomic(&procdetails->pdet_refcnt);
918 break;
919 }
920 }
921 NSTAT_UNLOCK_SHARED();
922
923 if (!procdetails) {
924 // No need for paranoia on locking, it would be OK if there are duplicate structs on the list
925 procdetails = kalloc_type(struct nstat_procdetails,
926 Z_WAITOK | Z_NOFAIL);
927 procdetails->pdet_pid = proc_selfpid();
928 procdetails->pdet_upid = upid;
929 proc_selfname(procdetails->pdet_procname, sizeof(procdetails->pdet_procname));
930 proc_getexecutableuuid(current_proc(), procdetails->pdet_uuid, sizeof(uuid_t));
931 procdetails->pdet_refcnt = 1;
932 procdetails->pdet_magic = NSTAT_PROCDETAILS_MAGIC;
933 NSTAT_LOCK_EXCLUSIVE();
934 TAILQ_INSERT_HEAD(&nstat_procdetails_head, procdetails, pdet_link);
935 NSTAT_UNLOCK_EXCLUSIVE();
936 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_procdetails_allocs);
937 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_procdetails_current, nstat_global_procdetails_max);
938 }
939
940 return procdetails;
941 }
942
943 static void
nstat_release_procdetails(struct nstat_procdetails * procdetails)944 nstat_release_procdetails(struct nstat_procdetails *procdetails)
945 {
946 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
947 // These are harvested later to amortize costs
948 OSDecrementAtomic(&procdetails->pdet_refcnt);
949 }
950
951 static void
nstat_prune_procdetails(void)952 nstat_prune_procdetails(void)
953 {
954 struct nstat_procdetails *procdetails;
955 struct nstat_procdetails *tmpdetails;
956 tailq_head_procdetails dead_list;
957
958 TAILQ_INIT(&dead_list);
959 NSTAT_LOCK_EXCLUSIVE();
960
961 TAILQ_FOREACH_SAFE(procdetails, &nstat_procdetails_head, pdet_link, tmpdetails)
962 {
963 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
964 if (procdetails->pdet_refcnt == 0) {
965 // Pull it off the list
966 TAILQ_REMOVE(&nstat_procdetails_head, procdetails, pdet_link);
967 TAILQ_INSERT_TAIL(&dead_list, procdetails, pdet_link);
968 }
969 }
970 NSTAT_UNLOCK_EXCLUSIVE();
971
972 while ((procdetails = TAILQ_FIRST(&dead_list))) {
973 TAILQ_REMOVE(&dead_list, procdetails, pdet_link);
974 procdetails->pdet_magic = NSTAT_PROCDETAILS_UNMAGIC;
975 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_procdetails_current);
976 kfree_type(struct nstat_procdetails, procdetails);
977 }
978 }
979
980 #pragma mark -- Route Provider --
981
982 static nstat_provider nstat_route_provider;
983
984 static errno_t
nstat_route_lookup(const void * data,u_int32_t length,nstat_provider_cookie_t * out_cookie)985 nstat_route_lookup(
986 const void *data,
987 u_int32_t length,
988 nstat_provider_cookie_t *out_cookie)
989 {
990 struct sockaddr *dst = NULL;
991 struct sockaddr *mask = NULL;
992 const nstat_route_add_param *param = (const nstat_route_add_param*)data;
993 *out_cookie = NULL;
994
995 if (length < sizeof(*param)) {
996 return EINVAL;
997 }
998
999 if (param->dst.v4.sin_family == 0 ||
1000 param->dst.v4.sin_family > AF_MAX ||
1001 (param->mask.v4.sin_family != 0 && param->mask.v4.sin_family != param->dst.v4.sin_family)) {
1002 return EINVAL;
1003 }
1004
1005 if (param->dst.v4.sin_len > sizeof(param->dst) ||
1006 (param->mask.v4.sin_family && param->mask.v4.sin_len > sizeof(param->mask.v4.sin_len))) {
1007 return EINVAL;
1008 }
1009 if ((param->dst.v4.sin_family == AF_INET &&
1010 param->dst.v4.sin_len < sizeof(struct sockaddr_in)) ||
1011 (param->dst.v6.sin6_family == AF_INET6 &&
1012 param->dst.v6.sin6_len < sizeof(struct sockaddr_in6))) {
1013 return EINVAL;
1014 }
1015
1016 dst = __DECONST_SA(¶m->dst.v4);
1017 mask = param->mask.v4.sin_family ? __DECONST_SA(¶m->mask.v4) : NULL;
1018
1019 struct radix_node_head *rnh = rt_tables[dst->sa_family];
1020 if (rnh == NULL) {
1021 return EAFNOSUPPORT;
1022 }
1023
1024 lck_mtx_lock(rnh_lock);
1025 struct rtentry *rt = rt_lookup(TRUE, dst, mask, rnh, param->ifindex);
1026 lck_mtx_unlock(rnh_lock);
1027
1028 if (rt) {
1029 *out_cookie = (nstat_provider_cookie_t)rt;
1030 }
1031
1032 return rt ? 0 : ENOENT;
1033 }
1034
1035 static int
nstat_route_gone(nstat_provider_cookie_t cookie)1036 nstat_route_gone(
1037 nstat_provider_cookie_t cookie)
1038 {
1039 struct rtentry *rt = (struct rtentry*)cookie;
1040 return ((rt->rt_flags & RTF_UP) == 0) ? 1 : 0;
1041 }
1042
1043 static errno_t
nstat_route_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)1044 nstat_route_counts(
1045 nstat_provider_cookie_t cookie,
1046 struct nstat_counts *out_counts,
1047 int *out_gone)
1048 {
1049 struct rtentry *rt = (struct rtentry*)cookie;
1050 struct nstat_counts *rt_stats = rt->rt_stats;
1051
1052 if (out_gone) {
1053 *out_gone = 0;
1054 }
1055
1056 if (out_gone && (rt->rt_flags & RTF_UP) == 0) {
1057 *out_gone = 1;
1058 }
1059
1060 if (rt_stats) {
1061 out_counts->nstat_rxpackets = os_atomic_load(&rt_stats->nstat_rxpackets, relaxed);
1062 out_counts->nstat_rxbytes = os_atomic_load(&rt_stats->nstat_rxbytes, relaxed);
1063 out_counts->nstat_txpackets = os_atomic_load(&rt_stats->nstat_txpackets, relaxed);
1064 out_counts->nstat_txbytes = os_atomic_load(&rt_stats->nstat_txbytes, relaxed);
1065 out_counts->nstat_rxduplicatebytes = rt_stats->nstat_rxduplicatebytes;
1066 out_counts->nstat_rxoutoforderbytes = rt_stats->nstat_rxoutoforderbytes;
1067 out_counts->nstat_txretransmit = rt_stats->nstat_txretransmit;
1068 out_counts->nstat_connectattempts = rt_stats->nstat_connectattempts;
1069 out_counts->nstat_connectsuccesses = rt_stats->nstat_connectsuccesses;
1070 out_counts->nstat_min_rtt = rt_stats->nstat_min_rtt;
1071 out_counts->nstat_avg_rtt = rt_stats->nstat_avg_rtt;
1072 out_counts->nstat_var_rtt = rt_stats->nstat_var_rtt;
1073 out_counts->nstat_cell_rxbytes = out_counts->nstat_cell_txbytes = 0;
1074 } else {
1075 bzero(out_counts, sizeof(*out_counts));
1076 }
1077
1078 return 0;
1079 }
1080
1081 static void
nstat_route_release(nstat_provider_cookie_t cookie,__unused int locked)1082 nstat_route_release(
1083 nstat_provider_cookie_t cookie,
1084 __unused int locked)
1085 {
1086 rtfree((struct rtentry*)cookie);
1087 }
1088
1089 static u_int32_t nstat_route_watchers = 0;
1090
1091 static int
nstat_route_walktree_add(struct radix_node * rn,void * context)1092 nstat_route_walktree_add(
1093 struct radix_node *rn,
1094 void *context)
1095 {
1096 errno_t result = 0;
1097 struct rtentry *rt = (struct rtentry *)rn;
1098 nstat_client *client = (nstat_client*)context;
1099
1100 LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED);
1101
1102 /* RTF_UP can't change while rnh_lock is held */
1103 if ((rt->rt_flags & RTF_UP) != 0) {
1104 /* Clear RTPRF_OURS if the route is still usable */
1105 RT_LOCK(rt);
1106 if (rt_validate(rt)) {
1107 RT_ADDREF_LOCKED(rt);
1108 RT_UNLOCK(rt);
1109 } else {
1110 RT_UNLOCK(rt);
1111 rt = NULL;
1112 }
1113
1114 /* Otherwise if RTF_CONDEMNED, treat it as if it were down */
1115 if (rt == NULL) {
1116 return 0;
1117 }
1118
1119 result = nstat_client_source_add(0, client, &nstat_route_provider, rt, NSTAT_LOCK_NOTHELD);
1120 if (result != 0) {
1121 rtfree_locked(rt);
1122 }
1123 }
1124
1125 return result;
1126 }
1127
1128 static errno_t
nstat_route_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)1129 nstat_route_add_watcher(
1130 nstat_client *client,
1131 nstat_msg_add_all_srcs *req)
1132 {
1133 int i;
1134 errno_t result = 0;
1135
1136 lck_mtx_lock(rnh_lock);
1137
1138 result = nstat_set_provider_filter(client, req);
1139 if (result == 0) {
1140 OSIncrementAtomic(&nstat_route_watchers);
1141
1142 for (i = 1; i < AF_MAX; i++) {
1143 struct radix_node_head *rnh;
1144 rnh = rt_tables[i];
1145 if (!rnh) {
1146 continue;
1147 }
1148
1149 result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, client);
1150 if (result != 0) {
1151 // This is probably resource exhaustion.
1152 // There currently isn't a good way to recover from this.
1153 // Least bad seems to be to give up on the add-all but leave
1154 // the watcher in place.
1155 break;
1156 }
1157 }
1158 }
1159 lck_mtx_unlock(rnh_lock);
1160
1161 return result;
1162 }
1163
1164 __private_extern__ void
nstat_route_new_entry(struct rtentry * rt)1165 nstat_route_new_entry(
1166 struct rtentry *rt)
1167 {
1168 if (nstat_route_watchers == 0) {
1169 return;
1170 }
1171
1172 NSTAT_LOCK_EXCLUSIVE();
1173 if ((rt->rt_flags & RTF_UP) != 0) {
1174 nstat_client *client;
1175 for (client = nstat_clients; client; client = client->ntc_next) {
1176 if ((client->ntc_watching & (1 << NSTAT_PROVIDER_ROUTE)) != 0) {
1177 // this client is watching routes
1178 // acquire a reference for the route
1179 RT_ADDREF(rt);
1180
1181 // add the source, if that fails, release the reference
1182 if (nstat_client_source_add(0, client, &nstat_route_provider, rt, NSTAT_LOCK_HELD) != 0) {
1183 RT_REMREF(rt);
1184 }
1185 }
1186 }
1187 }
1188 NSTAT_UNLOCK_EXCLUSIVE();
1189 }
1190
1191 static void
nstat_route_remove_watcher(__unused nstat_client * client)1192 nstat_route_remove_watcher(
1193 __unused nstat_client *client)
1194 {
1195 OSDecrementAtomic(&nstat_route_watchers);
1196 }
1197
1198 static errno_t
nstat_route_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,size_t len)1199 nstat_route_copy_descriptor(
1200 nstat_provider_cookie_t cookie,
1201 void *__sized_by(len)data,
1202 size_t len)
1203 {
1204 nstat_route_descriptor *desc = (nstat_route_descriptor*)data;
1205 if (len < sizeof(*desc)) {
1206 return EINVAL;
1207 }
1208 bzero(desc, sizeof(*desc));
1209
1210 struct rtentry *rt = (struct rtentry*)cookie;
1211 desc->id = (uint64_t)VM_KERNEL_ADDRHASH(rt);
1212 desc->parent_id = (uint64_t)VM_KERNEL_ADDRHASH(rt->rt_parent);
1213 desc->gateway_id = (uint64_t)VM_KERNEL_ADDRHASH(rt->rt_gwroute);
1214
1215
1216 // key/dest
1217 struct sockaddr *sa;
1218 if ((sa = rt_key(rt))) {
1219 nstat_copy_sa_out(sa, &desc->dst.sa, sizeof(desc->dst));
1220 }
1221
1222 // mask
1223 if ((sa = rt_mask(rt))) {
1224 nstat_copy_sa_out(sa, &desc->mask.sa, sizeof(desc->mask));
1225 }
1226
1227 // gateway
1228 if ((sa = rt->rt_gateway)) {
1229 nstat_copy_sa_out(sa, &desc->gateway.sa, sizeof(desc->gateway));
1230 }
1231
1232 if (rt->rt_ifp) {
1233 desc->ifindex = rt->rt_ifp->if_index;
1234 }
1235
1236 desc->flags = rt->rt_flags;
1237
1238 return 0;
1239 }
1240
1241 static bool
nstat_route_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)1242 nstat_route_reporting_allowed(
1243 nstat_provider_cookie_t cookie,
1244 nstat_provider_filter *filter,
1245 __unused u_int64_t suppression_flags)
1246 {
1247 bool retval = true;
1248
1249 if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) {
1250 struct rtentry *rt = (struct rtentry*)cookie;
1251 struct ifnet *ifp = rt->rt_ifp;
1252
1253 if (ifp) {
1254 uint32_t interface_properties = nstat_ifnet_to_flags(ifp);
1255
1256 if ((filter->npf_flags & interface_properties) == 0) {
1257 retval = false;
1258 }
1259 }
1260 }
1261 return retval;
1262 }
1263
1264 static bool
nstat_route_cookie_equal(nstat_provider_cookie_t cookie1,nstat_provider_cookie_t cookie2)1265 nstat_route_cookie_equal(
1266 nstat_provider_cookie_t cookie1,
1267 nstat_provider_cookie_t cookie2)
1268 {
1269 struct rtentry *rt1 = (struct rtentry *)cookie1;
1270 struct rtentry *rt2 = (struct rtentry *)cookie2;
1271
1272 return (rt1 == rt2) ? true : false;
1273 }
1274
1275 static void
nstat_init_route_provider(void)1276 nstat_init_route_provider(void)
1277 {
1278 bzero(&nstat_route_provider, sizeof(nstat_route_provider));
1279 nstat_route_provider.nstat_descriptor_length = sizeof(nstat_route_descriptor);
1280 nstat_route_provider.nstat_provider_id = NSTAT_PROVIDER_ROUTE;
1281 nstat_route_provider.nstat_lookup = nstat_route_lookup;
1282 nstat_route_provider.nstat_gone = nstat_route_gone;
1283 nstat_route_provider.nstat_counts = nstat_route_counts;
1284 nstat_route_provider.nstat_release = nstat_route_release;
1285 nstat_route_provider.nstat_watcher_add = nstat_route_add_watcher;
1286 nstat_route_provider.nstat_watcher_remove = nstat_route_remove_watcher;
1287 nstat_route_provider.nstat_copy_descriptor = nstat_route_copy_descriptor;
1288 nstat_route_provider.nstat_reporting_allowed = nstat_route_reporting_allowed;
1289 nstat_route_provider.nstat_cookie_equal = nstat_route_cookie_equal;
1290 nstat_route_provider.next = nstat_providers;
1291 nstat_providers = &nstat_route_provider;
1292 }
1293
1294 #pragma mark -- Route Collection --
1295
1296 __private_extern__ struct nstat_counts*
nstat_route_attach(struct rtentry * rte)1297 nstat_route_attach(
1298 struct rtentry *rte)
1299 {
1300 struct nstat_counts * __single result = rte->rt_stats;
1301 if (result) {
1302 return result;
1303 }
1304
1305 result = kalloc_type(struct nstat_counts, Z_WAITOK | Z_ZERO);
1306 if (!result) {
1307 return result;
1308 }
1309
1310 if (!OSCompareAndSwapPtr(NULL, result, &rte->rt_stats)) {
1311 kfree_type(struct nstat_counts, result);
1312 result = rte->rt_stats;
1313 }
1314
1315 return result;
1316 }
1317
1318 __private_extern__ void
nstat_route_detach(struct rtentry * rte)1319 nstat_route_detach(
1320 struct rtentry *rte)
1321 {
1322 if (rte->rt_stats) {
1323 kfree_type(struct nstat_counts, rte->rt_stats);
1324 rte->rt_stats = NULL;
1325 }
1326 }
1327
1328 __private_extern__ void
nstat_route_connect_attempt(struct rtentry * rte)1329 nstat_route_connect_attempt(
1330 struct rtentry *rte)
1331 {
1332 while (rte) {
1333 struct nstat_counts* stats = nstat_route_attach(rte);
1334 if (stats) {
1335 OSIncrementAtomic(&stats->nstat_connectattempts);
1336 }
1337
1338 rte = rte->rt_parent;
1339 }
1340 }
1341
1342 __private_extern__ void
nstat_route_connect_success(struct rtentry * rte)1343 nstat_route_connect_success(
1344 struct rtentry *rte)
1345 {
1346 // This route
1347 while (rte) {
1348 struct nstat_counts* stats = nstat_route_attach(rte);
1349 if (stats) {
1350 OSIncrementAtomic(&stats->nstat_connectsuccesses);
1351 }
1352
1353 rte = rte->rt_parent;
1354 }
1355 }
1356
1357 __private_extern__ void
nstat_route_tx(struct rtentry * rte,u_int32_t packets,u_int32_t bytes,u_int32_t flags)1358 nstat_route_tx(
1359 struct rtentry *rte,
1360 u_int32_t packets,
1361 u_int32_t bytes,
1362 u_int32_t flags)
1363 {
1364 while (rte) {
1365 struct nstat_counts* stats = nstat_route_attach(rte);
1366 if (stats) {
1367 if ((flags & NSTAT_TX_FLAG_RETRANSMIT) != 0) {
1368 OSAddAtomic(bytes, &stats->nstat_txretransmit);
1369 } else {
1370 OSAddAtomic64((SInt64)packets, (SInt64*)&stats->nstat_txpackets);
1371 OSAddAtomic64((SInt64)bytes, (SInt64*)&stats->nstat_txbytes);
1372 }
1373 }
1374
1375 rte = rte->rt_parent;
1376 }
1377 }
1378
1379 __private_extern__ void
nstat_route_rx(struct rtentry * rte,u_int32_t packets,u_int32_t bytes,u_int32_t flags)1380 nstat_route_rx(
1381 struct rtentry *rte,
1382 u_int32_t packets,
1383 u_int32_t bytes,
1384 u_int32_t flags)
1385 {
1386 while (rte) {
1387 struct nstat_counts* stats = nstat_route_attach(rte);
1388 if (stats) {
1389 if (flags == 0) {
1390 OSAddAtomic64((SInt64)packets, (SInt64*)&stats->nstat_rxpackets);
1391 OSAddAtomic64((SInt64)bytes, (SInt64*)&stats->nstat_rxbytes);
1392 } else {
1393 if (flags & NSTAT_RX_FLAG_OUT_OF_ORDER) {
1394 OSAddAtomic(bytes, &stats->nstat_rxoutoforderbytes);
1395 }
1396 if (flags & NSTAT_RX_FLAG_DUPLICATE) {
1397 OSAddAtomic(bytes, &stats->nstat_rxduplicatebytes);
1398 }
1399 }
1400 }
1401
1402 rte = rte->rt_parent;
1403 }
1404 }
1405
1406 /* atomically average current value at _val_addr with _new_val and store */
1407 #define NSTAT_EWMA_ATOMIC(_val_addr, _new_val, _decay) do { \
1408 volatile uint32_t _old_val; \
1409 volatile uint32_t _avg; \
1410 do { \
1411 _old_val = *_val_addr; \
1412 if (_old_val == 0) \
1413 { \
1414 _avg = _new_val; \
1415 } \
1416 else \
1417 { \
1418 _avg = _old_val - (_old_val >> _decay) + (_new_val >> _decay); \
1419 } \
1420 if (_old_val == _avg) break; \
1421 } while (!OSCompareAndSwap(_old_val, _avg, _val_addr)); \
1422 } while (0);
1423
1424 /* atomically compute minimum of current value at _val_addr with _new_val and store */
1425 #define NSTAT_MIN_ATOMIC(_val_addr, _new_val) do { \
1426 volatile uint32_t _old_val; \
1427 do { \
1428 _old_val = *_val_addr; \
1429 if (_old_val != 0 && _old_val < _new_val) \
1430 { \
1431 break; \
1432 } \
1433 } while (!OSCompareAndSwap(_old_val, _new_val, _val_addr)); \
1434 } while (0);
1435
1436 __private_extern__ void
nstat_route_rtt(struct rtentry * rte,u_int32_t rtt,u_int32_t rtt_var)1437 nstat_route_rtt(
1438 struct rtentry *rte,
1439 u_int32_t rtt,
1440 u_int32_t rtt_var)
1441 {
1442 const uint32_t decay = 3;
1443
1444 while (rte) {
1445 struct nstat_counts* stats = nstat_route_attach(rte);
1446 if (stats) {
1447 NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay);
1448 NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt);
1449 NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay);
1450 }
1451 rte = rte->rt_parent;
1452 }
1453 }
1454
1455 __private_extern__ void
nstat_route_update(struct rtentry * rte,uint32_t connect_attempts,uint32_t connect_successes,uint32_t rx_packets,uint32_t rx_bytes,uint32_t rx_duplicatebytes,uint32_t rx_outoforderbytes,uint32_t tx_packets,uint32_t tx_bytes,uint32_t tx_retransmit,uint32_t rtt,uint32_t rtt_var)1456 nstat_route_update(
1457 struct rtentry *rte,
1458 uint32_t connect_attempts,
1459 uint32_t connect_successes,
1460 uint32_t rx_packets,
1461 uint32_t rx_bytes,
1462 uint32_t rx_duplicatebytes,
1463 uint32_t rx_outoforderbytes,
1464 uint32_t tx_packets,
1465 uint32_t tx_bytes,
1466 uint32_t tx_retransmit,
1467 uint32_t rtt,
1468 uint32_t rtt_var)
1469 {
1470 const uint32_t decay = 3;
1471
1472 while (rte) {
1473 struct nstat_counts* stats = nstat_route_attach(rte);
1474 if (stats) {
1475 OSAddAtomic(connect_attempts, &stats->nstat_connectattempts);
1476 OSAddAtomic(connect_successes, &stats->nstat_connectsuccesses);
1477 OSAddAtomic64((SInt64)tx_packets, (SInt64*)&stats->nstat_txpackets);
1478 OSAddAtomic64((SInt64)tx_bytes, (SInt64*)&stats->nstat_txbytes);
1479 OSAddAtomic(tx_retransmit, &stats->nstat_txretransmit);
1480 OSAddAtomic64((SInt64)rx_packets, (SInt64*)&stats->nstat_rxpackets);
1481 OSAddAtomic64((SInt64)rx_bytes, (SInt64*)&stats->nstat_rxbytes);
1482 OSAddAtomic(rx_outoforderbytes, &stats->nstat_rxoutoforderbytes);
1483 OSAddAtomic(rx_duplicatebytes, &stats->nstat_rxduplicatebytes);
1484
1485 if (rtt != 0) {
1486 NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay);
1487 NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt);
1488 NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay);
1489 }
1490 }
1491 rte = rte->rt_parent;
1492 }
1493 }
1494
1495 #pragma mark -- TCP Kernel Provider --
1496
1497 #define PNAME_MAX_LENGTH ((2 * MAXCOMLEN) + 1)
1498
1499 /*
1500 * Due to the way the kernel deallocates a process (the process structure
1501 * might be gone by the time we get the PCB detach notification),
1502 * we need to cache the process name. Without this, proc_best_name_for_pid()
1503 * would return null and the process name would never be sent to userland.
1504 *
1505 * For UDP sockets, we also store the cached the connection tuples along with
1506 * the interface index. This is necessary because when UDP sockets are
1507 * disconnected, the connection tuples are forever lost from the inpcb, thus
1508 * we need to keep track of the last call to connect() in ntstat.
1509 */
1510 struct nstat_tucookie {
1511 struct inpcb *inp;
1512 char pname[PNAME_MAX_LENGTH];
1513 bool cached;
1514 union{
1515 struct sockaddr_in v4;
1516 struct sockaddr_in6 v6;
1517 } local;
1518 union{
1519 struct sockaddr_in v4;
1520 struct sockaddr_in6 v6;
1521 } remote;
1522 unsigned int if_index;
1523 uint32_t ifnet_properties;
1524 };
1525
1526 static struct nstat_tucookie *
nstat_tucookie_alloc_ref_internal(struct inpcb * inp,bool locked)1527 nstat_tucookie_alloc_ref_internal(
1528 struct inpcb *inp,
1529 bool locked)
1530 {
1531 struct nstat_tucookie *cookie;
1532
1533 if (inp->inp_state == INPCB_STATE_DEAD) {
1534 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tucookie_skip_dead);
1535 return NULL;
1536 }
1537
1538 cookie = kalloc_type(struct nstat_tucookie, Z_WAITOK | Z_ZERO);
1539
1540 if (cookie == NULL) {
1541 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tucookie_alloc_fail);
1542 return NULL;
1543 }
1544 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tucookie_allocs);
1545 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_tucookie_current, nstat_global_tucookie_max);
1546
1547 if (in_pcb_checkstate(inp, WNT_ACQUIRE, locked) == WNT_STOPUSING) {
1548 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tucookie_skip_stopusing);
1549 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tucookie_current);
1550 kfree_type(struct nstat_tucookie, cookie);
1551 return NULL;
1552 }
1553 cookie->inp = inp;
1554 proc_best_name_for_pid(inp->inp_socket->last_pid, cookie->pname, sizeof(cookie->pname));
1555 /*
1556 * We only increment the reference count for UDP sockets because we
1557 * only cache UDP socket tuples.
1558 */
1559 if (SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP) {
1560 OSIncrementAtomic(&inp->inp_nstat_refcnt);
1561 }
1562
1563 return cookie;
1564 }
1565
1566 static struct nstat_tucookie *
nstat_tucookie_alloc_ref(struct inpcb * inp)1567 nstat_tucookie_alloc_ref(
1568 struct inpcb *inp)
1569 {
1570 return nstat_tucookie_alloc_ref_internal(inp, false);
1571 }
1572
1573 static struct nstat_tucookie *
nstat_tucookie_alloc_ref_locked(struct inpcb * inp)1574 nstat_tucookie_alloc_ref_locked(
1575 struct inpcb *inp)
1576 {
1577 return nstat_tucookie_alloc_ref_internal(inp, true);
1578 }
1579
1580 static void
nstat_tucookie_release_internal(struct nstat_tucookie * cookie,int inplock)1581 nstat_tucookie_release_internal(
1582 struct nstat_tucookie *cookie,
1583 int inplock)
1584 {
1585 if (SOCK_PROTO(cookie->inp->inp_socket) == IPPROTO_UDP) {
1586 OSDecrementAtomic(&cookie->inp->inp_nstat_refcnt);
1587 }
1588 in_pcb_checkstate(cookie->inp, WNT_RELEASE, inplock);
1589 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tucookie_current);
1590 kfree_type(struct nstat_tucookie, cookie);
1591 }
1592
1593 static void
nstat_tucookie_release(struct nstat_tucookie * cookie)1594 nstat_tucookie_release(
1595 struct nstat_tucookie *cookie)
1596 {
1597 nstat_tucookie_release_internal(cookie, false);
1598 }
1599
1600 static void
nstat_tucookie_release_locked(struct nstat_tucookie * cookie)1601 nstat_tucookie_release_locked(
1602 struct nstat_tucookie *cookie)
1603 {
1604 nstat_tucookie_release_internal(cookie, true);
1605 }
1606
1607
1608 static size_t
nstat_inp_domain_info(struct inpcb * inp,nstat_domain_info * domain_info,size_t len)1609 nstat_inp_domain_info(struct inpcb *inp, nstat_domain_info *domain_info, size_t len)
1610 {
1611 // Note, the caller has guaranteed that the buffer has been zeroed, there is no need to clear it again
1612 struct socket *so = inp->inp_socket;
1613
1614 if (so == NULL) {
1615 return 0;
1616 }
1617
1618 NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: Collecting stats");
1619
1620 if (domain_info == NULL) {
1621 return sizeof(nstat_domain_info);
1622 }
1623
1624 if (len < sizeof(nstat_domain_info)) {
1625 return 0;
1626 }
1627
1628 necp_copy_inp_domain_info(inp, so, domain_info);
1629
1630 NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: <pid %d> Collected stats - domain <%s> owner <%s> ctxt <%s> bundle id <%s> "
1631 "is_tracker %d is_non_app_initiated %d is_silent %d",
1632 so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid,
1633 domain_info->domain_name,
1634 domain_info->domain_owner,
1635 domain_info->domain_tracker_ctxt,
1636 domain_info->domain_attributed_bundle_id,
1637 domain_info->is_tracker,
1638 domain_info->is_non_app_initiated,
1639 domain_info->is_silent);
1640
1641 return sizeof(nstat_domain_info);
1642 }
1643
1644 static size_t
nstat_inp_bluetooth_counts(struct inpcb * inp,nstat_interface_counts * buf,size_t len)1645 nstat_inp_bluetooth_counts(struct inpcb *inp, nstat_interface_counts *buf, size_t len)
1646 {
1647 // Note, the caller has guaranteed that the buffer has been zeroed, there is no need to clear it again
1648 struct socket *so = inp->inp_socket;
1649
1650 if (so == NULL) {
1651 return 0;
1652 }
1653
1654 if (buf == NULL) {
1655 uint64_t rxbytes = 0;
1656 uint64_t txbytes = 0;
1657 rxbytes = os_atomic_load(&inp->inp_btstat->rxbytes, relaxed);
1658 txbytes = os_atomic_load(&inp->inp_btstat->txbytes, relaxed);
1659
1660 if ((rxbytes == 0) && (txbytes == 0)) {
1661 // It's more efficient to skip sending counts if they're only going to be zero
1662 return 0;
1663 }
1664 return sizeof(nstat_interface_counts);
1665 }
1666
1667 if (len < sizeof(nstat_interface_counts)) {
1668 return 0;
1669 }
1670
1671 // if the pcb is in the dead state, we should stop using it
1672 if (!(intotcpcb(inp)) || inp->inp_state == INPCB_STATE_DEAD) {
1673 return 0;
1674 }
1675 buf->nstat_rxpackets = os_atomic_load(&inp->inp_btstat->rxpackets, relaxed);
1676 buf->nstat_rxbytes = os_atomic_load(&inp->inp_btstat->rxbytes, relaxed);
1677 buf->nstat_txpackets = os_atomic_load(&inp->inp_btstat->txpackets, relaxed);
1678 buf->nstat_txbytes = os_atomic_load(&inp->inp_btstat->txbytes, relaxed);
1679 return sizeof(nstat_interface_counts);
1680 }
1681
1682 static nstat_provider nstat_tcp_provider;
1683
1684 static errno_t
nstat_tcp_lookup(__unused const void * data,__unused u_int32_t length,__unused nstat_provider_cookie_t * out_cookie)1685 nstat_tcp_lookup(
1686 __unused const void *data,
1687 __unused u_int32_t length,
1688 __unused nstat_provider_cookie_t *out_cookie)
1689 {
1690 // Looking up a specific connection is not supported.
1691 return ENOTSUP;
1692 }
1693
1694 static int
nstat_tcp_gone(nstat_provider_cookie_t cookie)1695 nstat_tcp_gone(
1696 nstat_provider_cookie_t cookie)
1697 {
1698 struct nstat_tucookie *tucookie =
1699 (struct nstat_tucookie *)cookie;
1700 struct inpcb *inp;
1701 struct tcpcb *tp;
1702
1703 return (!(inp = tucookie->inp) ||
1704 !(tp = intotcpcb(inp)) ||
1705 inp->inp_state == INPCB_STATE_DEAD) ? 1 : 0;
1706 }
1707
1708 static errno_t
nstat_tcp_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)1709 nstat_tcp_counts(
1710 nstat_provider_cookie_t cookie,
1711 struct nstat_counts *out_counts,
1712 int *out_gone)
1713 {
1714 struct nstat_tucookie *tucookie =
1715 (struct nstat_tucookie *)cookie;
1716 struct inpcb *inp;
1717
1718 bzero(out_counts, sizeof(*out_counts));
1719
1720 if (out_gone) {
1721 *out_gone = 0;
1722 }
1723
1724 // if the pcb is in the dead state, we should stop using it
1725 if (nstat_tcp_gone(cookie)) {
1726 if (out_gone) {
1727 *out_gone = 1;
1728 }
1729 if (!(inp = tucookie->inp) || !intotcpcb(inp)) {
1730 return EINVAL;
1731 }
1732 }
1733 inp = tucookie->inp;
1734 struct tcpcb *tp = intotcpcb(inp);
1735
1736 out_counts->nstat_rxpackets = os_atomic_load(&inp->inp_stat->rxpackets, relaxed);
1737 out_counts->nstat_rxbytes = os_atomic_load(&inp->inp_stat->rxbytes, relaxed);
1738 out_counts->nstat_txpackets = os_atomic_load(&inp->inp_stat->txpackets, relaxed);
1739 out_counts->nstat_txbytes = os_atomic_load(&inp->inp_stat->txbytes, relaxed);
1740 out_counts->nstat_rxduplicatebytes = tp->t_stat.rxduplicatebytes;
1741 out_counts->nstat_rxoutoforderbytes = tp->t_stat.rxoutoforderbytes;
1742 out_counts->nstat_txretransmit = tp->t_stat.txretransmitbytes;
1743 out_counts->nstat_connectattempts = tp->t_state >= TCPS_SYN_SENT ? 1 : 0;
1744 out_counts->nstat_connectsuccesses = tp->t_state >= TCPS_ESTABLISHED ? 1 : 0;
1745 out_counts->nstat_avg_rtt = tp->t_srtt;
1746 out_counts->nstat_min_rtt = tp->t_rttbest;
1747 out_counts->nstat_var_rtt = tp->t_rttvar;
1748 if (out_counts->nstat_avg_rtt < out_counts->nstat_min_rtt) {
1749 out_counts->nstat_min_rtt = out_counts->nstat_avg_rtt;
1750 }
1751 out_counts->nstat_cell_rxbytes = os_atomic_load(&inp->inp_cstat->rxbytes, relaxed);
1752 out_counts->nstat_cell_txbytes = os_atomic_load(&inp->inp_cstat->txbytes, relaxed);
1753 out_counts->nstat_wifi_rxbytes = os_atomic_load(&inp->inp_wstat->rxbytes, relaxed);
1754 out_counts->nstat_wifi_txbytes = os_atomic_load(&inp->inp_wstat->txbytes, relaxed);
1755 out_counts->nstat_wired_rxbytes = os_atomic_load(&inp->inp_Wstat->rxbytes, relaxed);
1756 out_counts->nstat_wired_txbytes = os_atomic_load(&inp->inp_Wstat->txbytes, relaxed);
1757
1758 return 0;
1759 }
1760
1761 static void
nstat_tcp_release(nstat_provider_cookie_t cookie,int locked)1762 nstat_tcp_release(
1763 nstat_provider_cookie_t cookie,
1764 int locked)
1765 {
1766 struct nstat_tucookie *tucookie =
1767 (struct nstat_tucookie *)cookie;
1768
1769 nstat_tucookie_release_internal(tucookie, locked);
1770 }
1771
1772 static errno_t
nstat_tcp_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)1773 nstat_tcp_add_watcher(
1774 nstat_client *client,
1775 nstat_msg_add_all_srcs *req)
1776 {
1777 // There is a tricky issue around getting all TCP sockets added once
1778 // and only once. nstat_tcp_new_pcb() is called prior to the new item
1779 // being placed on any lists where it might be found.
1780 // By locking the tcbinfo.ipi_lock prior to marking the client as a watcher,
1781 // it should be impossible for a new socket to be added twice.
1782 // On the other hand, there is still a timing issue where a new socket
1783 // results in a call to nstat_tcp_new_pcb() before this watcher
1784 // is instantiated and yet the socket doesn't make it into ipi_listhead
1785 // prior to the scan. <rdar://problem/30361716>
1786
1787 errno_t result;
1788
1789 lck_rw_lock_shared(&tcbinfo.ipi_lock);
1790 result = nstat_set_provider_filter(client, req);
1791 if (result == 0) {
1792 OSIncrementAtomic(&nstat_tcp_watchers);
1793
1794 // Add all current tcp inpcbs. Ignore those in timewait
1795 struct inpcb *inp;
1796 struct nstat_tucookie *cookie;
1797 LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list)
1798 {
1799 cookie = nstat_tucookie_alloc_ref(inp);
1800 if (cookie == NULL) {
1801 continue;
1802 }
1803 if (nstat_client_source_add(0, client, &nstat_tcp_provider,
1804 cookie, NSTAT_LOCK_NOTHELD) != 0) {
1805 nstat_tucookie_release(cookie);
1806 break;
1807 }
1808 }
1809 }
1810
1811 lck_rw_done(&tcbinfo.ipi_lock);
1812
1813 return result;
1814 }
1815
1816 static void
nstat_tcp_remove_watcher(__unused nstat_client * client)1817 nstat_tcp_remove_watcher(
1818 __unused nstat_client *client)
1819 {
1820 OSDecrementAtomic(&nstat_tcp_watchers);
1821 }
1822
1823 __private_extern__ void
nstat_tcp_new_pcb(struct inpcb * inp)1824 nstat_tcp_new_pcb(
1825 struct inpcb *inp)
1826 {
1827 struct nstat_tucookie *cookie;
1828
1829 inp->inp_start_timestamp = mach_continuous_time();
1830
1831 if (nstat_tcp_watchers == 0) {
1832 return;
1833 }
1834
1835 socket_lock(inp->inp_socket, 0);
1836 NSTAT_LOCK_EXCLUSIVE();
1837 nstat_client *client;
1838 for (client = nstat_clients; client; client = client->ntc_next) {
1839 if ((client->ntc_watching & (1 << NSTAT_PROVIDER_TCP_KERNEL)) != 0) {
1840 // this client is watching tcp
1841 // acquire a reference for it
1842 cookie = nstat_tucookie_alloc_ref_locked(inp);
1843 if (cookie == NULL) {
1844 continue;
1845 }
1846 // add the source, if that fails, release the reference
1847 if (nstat_client_source_add(0, client,
1848 &nstat_tcp_provider, cookie, NSTAT_LOCK_HELD) != 0) {
1849 nstat_tucookie_release_locked(cookie);
1850 }
1851 }
1852 }
1853 NSTAT_UNLOCK_EXCLUSIVE();
1854 socket_unlock(inp->inp_socket, 0);
1855 }
1856
1857 __private_extern__ void
nstat_pcb_detach(struct inpcb * inp)1858 nstat_pcb_detach(struct inpcb *inp)
1859 {
1860 nstat_client *client;
1861 nstat_src *src;
1862 tailq_head_nstat_src dead_list;
1863 struct nstat_tucookie *tucookie;
1864 errno_t result;
1865
1866 if (inp == NULL || (nstat_tcp_watchers == 0 && nstat_udp_watchers == 0)) {
1867 return;
1868 }
1869
1870 TAILQ_INIT(&dead_list);
1871 NSTAT_LOCK_EXCLUSIVE();
1872 for (client = nstat_clients; client; client = client->ntc_next) {
1873 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
1874 {
1875 nstat_provider_id_t provider_id = src->nts_provider->nstat_provider_id;
1876 if (provider_id == NSTAT_PROVIDER_TCP_KERNEL || provider_id == NSTAT_PROVIDER_UDP_KERNEL) {
1877 tucookie = (struct nstat_tucookie *)src->nts_cookie;
1878 if (tucookie->inp == inp) {
1879 break;
1880 }
1881 }
1882 }
1883
1884 if (src) {
1885 result = nstat_client_send_goodbye(client, src);
1886
1887 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
1888 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
1889 }
1890 }
1891 NSTAT_UNLOCK_EXCLUSIVE();
1892
1893 while ((src = TAILQ_FIRST(&dead_list))) {
1894 TAILQ_REMOVE(&dead_list, src, nts_client_link);
1895 nstat_client_cleanup_source(NULL, src, TRUE);
1896 }
1897 }
1898
1899 __private_extern__ void
nstat_pcb_event(struct inpcb * inp,u_int64_t event)1900 nstat_pcb_event(struct inpcb *inp, u_int64_t event)
1901 {
1902 nstat_client *client;
1903 nstat_src *src;
1904 struct nstat_tucookie *tucookie;
1905 errno_t result;
1906 nstat_provider_id_t provider_id;
1907
1908 if (inp == NULL || (nstat_tcp_watchers == 0 && nstat_udp_watchers == 0)) {
1909 return;
1910 }
1911 if (((merged_filters.mpf_filters[NSTAT_PROVIDER_TCP_KERNEL].mf_events & event) == 0) &&
1912 ((merged_filters.mpf_filters[NSTAT_PROVIDER_UDP_KERNEL].mf_events & event) == 0)) {
1913 // There are clients for TCP and UDP, but none are interested in the event
1914 // This check saves taking the mutex and scanning the list
1915 return;
1916 }
1917 NSTAT_LOCK_EXCLUSIVE();
1918 for (client = nstat_clients; client; client = client->ntc_next) {
1919 if (((client->ntc_provider_filters[NSTAT_PROVIDER_TCP_KERNEL].npf_events & event) == 0) &&
1920 ((client->ntc_provider_filters[NSTAT_PROVIDER_UDP_KERNEL].npf_events & event) == 0)) {
1921 continue;
1922 }
1923 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
1924 {
1925 provider_id = src->nts_provider->nstat_provider_id;
1926 if (provider_id == NSTAT_PROVIDER_TCP_KERNEL || provider_id == NSTAT_PROVIDER_UDP_KERNEL) {
1927 tucookie = (struct nstat_tucookie *)src->nts_cookie;
1928 if (tucookie->inp == inp) {
1929 break;
1930 }
1931 }
1932 }
1933
1934 if (src && ((client->ntc_provider_filters[provider_id].npf_events & event) != 0)) {
1935 result = nstat_client_send_event(client, src, event);
1936 }
1937 }
1938 NSTAT_UNLOCK_EXCLUSIVE();
1939 }
1940
1941
1942 __private_extern__ void
nstat_pcb_cache(struct inpcb * inp)1943 nstat_pcb_cache(struct inpcb *inp)
1944 {
1945 nstat_client *client;
1946 nstat_src *src;
1947 struct nstat_tucookie *tucookie;
1948
1949 if (inp == NULL || nstat_udp_watchers == 0 ||
1950 inp->inp_nstat_refcnt == 0) {
1951 return;
1952 }
1953 VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP);
1954 NSTAT_LOCK_EXCLUSIVE();
1955 for (client = nstat_clients; client; client = client->ntc_next) {
1956 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
1957 {
1958 tucookie = (struct nstat_tucookie *)src->nts_cookie;
1959 if (tucookie->inp == inp) {
1960 if (inp->inp_vflag & INP_IPV6) {
1961 in6_ip6_to_sockaddr(&inp->in6p_laddr,
1962 inp->inp_lport,
1963 inp->inp_lifscope,
1964 &tucookie->local.v6,
1965 sizeof(tucookie->local));
1966 in6_ip6_to_sockaddr(&inp->in6p_faddr,
1967 inp->inp_fport,
1968 inp->inp_fifscope,
1969 &tucookie->remote.v6,
1970 sizeof(tucookie->remote));
1971 } else if (inp->inp_vflag & INP_IPV4) {
1972 nstat_ip_to_sockaddr(&inp->inp_laddr,
1973 inp->inp_lport,
1974 &tucookie->local.v4,
1975 sizeof(tucookie->local));
1976 nstat_ip_to_sockaddr(&inp->inp_faddr,
1977 inp->inp_fport,
1978 &tucookie->remote.v4,
1979 sizeof(tucookie->remote));
1980 }
1981 if (inp->inp_last_outifp) {
1982 tucookie->if_index =
1983 inp->inp_last_outifp->if_index;
1984 }
1985
1986 tucookie->ifnet_properties = nstat_inpcb_to_flags(inp);
1987 tucookie->cached = true;
1988 break;
1989 }
1990 }
1991 }
1992 NSTAT_UNLOCK_EXCLUSIVE();
1993 }
1994
1995 __private_extern__ void
nstat_pcb_invalidate_cache(struct inpcb * inp)1996 nstat_pcb_invalidate_cache(struct inpcb *inp)
1997 {
1998 nstat_client *client;
1999 nstat_src *src;
2000 struct nstat_tucookie *tucookie;
2001
2002 if (inp == NULL || nstat_udp_watchers == 0 ||
2003 inp->inp_nstat_refcnt == 0) {
2004 return;
2005 }
2006 VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP);
2007 NSTAT_LOCK_EXCLUSIVE();
2008 for (client = nstat_clients; client; client = client->ntc_next) {
2009 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
2010 {
2011 tucookie = (struct nstat_tucookie *)src->nts_cookie;
2012 if (tucookie->inp == inp) {
2013 tucookie->cached = false;
2014 break;
2015 }
2016 }
2017 }
2018 NSTAT_UNLOCK_EXCLUSIVE();
2019 }
2020
2021 static errno_t
nstat_tcp_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,size_t len)2022 nstat_tcp_copy_descriptor(
2023 nstat_provider_cookie_t cookie,
2024 void *__sized_by(len)data,
2025 size_t len)
2026 {
2027 if (len < sizeof(nstat_tcp_descriptor)) {
2028 return EINVAL;
2029 }
2030
2031 if (nstat_tcp_gone(cookie)) {
2032 return EINVAL;
2033 }
2034
2035 nstat_tcp_descriptor *desc = (nstat_tcp_descriptor*)data;
2036 struct nstat_tucookie *tucookie =
2037 (struct nstat_tucookie *)cookie;
2038 struct inpcb *inp = tucookie->inp;
2039 struct tcpcb *tp = intotcpcb(inp);
2040 bzero(desc, sizeof(*desc));
2041
2042 if (inp->inp_vflag & INP_IPV6) {
2043 in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, inp->inp_lifscope,
2044 &desc->local.v6, sizeof(desc->local));
2045 in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, inp->inp_fifscope,
2046 &desc->remote.v6, sizeof(desc->remote));
2047 } else if (inp->inp_vflag & INP_IPV4) {
2048 nstat_ip_to_sockaddr(&inp->inp_laddr, inp->inp_lport,
2049 &desc->local.v4, sizeof(desc->local));
2050 nstat_ip_to_sockaddr(&inp->inp_faddr, inp->inp_fport,
2051 &desc->remote.v4, sizeof(desc->remote));
2052 }
2053
2054 desc->state = intotcpcb(inp)->t_state;
2055 desc->ifindex = (inp->inp_last_outifp == NULL) ? 0 :
2056 inp->inp_last_outifp->if_index;
2057
2058 // danger - not locked, values could be bogus
2059 desc->txunacked = tp->snd_max - tp->snd_una;
2060 desc->txwindow = tp->snd_wnd;
2061 desc->txcwindow = tp->snd_cwnd;
2062 desc->ifnet_properties = nstat_inpcb_to_flags(inp);
2063
2064 if (CC_ALGO(tp)->name != NULL) {
2065 strbufcpy(desc->cc_algo, CC_ALGO(tp)->name);
2066 }
2067
2068 struct socket *so = inp->inp_socket;
2069 if (so) {
2070 // TBD - take the socket lock around these to make sure
2071 // they're in sync?
2072 desc->upid = so->last_upid;
2073 desc->pid = so->last_pid;
2074 desc->traffic_class = so->so_traffic_class;
2075 if ((so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND)) {
2076 desc->traffic_mgt_flags |= TRAFFIC_MGT_SO_BACKGROUND;
2077 }
2078 if ((so->so_flags1 & SOF1_TRAFFIC_MGT_TCP_RECVBG)) {
2079 desc->traffic_mgt_flags |= TRAFFIC_MGT_TCP_RECVBG;
2080 }
2081 if (so->so_flags1 & SOF1_INBOUND) {
2082 desc->ifnet_properties |= NSTAT_SOURCE_IS_INBOUND;
2083 } else if (desc->state == TCPS_LISTEN) {
2084 desc->ifnet_properties |= NSTAT_SOURCE_IS_LISTENER;
2085 tucookie->ifnet_properties = NSTAT_SOURCE_IS_LISTENER;
2086 } else if (desc->state != TCPS_CLOSED) {
2087 desc->ifnet_properties |= NSTAT_SOURCE_IS_OUTBOUND;
2088 tucookie->ifnet_properties = NSTAT_SOURCE_IS_OUTBOUND;
2089 } else {
2090 desc->ifnet_properties |= tucookie->ifnet_properties;
2091 }
2092 proc_best_name_for_pid(desc->pid, desc->pname, sizeof(desc->pname));
2093 if (desc->pname[0] == 0) {
2094 strbufcpy(desc->pname, tucookie->pname);
2095 } else {
2096 desc->pname[sizeof(desc->pname) - 1] = 0;
2097 strbufcpy(tucookie->pname, desc->pname);
2098 }
2099 memcpy(desc->uuid, so->last_uuid, sizeof(so->last_uuid));
2100 memcpy(desc->vuuid, so->so_vuuid, sizeof(so->so_vuuid));
2101 if (so->so_flags & SOF_DELEGATED) {
2102 desc->eupid = so->e_upid;
2103 desc->epid = so->e_pid;
2104 memcpy(desc->euuid, so->e_uuid, sizeof(so->e_uuid));
2105 } else if (!uuid_is_null(so->so_ruuid)) {
2106 memcpy(desc->euuid, so->so_ruuid, sizeof(so->so_ruuid));
2107 } else {
2108 desc->eupid = desc->upid;
2109 desc->epid = desc->pid;
2110 memcpy(desc->euuid, desc->uuid, sizeof(desc->uuid));
2111 }
2112 uuid_copy(desc->fuuid, inp->necp_client_uuid);
2113 desc->persona_id = so->so_persona_id;
2114 desc->uid = kauth_cred_getuid(so->so_cred);
2115 desc->sndbufsize = so->so_snd.sb_hiwat;
2116 desc->sndbufused = so->so_snd.sb_cc;
2117 desc->rcvbufsize = so->so_rcv.sb_hiwat;
2118 desc->rcvbufused = so->so_rcv.sb_cc;
2119 desc->fallback_mode = so->so_fallback_mode;
2120
2121 if (nstat_debug) {
2122 uuid_string_t euuid_str = { 0 };
2123 uuid_unparse(desc->euuid, euuid_str);
2124 NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: TCP - pid %d uid %d euuid %s persona id %d", desc->pid, desc->uid, euuid_str, desc->persona_id);
2125 }
2126 }
2127
2128 tcp_get_connectivity_status(tp, &desc->connstatus);
2129 inp_get_activity_bitmap(inp, &desc->activity_bitmap);
2130 desc->start_timestamp = inp->inp_start_timestamp;
2131 desc->timestamp = mach_continuous_time();
2132 return 0;
2133 }
2134
2135 static bool
nstat_tcpudp_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,bool is_UDP)2136 nstat_tcpudp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter, bool is_UDP)
2137 {
2138 bool retval = true;
2139
2140 if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS | NSTAT_FILTER_SPECIFIC_USER)) != 0) {
2141 struct nstat_tucookie *tucookie = (struct nstat_tucookie *)cookie;
2142 struct inpcb *inp = tucookie->inp;
2143
2144 /* Only apply interface filter if at least one is allowed. */
2145 if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) {
2146 uint32_t interface_properties = nstat_inpcb_to_flags(inp);
2147
2148 if ((filter->npf_flags & interface_properties) == 0) {
2149 // For UDP, we could have an undefined interface and yet transfers may have occurred.
2150 // We allow reporting if there have been transfers of the requested kind.
2151 // This is imperfect as we cannot account for the expensive attribute over wifi.
2152 // We also assume that cellular is expensive and we have no way to select for AWDL
2153 if (is_UDP) {
2154 do{
2155 if ((filter->npf_flags & (NSTAT_FILTER_ACCEPT_CELLULAR | NSTAT_FILTER_ACCEPT_EXPENSIVE)) &&
2156 (inp->inp_cstat->rxbytes || inp->inp_cstat->txbytes)) {
2157 break;
2158 }
2159 if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_WIFI) &&
2160 (inp->inp_wstat->rxbytes || inp->inp_wstat->txbytes)) {
2161 break;
2162 }
2163 if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_WIRED) &&
2164 (inp->inp_Wstat->rxbytes || inp->inp_Wstat->txbytes)) {
2165 break;
2166 }
2167 if ((filter->npf_flags & NSTAT_FILTER_ACCEPT_COMPANIONLINK_BT) &&
2168 (inp->inp_btstat->rxbytes || inp->inp_btstat->txbytes)) {
2169 break;
2170 }
2171 return false;
2172 } while (0);
2173 } else {
2174 return false;
2175 }
2176 }
2177 }
2178
2179 if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) && (retval)) {
2180 struct socket *so = inp->inp_socket;
2181 retval = false;
2182
2183 if (so) {
2184 if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) &&
2185 (filter->npf_pid == so->last_pid)) {
2186 retval = true;
2187 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) &&
2188 (filter->npf_pid == (so->so_flags & SOF_DELEGATED)? so->e_upid : so->last_pid)) {
2189 retval = true;
2190 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) &&
2191 (memcmp(filter->npf_uuid, so->last_uuid, sizeof(so->last_uuid)) == 0)) {
2192 retval = true;
2193 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) &&
2194 (memcmp(filter->npf_uuid, (so->so_flags & SOF_DELEGATED)? so->e_uuid : so->last_uuid,
2195 sizeof(so->last_uuid)) == 0)) {
2196 retval = true;
2197 }
2198 }
2199 }
2200 }
2201 return retval;
2202 }
2203
2204 static bool
nstat_tcp_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)2205 nstat_tcp_reporting_allowed(
2206 nstat_provider_cookie_t cookie,
2207 nstat_provider_filter *filter,
2208 __unused u_int64_t suppression_flags)
2209 {
2210 return nstat_tcpudp_reporting_allowed(cookie, filter, FALSE);
2211 }
2212
2213 static size_t
nstat_tcp_extensions(nstat_provider_cookie_t cookie,u_int32_t extension_id,void * buf,size_t len)2214 nstat_tcp_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len)
2215 {
2216 struct nstat_tucookie *tucookie = (struct nstat_tucookie *)cookie;
2217 struct inpcb *inp = tucookie->inp;
2218
2219 if (nstat_tcp_gone(cookie)) {
2220 return 0;
2221 }
2222
2223 switch (extension_id) {
2224 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
2225 return nstat_inp_domain_info(inp, (nstat_domain_info *)buf, len);
2226
2227 case NSTAT_EXTENDED_UPDATE_TYPE_BLUETOOTH_COUNTS:
2228 return nstat_inp_bluetooth_counts(inp, (nstat_interface_counts *)buf, len);
2229
2230 case NSTAT_EXTENDED_UPDATE_TYPE_NECP_TLV:
2231 default:
2232 break;
2233 }
2234 return 0;
2235 }
2236
2237 static void
nstat_init_tcp_provider(void)2238 nstat_init_tcp_provider(void)
2239 {
2240 bzero(&nstat_tcp_provider, sizeof(nstat_tcp_provider));
2241 nstat_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor);
2242 nstat_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_KERNEL;
2243 nstat_tcp_provider.nstat_lookup = nstat_tcp_lookup;
2244 nstat_tcp_provider.nstat_gone = nstat_tcp_gone;
2245 nstat_tcp_provider.nstat_counts = nstat_tcp_counts;
2246 nstat_tcp_provider.nstat_release = nstat_tcp_release;
2247 nstat_tcp_provider.nstat_watcher_add = nstat_tcp_add_watcher;
2248 nstat_tcp_provider.nstat_watcher_remove = nstat_tcp_remove_watcher;
2249 nstat_tcp_provider.nstat_copy_descriptor = nstat_tcp_copy_descriptor;
2250 nstat_tcp_provider.nstat_reporting_allowed = nstat_tcp_reporting_allowed;
2251 nstat_tcp_provider.nstat_copy_extension = nstat_tcp_extensions;
2252 nstat_tcp_provider.next = nstat_providers;
2253 nstat_providers = &nstat_tcp_provider;
2254 }
2255
2256 #pragma mark -- UDP Provider --
2257
2258 static nstat_provider nstat_udp_provider;
2259
2260 static errno_t
nstat_udp_lookup(__unused const void * data,__unused u_int32_t length,__unused nstat_provider_cookie_t * out_cookie)2261 nstat_udp_lookup(
2262 __unused const void *data,
2263 __unused u_int32_t length,
2264 __unused nstat_provider_cookie_t *out_cookie)
2265 {
2266 // Looking up a specific connection is not supported.
2267 return ENOTSUP;
2268 }
2269
2270 static int
nstat_udp_gone(nstat_provider_cookie_t cookie)2271 nstat_udp_gone(
2272 nstat_provider_cookie_t cookie)
2273 {
2274 struct nstat_tucookie *tucookie =
2275 (struct nstat_tucookie *)cookie;
2276 struct inpcb *inp;
2277
2278 return (!(inp = tucookie->inp) ||
2279 inp->inp_state == INPCB_STATE_DEAD) ? 1 : 0;
2280 }
2281
2282 static errno_t
nstat_udp_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)2283 nstat_udp_counts(
2284 nstat_provider_cookie_t cookie,
2285 struct nstat_counts *out_counts,
2286 int *out_gone)
2287 {
2288 struct nstat_tucookie *tucookie =
2289 (struct nstat_tucookie *)cookie;
2290
2291 if (out_gone) {
2292 *out_gone = 0;
2293 }
2294
2295 // if the pcb is in the dead state, we should stop using it
2296 if (nstat_udp_gone(cookie)) {
2297 if (out_gone) {
2298 *out_gone = 1;
2299 }
2300 if (!tucookie->inp) {
2301 return EINVAL;
2302 }
2303 }
2304 struct inpcb *inp = tucookie->inp;
2305
2306 out_counts->nstat_rxpackets = os_atomic_load(&inp->inp_stat->rxpackets, relaxed);
2307 out_counts->nstat_rxbytes = os_atomic_load(&inp->inp_stat->rxbytes, relaxed);
2308 out_counts->nstat_txpackets = os_atomic_load(&inp->inp_stat->txpackets, relaxed);
2309 out_counts->nstat_txbytes = os_atomic_load(&inp->inp_stat->txbytes, relaxed);
2310 out_counts->nstat_cell_rxbytes = os_atomic_load(&inp->inp_cstat->rxbytes, relaxed);
2311 out_counts->nstat_cell_txbytes = os_atomic_load(&inp->inp_cstat->txbytes, relaxed);
2312 out_counts->nstat_wifi_rxbytes = os_atomic_load(&inp->inp_wstat->rxbytes, relaxed);
2313 out_counts->nstat_wifi_txbytes = os_atomic_load(&inp->inp_wstat->txbytes, relaxed);
2314 out_counts->nstat_wired_rxbytes = os_atomic_load(&inp->inp_Wstat->rxbytes, relaxed);
2315 out_counts->nstat_wired_txbytes = os_atomic_load(&inp->inp_Wstat->txbytes, relaxed);
2316
2317 return 0;
2318 }
2319
2320 static void
nstat_udp_release(nstat_provider_cookie_t cookie,int locked)2321 nstat_udp_release(
2322 nstat_provider_cookie_t cookie,
2323 int locked)
2324 {
2325 struct nstat_tucookie *tucookie =
2326 (struct nstat_tucookie *)cookie;
2327
2328 nstat_tucookie_release_internal(tucookie, locked);
2329 }
2330
2331 static errno_t
nstat_udp_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)2332 nstat_udp_add_watcher(
2333 nstat_client *client,
2334 nstat_msg_add_all_srcs *req)
2335 {
2336 // There is a tricky issue around getting all UDP sockets added once
2337 // and only once. nstat_udp_new_pcb() is called prior to the new item
2338 // being placed on any lists where it might be found.
2339 // By locking the udpinfo.ipi_lock prior to marking the client as a watcher,
2340 // it should be impossible for a new socket to be added twice.
2341 // On the other hand, there is still a timing issue where a new socket
2342 // results in a call to nstat_udp_new_pcb() before this watcher
2343 // is instantiated and yet the socket doesn't make it into ipi_listhead
2344 // prior to the scan. <rdar://problem/30361716>
2345
2346 errno_t result;
2347
2348 lck_rw_lock_shared(&udbinfo.ipi_lock);
2349 result = nstat_set_provider_filter(client, req);
2350
2351 if (result == 0) {
2352 struct inpcb *inp;
2353 struct nstat_tucookie *cookie;
2354
2355 OSIncrementAtomic(&nstat_udp_watchers);
2356
2357 // Add all current UDP inpcbs.
2358 LIST_FOREACH(inp, udbinfo.ipi_listhead, inp_list)
2359 {
2360 cookie = nstat_tucookie_alloc_ref(inp);
2361 if (cookie == NULL) {
2362 continue;
2363 }
2364 if (nstat_client_source_add(0, client, &nstat_udp_provider,
2365 cookie, NSTAT_LOCK_NOTHELD) != 0) {
2366 nstat_tucookie_release(cookie);
2367 break;
2368 }
2369 }
2370 }
2371
2372 lck_rw_done(&udbinfo.ipi_lock);
2373
2374 return result;
2375 }
2376
2377 static void
nstat_udp_remove_watcher(__unused nstat_client * client)2378 nstat_udp_remove_watcher(
2379 __unused nstat_client *client)
2380 {
2381 OSDecrementAtomic(&nstat_udp_watchers);
2382 }
2383
2384 __private_extern__ void
nstat_udp_new_pcb(struct inpcb * inp)2385 nstat_udp_new_pcb(
2386 struct inpcb *inp)
2387 {
2388 struct nstat_tucookie *cookie;
2389
2390 inp->inp_start_timestamp = mach_continuous_time();
2391
2392 if (nstat_udp_watchers == 0) {
2393 return;
2394 }
2395
2396 socket_lock(inp->inp_socket, 0);
2397 NSTAT_LOCK_EXCLUSIVE();
2398 nstat_client *client;
2399 for (client = nstat_clients; client; client = client->ntc_next) {
2400 if ((client->ntc_watching & (1 << NSTAT_PROVIDER_UDP_KERNEL)) != 0) {
2401 // this client is watching udp
2402 // acquire a reference for it
2403 cookie = nstat_tucookie_alloc_ref_locked(inp);
2404 if (cookie == NULL) {
2405 continue;
2406 }
2407 // add the source, if that fails, release the reference
2408 if (nstat_client_source_add(0, client,
2409 &nstat_udp_provider, cookie, NSTAT_LOCK_HELD) != 0) {
2410 nstat_tucookie_release_locked(cookie);
2411 }
2412 }
2413 }
2414 NSTAT_UNLOCK_EXCLUSIVE();
2415 socket_unlock(inp->inp_socket, 0);
2416 }
2417
2418 static errno_t
nstat_udp_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,size_t len)2419 nstat_udp_copy_descriptor(
2420 nstat_provider_cookie_t cookie,
2421 void *__sized_by(len)data,
2422 size_t len)
2423 {
2424 if (len < sizeof(nstat_udp_descriptor)) {
2425 return EINVAL;
2426 }
2427
2428 if (nstat_udp_gone(cookie)) {
2429 return EINVAL;
2430 }
2431
2432 struct nstat_tucookie *tucookie =
2433 (struct nstat_tucookie *)cookie;
2434 nstat_udp_descriptor *desc = (nstat_udp_descriptor*)data;
2435 struct inpcb *inp = tucookie->inp;
2436
2437 bzero(desc, sizeof(*desc));
2438
2439 if (tucookie->cached == false) {
2440 if (inp->inp_vflag & INP_IPV6) {
2441 in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, inp->inp_lifscope,
2442 &desc->local.v6, sizeof(desc->local.v6));
2443 in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, inp->inp_fifscope,
2444 &desc->remote.v6, sizeof(desc->remote.v6));
2445 } else if (inp->inp_vflag & INP_IPV4) {
2446 nstat_ip_to_sockaddr(&inp->inp_laddr, inp->inp_lport,
2447 &desc->local.v4, sizeof(desc->local.v4));
2448 nstat_ip_to_sockaddr(&inp->inp_faddr, inp->inp_fport,
2449 &desc->remote.v4, sizeof(desc->remote.v4));
2450 }
2451 desc->ifnet_properties = nstat_inpcb_to_flags(inp);
2452 } else {
2453 if (inp->inp_vflag & INP_IPV6) {
2454 memcpy(&desc->local.v6, &tucookie->local.v6,
2455 sizeof(desc->local.v6));
2456 memcpy(&desc->remote.v6, &tucookie->remote.v6,
2457 sizeof(desc->remote.v6));
2458 } else if (inp->inp_vflag & INP_IPV4) {
2459 memcpy(&desc->local.v4, &tucookie->local.v4,
2460 sizeof(desc->local.v4));
2461 memcpy(&desc->remote.v4, &tucookie->remote.v4,
2462 sizeof(desc->remote.v4));
2463 }
2464 desc->ifnet_properties = tucookie->ifnet_properties;
2465 }
2466
2467 if (inp->inp_last_outifp) {
2468 desc->ifindex = inp->inp_last_outifp->if_index;
2469 } else {
2470 desc->ifindex = tucookie->if_index;
2471 }
2472
2473 struct socket *so = inp->inp_socket;
2474 if (so) {
2475 // TBD - take the socket lock around these to make sure
2476 // they're in sync?
2477 desc->upid = so->last_upid;
2478 desc->pid = so->last_pid;
2479 proc_best_name_for_pid(desc->pid, desc->pname, sizeof(desc->pname));
2480 if (desc->pname[0] == 0) {
2481 strbufcpy(desc->pname, tucookie->pname);
2482 } else {
2483 desc->pname[sizeof(desc->pname) - 1] = 0;
2484 strbufcpy(tucookie->pname, desc->pname);
2485 }
2486 memcpy(desc->uuid, so->last_uuid, sizeof(so->last_uuid));
2487 memcpy(desc->vuuid, so->so_vuuid, sizeof(so->so_vuuid));
2488 if (so->so_flags & SOF_DELEGATED) {
2489 desc->eupid = so->e_upid;
2490 desc->epid = so->e_pid;
2491 memcpy(desc->euuid, so->e_uuid, sizeof(so->e_uuid));
2492 } else if (!uuid_is_null(so->so_ruuid)) {
2493 memcpy(desc->euuid, so->so_ruuid, sizeof(so->so_ruuid));
2494 } else {
2495 desc->eupid = desc->upid;
2496 desc->epid = desc->pid;
2497 memcpy(desc->euuid, desc->uuid, sizeof(desc->uuid));
2498 }
2499 uuid_copy(desc->fuuid, inp->necp_client_uuid);
2500 desc->persona_id = so->so_persona_id;
2501 desc->uid = kauth_cred_getuid(so->so_cred);
2502 desc->rcvbufsize = so->so_rcv.sb_hiwat;
2503 desc->rcvbufused = so->so_rcv.sb_cc;
2504 desc->traffic_class = so->so_traffic_class;
2505 desc->fallback_mode = so->so_fallback_mode;
2506 inp_get_activity_bitmap(inp, &desc->activity_bitmap);
2507 desc->start_timestamp = inp->inp_start_timestamp;
2508 desc->timestamp = mach_continuous_time();
2509
2510 if (nstat_debug) {
2511 uuid_string_t euuid_str = { 0 };
2512 uuid_unparse(desc->euuid, euuid_str);
2513 NSTAT_DEBUG_SOCKET_LOG(so, "NSTAT: UDP - pid %d uid %d euuid %s persona id %d", desc->pid, desc->uid, euuid_str, desc->persona_id);
2514 }
2515 }
2516
2517 return 0;
2518 }
2519
2520 static bool
nstat_udp_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)2521 nstat_udp_reporting_allowed(
2522 nstat_provider_cookie_t cookie,
2523 nstat_provider_filter *filter,
2524 __unused u_int64_t suppression_flags)
2525 {
2526 return nstat_tcpudp_reporting_allowed(cookie, filter, TRUE);
2527 }
2528
2529
2530 static size_t
nstat_udp_extensions(nstat_provider_cookie_t cookie,u_int32_t extension_id,void * buf,size_t len)2531 nstat_udp_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *buf, size_t len)
2532 {
2533 struct nstat_tucookie *tucookie = (struct nstat_tucookie *)cookie;
2534 struct inpcb *inp = tucookie->inp;
2535 if (nstat_udp_gone(cookie)) {
2536 return 0;
2537 }
2538
2539 switch (extension_id) {
2540 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
2541 return nstat_inp_domain_info(inp, (nstat_domain_info *)buf, len);
2542
2543 case NSTAT_EXTENDED_UPDATE_TYPE_BLUETOOTH_COUNTS:
2544 return nstat_inp_bluetooth_counts(inp, (nstat_interface_counts *)buf, len);
2545
2546 default:
2547 break;
2548 }
2549 return 0;
2550 }
2551
2552
2553 static void
nstat_init_udp_provider(void)2554 nstat_init_udp_provider(void)
2555 {
2556 bzero(&nstat_udp_provider, sizeof(nstat_udp_provider));
2557 nstat_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_KERNEL;
2558 nstat_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor);
2559 nstat_udp_provider.nstat_lookup = nstat_udp_lookup;
2560 nstat_udp_provider.nstat_gone = nstat_udp_gone;
2561 nstat_udp_provider.nstat_counts = nstat_udp_counts;
2562 nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher;
2563 nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher;
2564 nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor;
2565 nstat_udp_provider.nstat_release = nstat_udp_release;
2566 nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed;
2567 nstat_udp_provider.nstat_copy_extension = nstat_udp_extensions;
2568 nstat_udp_provider.next = nstat_providers;
2569 nstat_providers = &nstat_udp_provider;
2570 }
2571
2572 #if SKYWALK
2573
2574 #pragma mark -- TCP/UDP/QUIC Userland
2575
2576 // Almost all of this infrastucture is common to both TCP and UDP
2577
2578 static u_int32_t nstat_userland_quic_watchers = 0;
2579 static u_int32_t nstat_userland_udp_watchers = 0;
2580 static u_int32_t nstat_userland_tcp_watchers = 0;
2581
2582 static u_int32_t nstat_userland_quic_shadows = 0;
2583 static u_int32_t nstat_userland_udp_shadows = 0;
2584 static u_int32_t nstat_userland_tcp_shadows = 0;
2585
2586 static nstat_provider nstat_userland_quic_provider;
2587 static nstat_provider nstat_userland_udp_provider;
2588 static nstat_provider nstat_userland_tcp_provider;
2589
2590 enum nstat_rnf_override {
2591 nstat_rnf_override_not_set,
2592 nstat_rnf_override_enabled,
2593 nstat_rnf_override_disabled
2594 };
2595
2596 struct nstat_tu_shadow {
2597 tailq_entry_tu_shadow shad_link;
2598 userland_stats_request_vals_fn *shad_getvals_fn;
2599 userland_stats_request_extension_fn *shad_get_extension_fn;
2600 userland_stats_provider_context *shad_provider_context;
2601 u_int64_t shad_properties;
2602 u_int64_t shad_start_timestamp;
2603 nstat_provider_id_t shad_provider;
2604 struct nstat_procdetails *shad_procdetails;
2605 bool shad_live; // false if defunct
2606 enum nstat_rnf_override shad_rnf_override;
2607 uint32_t shad_magic;
2608 };
2609
2610 // Magic number checking should remain in place until the userland provider has been fully proven
2611 #define TU_SHADOW_MAGIC 0xfeedf00d
2612 #define TU_SHADOW_UNMAGIC 0xdeaddeed
2613
2614 static tailq_head_tu_shadow nstat_userprot_shad_head = TAILQ_HEAD_INITIALIZER(nstat_userprot_shad_head);
2615
2616 static errno_t
nstat_userland_tu_lookup(__unused const void * data,__unused u_int32_t length,__unused nstat_provider_cookie_t * out_cookie)2617 nstat_userland_tu_lookup(
2618 __unused const void *data,
2619 __unused u_int32_t length,
2620 __unused nstat_provider_cookie_t *out_cookie)
2621 {
2622 // Looking up a specific connection is not supported
2623 return ENOTSUP;
2624 }
2625
2626 static int
nstat_userland_tu_gone(__unused nstat_provider_cookie_t cookie)2627 nstat_userland_tu_gone(
2628 __unused nstat_provider_cookie_t cookie)
2629 {
2630 // Returns non-zero if the source has gone.
2631 // We don't keep a source hanging around, so the answer is always 0
2632 return 0;
2633 }
2634
2635 static errno_t
nstat_userland_tu_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)2636 nstat_userland_tu_counts(
2637 nstat_provider_cookie_t cookie,
2638 struct nstat_counts *out_counts,
2639 int *out_gone)
2640 {
2641 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2642 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2643 assert(shad->shad_live);
2644
2645 bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, out_counts, NULL);
2646
2647 if (out_gone) {
2648 *out_gone = 0;
2649 }
2650
2651 return (result)? 0 : EIO;
2652 }
2653
2654
2655 static errno_t
nstat_userland_tu_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,__unused size_t len)2656 nstat_userland_tu_copy_descriptor(
2657 nstat_provider_cookie_t cookie,
2658 void *__sized_by(len)data,
2659 __unused size_t len)
2660 {
2661 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2662 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2663 assert(shad->shad_live);
2664 struct nstat_procdetails *procdetails = shad->shad_procdetails;
2665 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
2666
2667 bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, data);
2668
2669 switch (shad->shad_provider) {
2670 case NSTAT_PROVIDER_TCP_USERLAND:
2671 {
2672 nstat_tcp_descriptor *desc = (nstat_tcp_descriptor *)data;
2673 desc->pid = procdetails->pdet_pid;
2674 desc->upid = procdetails->pdet_upid;
2675 uuid_copy(desc->uuid, procdetails->pdet_uuid);
2676 strbufcpy(desc->pname, procdetails->pdet_procname);
2677 if (shad->shad_rnf_override == nstat_rnf_override_enabled) {
2678 desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK;
2679 desc->fallback_mode = SO_FALLBACK_MODE_FAST;
2680 } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) {
2681 desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK;
2682 desc->fallback_mode = SO_FALLBACK_MODE_NONE;
2683 }
2684 desc->ifnet_properties |= (uint32_t)shad->shad_properties;
2685 desc->start_timestamp = shad->shad_start_timestamp;
2686 desc->timestamp = mach_continuous_time();
2687 }
2688 break;
2689 case NSTAT_PROVIDER_UDP_USERLAND:
2690 {
2691 nstat_udp_descriptor *desc = (nstat_udp_descriptor *)data;
2692 desc->pid = procdetails->pdet_pid;
2693 desc->upid = procdetails->pdet_upid;
2694 uuid_copy(desc->uuid, procdetails->pdet_uuid);
2695 strbufcpy(desc->pname, procdetails->pdet_procname);
2696 if (shad->shad_rnf_override == nstat_rnf_override_enabled) {
2697 desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK;
2698 desc->fallback_mode = SO_FALLBACK_MODE_FAST;
2699 } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) {
2700 desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK;
2701 desc->fallback_mode = SO_FALLBACK_MODE_NONE;
2702 }
2703 desc->ifnet_properties |= (uint32_t)shad->shad_properties;
2704 desc->start_timestamp = shad->shad_start_timestamp;
2705 desc->timestamp = mach_continuous_time();
2706 }
2707 break;
2708 case NSTAT_PROVIDER_QUIC_USERLAND:
2709 {
2710 nstat_quic_descriptor *desc = (nstat_quic_descriptor *)data;
2711 desc->pid = procdetails->pdet_pid;
2712 desc->upid = procdetails->pdet_upid;
2713 uuid_copy(desc->uuid, procdetails->pdet_uuid);
2714 strbufcpy(desc->pname, procdetails->pdet_procname);
2715 if (shad->shad_rnf_override == nstat_rnf_override_enabled) {
2716 desc->ifnet_properties |= NSTAT_IFNET_VIA_CELLFALLBACK;
2717 desc->fallback_mode = SO_FALLBACK_MODE_FAST;
2718 } else if (shad->shad_rnf_override == nstat_rnf_override_disabled) {
2719 desc->ifnet_properties &= ~NSTAT_IFNET_VIA_CELLFALLBACK;
2720 desc->fallback_mode = SO_FALLBACK_MODE_NONE;
2721 }
2722 desc->ifnet_properties |= (uint32_t)shad->shad_properties;
2723 desc->start_timestamp = shad->shad_start_timestamp;
2724 desc->timestamp = mach_continuous_time();
2725 }
2726 break;
2727 default:
2728 break;
2729 }
2730 return (result)? 0 : EIO;
2731 }
2732
2733 static void
nstat_userland_tu_release(__unused nstat_provider_cookie_t cookie,__unused int locked)2734 nstat_userland_tu_release(
2735 __unused nstat_provider_cookie_t cookie,
2736 __unused int locked)
2737 {
2738 // Called when a nstat_src is detached.
2739 // We don't reference count or ask for delayed release so nothing to do here.
2740 // Note that any associated nstat_tu_shadow may already have been released.
2741 }
2742
2743 static bool
check_reporting_for_user(nstat_provider_filter * filter,pid_t pid,pid_t epid,uuid_t * uuid,uuid_t * euuid)2744 check_reporting_for_user(nstat_provider_filter *filter, pid_t pid, pid_t epid, uuid_t *uuid, uuid_t *euuid)
2745 {
2746 bool retval = true;
2747
2748 if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) {
2749 retval = false;
2750
2751 if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) &&
2752 (filter->npf_pid == pid)) {
2753 retval = true;
2754 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) &&
2755 (filter->npf_pid == epid)) {
2756 retval = true;
2757 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) &&
2758 (memcmp(filter->npf_uuid, uuid, sizeof(*uuid)) == 0)) {
2759 retval = true;
2760 } else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) &&
2761 (memcmp(filter->npf_uuid, euuid, sizeof(*euuid)) == 0)) {
2762 retval = true;
2763 }
2764 }
2765 return retval;
2766 }
2767
2768 static bool
nstat_userland_tcp_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)2769 nstat_userland_tcp_reporting_allowed(
2770 nstat_provider_cookie_t cookie,
2771 nstat_provider_filter *filter,
2772 __unused u_int64_t suppression_flags)
2773 {
2774 bool retval = true;
2775 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2776
2777 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2778
2779 if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) {
2780 u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
2781
2782 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL)) {
2783 if ((filter->npf_flags & ifflags) == 0) {
2784 return false;
2785 }
2786 }
2787 }
2788
2789 if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) {
2790 nstat_tcp_descriptor tcp_desc; // Stack allocation - OK or pushing the limits too far?
2791 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, &tcp_desc)) {
2792 retval = check_reporting_for_user(filter, (pid_t)tcp_desc.pid, (pid_t)tcp_desc.epid,
2793 &tcp_desc.uuid, &tcp_desc.euuid);
2794 } else {
2795 retval = false; // No further information, so might as well give up now.
2796 }
2797 }
2798
2799 return retval;
2800 }
2801
2802 static size_t
nstat_userland_extensions(nstat_provider_cookie_t cookie,u_int32_t extension_id,void * __sized_by (len)buf,size_t len)2803 nstat_userland_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *__sized_by(len)buf, size_t len)
2804 {
2805 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2806 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2807 assert(shad->shad_live);
2808 assert(shad->shad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
2809
2810 return shad->shad_get_extension_fn(shad->shad_provider_context, extension_id, buf, len);
2811 }
2812
2813
2814 static bool
nstat_userland_udp_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)2815 nstat_userland_udp_reporting_allowed(
2816 nstat_provider_cookie_t cookie,
2817 nstat_provider_filter *filter,
2818 __unused u_int64_t suppression_flags)
2819 {
2820 bool retval = true;
2821 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2822
2823 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2824
2825 if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) {
2826 u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
2827
2828 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL)) {
2829 if ((filter->npf_flags & ifflags) == 0) {
2830 return false;
2831 }
2832 }
2833 }
2834 if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) {
2835 nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far?
2836 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, &udp_desc)) {
2837 retval = check_reporting_for_user(filter, (pid_t)udp_desc.pid, (pid_t)udp_desc.epid,
2838 &udp_desc.uuid, &udp_desc.euuid);
2839 } else {
2840 retval = false; // No further information, so might as well give up now.
2841 }
2842 }
2843 return retval;
2844 }
2845
2846 static bool
nstat_userland_quic_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,__unused u_int64_t suppression_flags)2847 nstat_userland_quic_reporting_allowed(
2848 nstat_provider_cookie_t cookie,
2849 nstat_provider_filter *filter,
2850 __unused u_int64_t suppression_flags)
2851 {
2852 bool retval = true;
2853 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie;
2854
2855 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2856
2857 if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) {
2858 u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
2859
2860 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL)) {
2861 if ((filter->npf_flags & ifflags) == 0) {
2862 return false;
2863 }
2864 }
2865 }
2866 if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) {
2867 nstat_quic_descriptor quic_desc; // Stack allocation - OK or pushing the limits too far?
2868 if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, NULL, NULL, &quic_desc)) {
2869 retval = check_reporting_for_user(filter, (pid_t)quic_desc.pid, (pid_t)quic_desc.epid,
2870 &quic_desc.uuid, &quic_desc.euuid);
2871 } else {
2872 retval = false; // No further information, so might as well give up now.
2873 }
2874 }
2875 return retval;
2876 }
2877
2878 static errno_t
nstat_userland_protocol_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req,nstat_provider_type_t nstat_provider_type,nstat_provider * nstat_provider,u_int32_t * proto_watcher_cnt)2879 nstat_userland_protocol_add_watcher(
2880 nstat_client *client,
2881 nstat_msg_add_all_srcs *req,
2882 nstat_provider_type_t nstat_provider_type,
2883 nstat_provider *nstat_provider,
2884 u_int32_t *proto_watcher_cnt)
2885 {
2886 errno_t result;
2887
2888 NSTAT_LOCK_EXCLUSIVE();
2889 result = nstat_set_provider_filter(client, req);
2890
2891 if (result == 0) {
2892 struct nstat_tu_shadow *shad;
2893
2894 OSIncrementAtomic(proto_watcher_cnt);
2895
2896 TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) {
2897 assert(shad->shad_magic == TU_SHADOW_MAGIC);
2898
2899 if ((shad->shad_provider == nstat_provider_type) && (shad->shad_live)) {
2900 result = nstat_client_source_add(0, client, nstat_provider, shad, NSTAT_LOCK_HELD);
2901 if (result != 0) {
2902 NSTAT_LOG_ERROR("nstat_client_source_add returned %d for "
2903 "provider type: %d", result, nstat_provider_type);
2904 break;
2905 }
2906 }
2907 }
2908 }
2909 NSTAT_UNLOCK_EXCLUSIVE();
2910
2911 return result;
2912 }
2913
2914 static errno_t
nstat_userland_tcp_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)2915 nstat_userland_tcp_add_watcher(
2916 nstat_client *client,
2917 nstat_msg_add_all_srcs *req)
2918 {
2919 return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_TCP_USERLAND,
2920 &nstat_userland_tcp_provider, &nstat_userland_tcp_watchers);
2921 }
2922
2923 static errno_t
nstat_userland_udp_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)2924 nstat_userland_udp_add_watcher(
2925 nstat_client *client,
2926 nstat_msg_add_all_srcs *req)
2927 {
2928 return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_UDP_USERLAND,
2929 &nstat_userland_udp_provider, &nstat_userland_udp_watchers);
2930 }
2931
2932 static errno_t
nstat_userland_quic_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)2933 nstat_userland_quic_add_watcher(
2934 nstat_client *client,
2935 nstat_msg_add_all_srcs *req)
2936 {
2937 return nstat_userland_protocol_add_watcher(client, req, NSTAT_PROVIDER_QUIC_USERLAND,
2938 &nstat_userland_quic_provider, &nstat_userland_quic_watchers);
2939 }
2940
2941 static void
nstat_userland_tcp_remove_watcher(__unused nstat_client * client)2942 nstat_userland_tcp_remove_watcher(
2943 __unused nstat_client *client)
2944 {
2945 OSDecrementAtomic(&nstat_userland_tcp_watchers);
2946 }
2947
2948 static void
nstat_userland_udp_remove_watcher(__unused nstat_client * client)2949 nstat_userland_udp_remove_watcher(
2950 __unused nstat_client *client)
2951 {
2952 OSDecrementAtomic(&nstat_userland_udp_watchers);
2953 }
2954
2955 static void
nstat_userland_quic_remove_watcher(__unused nstat_client * client)2956 nstat_userland_quic_remove_watcher(
2957 __unused nstat_client *client)
2958 {
2959 OSDecrementAtomic(&nstat_userland_quic_watchers);
2960 }
2961
2962
2963 static void
nstat_init_userland_tcp_provider(void)2964 nstat_init_userland_tcp_provider(void)
2965 {
2966 bzero(&nstat_userland_tcp_provider, sizeof(nstat_userland_tcp_provider));
2967 nstat_userland_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor);
2968 nstat_userland_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_USERLAND;
2969 nstat_userland_tcp_provider.nstat_lookup = nstat_userland_tu_lookup;
2970 nstat_userland_tcp_provider.nstat_gone = nstat_userland_tu_gone;
2971 nstat_userland_tcp_provider.nstat_counts = nstat_userland_tu_counts;
2972 nstat_userland_tcp_provider.nstat_release = nstat_userland_tu_release;
2973 nstat_userland_tcp_provider.nstat_watcher_add = nstat_userland_tcp_add_watcher;
2974 nstat_userland_tcp_provider.nstat_watcher_remove = nstat_userland_tcp_remove_watcher;
2975 nstat_userland_tcp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor;
2976 nstat_userland_tcp_provider.nstat_reporting_allowed = nstat_userland_tcp_reporting_allowed;
2977 nstat_userland_tcp_provider.nstat_copy_extension = nstat_userland_extensions;
2978 nstat_userland_tcp_provider.next = nstat_providers;
2979 nstat_providers = &nstat_userland_tcp_provider;
2980 }
2981
2982
2983 static void
nstat_init_userland_udp_provider(void)2984 nstat_init_userland_udp_provider(void)
2985 {
2986 bzero(&nstat_userland_udp_provider, sizeof(nstat_userland_udp_provider));
2987 nstat_userland_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor);
2988 nstat_userland_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_USERLAND;
2989 nstat_userland_udp_provider.nstat_lookup = nstat_userland_tu_lookup;
2990 nstat_userland_udp_provider.nstat_gone = nstat_userland_tu_gone;
2991 nstat_userland_udp_provider.nstat_counts = nstat_userland_tu_counts;
2992 nstat_userland_udp_provider.nstat_release = nstat_userland_tu_release;
2993 nstat_userland_udp_provider.nstat_watcher_add = nstat_userland_udp_add_watcher;
2994 nstat_userland_udp_provider.nstat_watcher_remove = nstat_userland_udp_remove_watcher;
2995 nstat_userland_udp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor;
2996 nstat_userland_udp_provider.nstat_reporting_allowed = nstat_userland_udp_reporting_allowed;
2997 nstat_userland_udp_provider.nstat_copy_extension = nstat_userland_extensions;
2998 nstat_userland_udp_provider.next = nstat_providers;
2999 nstat_providers = &nstat_userland_udp_provider;
3000 }
3001
3002 static void
nstat_init_userland_quic_provider(void)3003 nstat_init_userland_quic_provider(void)
3004 {
3005 bzero(&nstat_userland_quic_provider, sizeof(nstat_userland_quic_provider));
3006 nstat_userland_quic_provider.nstat_descriptor_length = sizeof(nstat_quic_descriptor);
3007 nstat_userland_quic_provider.nstat_provider_id = NSTAT_PROVIDER_QUIC_USERLAND;
3008 nstat_userland_quic_provider.nstat_lookup = nstat_userland_tu_lookup;
3009 nstat_userland_quic_provider.nstat_gone = nstat_userland_tu_gone;
3010 nstat_userland_quic_provider.nstat_counts = nstat_userland_tu_counts;
3011 nstat_userland_quic_provider.nstat_release = nstat_userland_tu_release;
3012 nstat_userland_quic_provider.nstat_watcher_add = nstat_userland_quic_add_watcher;
3013 nstat_userland_quic_provider.nstat_watcher_remove = nstat_userland_quic_remove_watcher;
3014 nstat_userland_quic_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor;
3015 nstat_userland_quic_provider.nstat_reporting_allowed = nstat_userland_quic_reporting_allowed;
3016 nstat_userland_quic_provider.nstat_copy_extension = nstat_userland_extensions;
3017 nstat_userland_quic_provider.next = nstat_providers;
3018 nstat_providers = &nstat_userland_quic_provider;
3019 }
3020
3021
3022 // Things get started with a call to netstats to say that there’s a new connection:
3023 __private_extern__ nstat_userland_context
ntstat_userland_stats_open(userland_stats_provider_context * ctx,int provider_id,u_int64_t properties,userland_stats_request_vals_fn req_fn,userland_stats_request_extension_fn req_extension_fn)3024 ntstat_userland_stats_open(userland_stats_provider_context *ctx,
3025 int provider_id,
3026 u_int64_t properties,
3027 userland_stats_request_vals_fn req_fn,
3028 userland_stats_request_extension_fn req_extension_fn)
3029 {
3030 struct nstat_tu_shadow *shad;
3031 struct nstat_procdetails *procdetails;
3032 nstat_provider *provider;
3033
3034 if ((provider_id != NSTAT_PROVIDER_TCP_USERLAND) &&
3035 (provider_id != NSTAT_PROVIDER_UDP_USERLAND) &&
3036 (provider_id != NSTAT_PROVIDER_QUIC_USERLAND)) {
3037 NSTAT_LOG_ERROR("incorrect provider is supplied, %d", provider_id);
3038 return NULL;
3039 }
3040
3041 shad = kalloc_type(struct nstat_tu_shadow, Z_WAITOK | Z_NOFAIL);
3042
3043 procdetails = nstat_retain_curprocdetails();
3044
3045 if (procdetails == NULL) {
3046 kfree_type(struct nstat_tu_shadow, shad);
3047 return NULL;
3048 }
3049
3050 shad->shad_getvals_fn = req_fn;
3051 shad->shad_get_extension_fn = req_extension_fn;
3052 shad->shad_provider_context = ctx;
3053 shad->shad_provider = provider_id;
3054 shad->shad_properties = properties;
3055 shad->shad_procdetails = procdetails;
3056 shad->shad_rnf_override = nstat_rnf_override_not_set;
3057 shad->shad_start_timestamp = mach_continuous_time();
3058 shad->shad_live = true;
3059 shad->shad_magic = TU_SHADOW_MAGIC;
3060
3061 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_tu_shad_allocs);
3062 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_tu_shad_current, nstat_global_tu_shad_max);
3063
3064 NSTAT_LOCK_EXCLUSIVE();
3065 nstat_client *client;
3066
3067 // Even if there are no watchers, we save the shadow structure
3068 TAILQ_INSERT_HEAD(&nstat_userprot_shad_head, shad, shad_link);
3069
3070 if (provider_id == NSTAT_PROVIDER_TCP_USERLAND) {
3071 nstat_userland_tcp_shadows++;
3072 provider = &nstat_userland_tcp_provider;
3073 } else if (provider_id == NSTAT_PROVIDER_UDP_USERLAND) {
3074 nstat_userland_udp_shadows++;
3075 provider = &nstat_userland_udp_provider;
3076 } else {
3077 nstat_userland_quic_shadows++;
3078 provider = &nstat_userland_quic_provider;
3079 }
3080
3081 for (client = nstat_clients; client; client = client->ntc_next) {
3082 if ((client->ntc_watching & (1 << provider_id)) != 0) {
3083 // this client is watching tcp/udp/quic userland
3084 // Link to it.
3085 int result = nstat_client_source_add(0, client, provider, shad, NSTAT_LOCK_HELD);
3086 if (result != 0) {
3087 // There should be some kind of statistics for failures like this.
3088 // <rdar://problem/31377195> The kernel ntstat component should keep some
3089 // internal counters reflecting operational state for eventual AWD reporting
3090 }
3091 }
3092 }
3093 NSTAT_UNLOCK_EXCLUSIVE();
3094
3095 return (nstat_userland_context)shad;
3096 }
3097
3098
3099 __private_extern__ void
ntstat_userland_stats_close(nstat_userland_context nstat_ctx)3100 ntstat_userland_stats_close(nstat_userland_context nstat_ctx)
3101 {
3102 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx;
3103 tailq_head_nstat_src dead_list;
3104 nstat_src *src;
3105
3106 if (shad == NULL) {
3107 return;
3108 }
3109
3110 assert(shad->shad_magic == TU_SHADOW_MAGIC);
3111 TAILQ_INIT(&dead_list);
3112
3113 NSTAT_LOCK_EXCLUSIVE();
3114 if (nstat_userland_udp_watchers != 0 ||
3115 nstat_userland_tcp_watchers != 0 ||
3116 nstat_userland_quic_watchers != 0) {
3117 nstat_client *client;
3118 errno_t result;
3119
3120 for (client = nstat_clients; client; client = client->ntc_next) {
3121 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
3122 {
3123 if (shad == (struct nstat_tu_shadow *)src->nts_cookie) {
3124 nstat_provider_id_t provider_id = src->nts_provider->nstat_provider_id;
3125 if (provider_id == NSTAT_PROVIDER_TCP_USERLAND ||
3126 provider_id == NSTAT_PROVIDER_UDP_USERLAND ||
3127 provider_id == NSTAT_PROVIDER_QUIC_USERLAND) {
3128 break;
3129 }
3130 }
3131 }
3132
3133 if (src) {
3134 result = nstat_client_send_goodbye(client, src);
3135
3136 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
3137 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
3138 }
3139 }
3140 }
3141 TAILQ_REMOVE(&nstat_userprot_shad_head, shad, shad_link);
3142
3143 if (shad->shad_live) {
3144 if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) {
3145 nstat_userland_tcp_shadows--;
3146 } else if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) {
3147 nstat_userland_udp_shadows--;
3148 } else {
3149 nstat_userland_quic_shadows--;
3150 }
3151 }
3152
3153 NSTAT_UNLOCK_EXCLUSIVE();
3154
3155 while ((src = TAILQ_FIRST(&dead_list))) {
3156 TAILQ_REMOVE(&dead_list, src, nts_client_link);
3157 nstat_client_cleanup_source(NULL, src, TRUE);
3158 }
3159 nstat_release_procdetails(shad->shad_procdetails);
3160 shad->shad_magic = TU_SHADOW_UNMAGIC;
3161 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_tu_shad_current);
3162 kfree_type(struct nstat_tu_shadow, shad);
3163 }
3164
3165 static void
ntstat_userland_stats_event_locked(struct nstat_tu_shadow * shad,uint64_t event)3166 ntstat_userland_stats_event_locked(
3167 struct nstat_tu_shadow *shad,
3168 uint64_t event)
3169 {
3170 nstat_client *client;
3171 nstat_src *src;
3172 errno_t result;
3173 nstat_provider_id_t provider_id;
3174
3175 NSTAT_ASSERT_LOCKED_EXCLUSIVE();
3176
3177 if (nstat_userland_udp_watchers != 0 || nstat_userland_tcp_watchers != 0 || nstat_userland_quic_watchers != 0) {
3178 for (client = nstat_clients; client; client = client->ntc_next) {
3179 if (((client->ntc_provider_filters[NSTAT_PROVIDER_TCP_USERLAND].npf_events & event) == 0) &&
3180 ((client->ntc_provider_filters[NSTAT_PROVIDER_UDP_USERLAND].npf_events & event) == 0) &&
3181 ((client->ntc_provider_filters[NSTAT_PROVIDER_QUIC_USERLAND].npf_events & event) == 0)) {
3182 continue;
3183 }
3184 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link) {
3185 provider_id = src->nts_provider->nstat_provider_id;
3186 if (provider_id == NSTAT_PROVIDER_TCP_USERLAND || provider_id == NSTAT_PROVIDER_UDP_USERLAND ||
3187 provider_id == NSTAT_PROVIDER_QUIC_USERLAND) {
3188 if (shad == (struct nstat_tu_shadow *)src->nts_cookie) {
3189 break;
3190 }
3191 }
3192 }
3193 if (src && ((client->ntc_provider_filters[provider_id].npf_events & event) != 0)) {
3194 result = nstat_client_send_event(client, src, event);
3195 }
3196 }
3197 }
3198 }
3199
3200 __private_extern__ void
ntstat_userland_stats_event(nstat_userland_context nstat_ctx,uint64_t event)3201 ntstat_userland_stats_event(
3202 nstat_userland_context nstat_ctx,
3203 uint64_t event)
3204 {
3205 // This will need refinement for when we do genuine stats filtering
3206 // See <rdar://problem/23022832> NetworkStatistics should provide opt-in notifications
3207 // For now it deals only with events that potentially cause any traditional netstat sources to be closed
3208
3209 struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx;
3210 tailq_head_nstat_src dead_list;
3211 nstat_src *src;
3212
3213 if (shad == NULL) {
3214 return;
3215 }
3216
3217 assert(shad->shad_magic == TU_SHADOW_MAGIC);
3218
3219 if (event & NECP_CLIENT_STATISTICS_EVENT_TIME_WAIT) {
3220 TAILQ_INIT(&dead_list);
3221
3222 NSTAT_LOCK_EXCLUSIVE();
3223 if (nstat_userland_udp_watchers != 0 ||
3224 nstat_userland_tcp_watchers != 0 ||
3225 nstat_userland_quic_watchers != 0) {
3226 nstat_client *client;
3227 errno_t result;
3228
3229 for (client = nstat_clients; client; client = client->ntc_next) {
3230 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
3231 {
3232 if (shad == (struct nstat_tu_shadow *)src->nts_cookie) {
3233 break;
3234 }
3235 }
3236
3237 if (src) {
3238 if (!(src->nts_filter & NSTAT_FILTER_TCP_NO_EARLY_CLOSE)) {
3239 result = nstat_client_send_goodbye(client, src);
3240
3241 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
3242 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
3243 }
3244 }
3245 }
3246 }
3247 NSTAT_UNLOCK_EXCLUSIVE();
3248
3249 while ((src = TAILQ_FIRST(&dead_list))) {
3250 TAILQ_REMOVE(&dead_list, src, nts_client_link);
3251 nstat_client_cleanup_source(NULL, src, TRUE);
3252 }
3253 }
3254 }
3255
3256 __private_extern__ void
nstats_userland_stats_defunct_for_process(int pid)3257 nstats_userland_stats_defunct_for_process(int pid)
3258 {
3259 // Note that this can be called multiple times for the same process
3260 tailq_head_nstat_src dead_list;
3261 nstat_src *src, *tmpsrc;
3262 struct nstat_tu_shadow *shad;
3263
3264 TAILQ_INIT(&dead_list);
3265
3266 NSTAT_LOCK_EXCLUSIVE();
3267
3268 if (nstat_userland_udp_watchers != 0 ||
3269 nstat_userland_tcp_watchers != 0 ||
3270 nstat_userland_quic_watchers != 0) {
3271 nstat_client *client;
3272 errno_t result;
3273
3274 for (client = nstat_clients; client; client = client->ntc_next) {
3275 TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc)
3276 {
3277 nstat_provider_id_t provider_id = src->nts_provider->nstat_provider_id;
3278 if (provider_id == NSTAT_PROVIDER_TCP_USERLAND ||
3279 provider_id == NSTAT_PROVIDER_UDP_USERLAND ||
3280 provider_id == NSTAT_PROVIDER_QUIC_USERLAND) {
3281 shad = (struct nstat_tu_shadow *)src->nts_cookie;
3282 if (shad->shad_procdetails->pdet_pid == pid) {
3283 result = nstat_client_send_goodbye(client, src);
3284
3285 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
3286 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
3287 }
3288 }
3289 }
3290 }
3291 }
3292
3293 TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) {
3294 assert(shad->shad_magic == TU_SHADOW_MAGIC);
3295
3296 if (shad->shad_live) {
3297 if (shad->shad_procdetails->pdet_pid == pid) {
3298 shad->shad_live = false;
3299 if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) {
3300 nstat_userland_tcp_shadows--;
3301 } else if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) {
3302 nstat_userland_udp_shadows--;
3303 } else {
3304 nstat_userland_quic_shadows--;
3305 }
3306 }
3307 }
3308 }
3309
3310 NSTAT_UNLOCK_EXCLUSIVE();
3311
3312 while ((src = TAILQ_FIRST(&dead_list))) {
3313 TAILQ_REMOVE(&dead_list, src, nts_client_link);
3314 nstat_client_cleanup_source(NULL, src, TRUE);
3315 }
3316 }
3317
3318 errno_t
nstat_userland_mark_rnf_override(uuid_t target_fuuid,bool rnf_override)3319 nstat_userland_mark_rnf_override(uuid_t target_fuuid, bool rnf_override)
3320 {
3321 // Note that this can be called multiple times for the same process
3322 struct nstat_tu_shadow *shad;
3323 uuid_t fuuid;
3324 errno_t result;
3325
3326 NSTAT_LOCK_EXCLUSIVE();
3327 // We set the fallback state regardles of watchers as there may be future ones that need to know
3328 TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) {
3329 assert(shad->shad_magic == TU_SHADOW_MAGIC);
3330 assert(shad->shad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
3331 if (shad->shad_get_extension_fn(shad->shad_provider_context, NSTAT_EXTENDED_UPDATE_TYPE_FUUID, fuuid, sizeof(fuuid))) {
3332 if (uuid_compare(fuuid, target_fuuid) == 0) {
3333 break;
3334 }
3335 }
3336 }
3337 if (shad) {
3338 if (shad->shad_procdetails->pdet_pid != proc_selfpid()) {
3339 result = EPERM;
3340 } else {
3341 result = 0;
3342 // It would be possible but awkward to check the previous value
3343 // for RNF override, and send an event only if changed.
3344 // In practice it's fine to send an event regardless,
3345 // which "pushes" the last statistics for the previous mode
3346 shad->shad_rnf_override = rnf_override ? nstat_rnf_override_enabled
3347 : nstat_rnf_override_disabled;
3348 ntstat_userland_stats_event_locked(shad,
3349 rnf_override ? NSTAT_EVENT_SRC_ENTER_CELLFALLBACK
3350 : NSTAT_EVENT_SRC_EXIT_CELLFALLBACK);
3351 }
3352 } else {
3353 result = EEXIST;
3354 }
3355
3356 NSTAT_UNLOCK_EXCLUSIVE();
3357
3358 return result;
3359 }
3360
3361 #pragma mark -- Generic Providers --
3362
3363 static nstat_provider nstat_userland_conn_provider;
3364 static nstat_provider nstat_udp_subflow_provider;
3365
3366 static u_int32_t nstat_generic_provider_watchers[NSTAT_PROVIDER_COUNT];
3367
3368 struct nstat_generic_shadow {
3369 tailq_entry_generic_shadow gshad_link;
3370 nstat_provider_context gshad_provider_context;
3371 nstat_provider_request_vals_fn *gshad_getvals_fn;
3372 nstat_provider_request_extensions_fn *gshad_getextensions_fn;
3373 u_int64_t gshad_properties;
3374 u_int64_t gshad_start_timestamp;
3375 struct nstat_procdetails *gshad_procdetails;
3376 nstat_provider_id_t gshad_provider;
3377 int32_t gshad_refcnt;
3378 uint32_t gshad_magic;
3379 };
3380
3381 // Magic number checking should remain in place until the userland provider has been fully proven
3382 #define NSTAT_GENERIC_SHADOW_MAGIC 0xfadef00d
3383 #define NSTAT_GENERIC_SHADOW_UNMAGIC 0xfadedead
3384
3385 static tailq_head_generic_shadow nstat_gshad_head = TAILQ_HEAD_INITIALIZER(nstat_gshad_head);
3386
3387 static inline void
nstat_retain_gshad(struct nstat_generic_shadow * gshad)3388 nstat_retain_gshad(
3389 struct nstat_generic_shadow *gshad)
3390 {
3391 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3392
3393 OSIncrementAtomic(&gshad->gshad_refcnt);
3394 }
3395
3396 static void
nstat_release_gshad(struct nstat_generic_shadow * gshad)3397 nstat_release_gshad(
3398 struct nstat_generic_shadow *gshad)
3399 {
3400 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3401
3402 if (OSDecrementAtomic(&gshad->gshad_refcnt) == 1) {
3403 nstat_release_procdetails(gshad->gshad_procdetails);
3404 gshad->gshad_magic = NSTAT_GENERIC_SHADOW_UNMAGIC;
3405 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_gshad_current);
3406 kfree_type(struct nstat_generic_shadow, gshad);
3407 }
3408 }
3409
3410 static errno_t
nstat_generic_provider_lookup(__unused const void * data,__unused u_int32_t length,__unused nstat_provider_cookie_t * out_cookie)3411 nstat_generic_provider_lookup(
3412 __unused const void *data,
3413 __unused u_int32_t length,
3414 __unused nstat_provider_cookie_t *out_cookie)
3415 {
3416 // Looking up a specific connection is not supported
3417 return ENOTSUP;
3418 }
3419
3420 static int
nstat_generic_provider_gone(__unused nstat_provider_cookie_t cookie)3421 nstat_generic_provider_gone(
3422 __unused nstat_provider_cookie_t cookie)
3423 {
3424 // Returns non-zero if the source has gone.
3425 // We don't keep a source hanging around, so the answer is always 0
3426 return 0;
3427 }
3428
3429 static errno_t
nstat_generic_provider_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)3430 nstat_generic_provider_counts(
3431 nstat_provider_cookie_t cookie,
3432 struct nstat_counts *out_counts,
3433 int *out_gone)
3434 {
3435 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie;
3436 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3437
3438 memset(out_counts, 0, sizeof(*out_counts));
3439
3440 bool result = (*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, out_counts, NULL);
3441
3442 if (out_gone) {
3443 *out_gone = 0;
3444 }
3445 return (result)? 0 : EIO;
3446 }
3447
3448
3449 static errno_t
nstat_generic_provider_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,__unused size_t len)3450 nstat_generic_provider_copy_descriptor(
3451 nstat_provider_cookie_t cookie,
3452 void *__sized_by(len)data,
3453 __unused size_t len)
3454 {
3455 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie;
3456 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3457 struct nstat_procdetails *procdetails = gshad->gshad_procdetails;
3458 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
3459
3460 bool result = (*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, NULL, data);
3461
3462 switch (gshad->gshad_provider) {
3463 case NSTAT_PROVIDER_CONN_USERLAND:
3464 {
3465 nstat_connection_descriptor *desc = (nstat_connection_descriptor *)data;
3466 desc->pid = procdetails->pdet_pid;
3467 desc->upid = procdetails->pdet_upid;
3468 uuid_copy(desc->uuid, procdetails->pdet_uuid);
3469 strbufcpy(desc->pname, procdetails->pdet_procname);
3470 desc->start_timestamp = gshad->gshad_start_timestamp;
3471 desc->timestamp = mach_continuous_time();
3472 break;
3473 }
3474 case NSTAT_PROVIDER_UDP_SUBFLOW:
3475 {
3476 nstat_udp_descriptor *desc = (nstat_udp_descriptor *)data;
3477 desc->pid = procdetails->pdet_pid;
3478 desc->upid = procdetails->pdet_upid;
3479 uuid_copy(desc->uuid, procdetails->pdet_uuid);
3480 strbufcpy(desc->pname, procdetails->pdet_procname);
3481 desc->start_timestamp = gshad->gshad_start_timestamp;
3482 desc->timestamp = mach_continuous_time();
3483 break;
3484 }
3485 default:
3486 break;
3487 }
3488 return (result)? 0 : EIO;
3489 }
3490
3491 static void
nstat_generic_provider_release(__unused nstat_provider_cookie_t cookie,__unused int locked)3492 nstat_generic_provider_release(
3493 __unused nstat_provider_cookie_t cookie,
3494 __unused int locked)
3495 {
3496 // Called when a nstat_src is detached.
3497 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie;
3498
3499 nstat_release_gshad(gshad);
3500 }
3501
3502 static bool
nstat_generic_provider_reporting_allowed(nstat_provider_cookie_t cookie,nstat_provider_filter * filter,u_int64_t suppression_flags)3503 nstat_generic_provider_reporting_allowed(
3504 nstat_provider_cookie_t cookie,
3505 nstat_provider_filter *filter,
3506 u_int64_t suppression_flags)
3507 {
3508 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie;
3509
3510 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3511
3512 if ((filter->npf_flags & NSTAT_FILTER_SUPPRESS_BORING_FLAGS) != 0) {
3513 if ((filter->npf_flags & suppression_flags) != 0) {
3514 return false;
3515 }
3516 }
3517
3518 // Filter based on interface and connection flags
3519 // If a provider doesn't support flags, a client shouldn't attempt to use filtering
3520 if ((filter->npf_flags & NSTAT_FILTER_IFNET_AND_CONN_FLAGS) != 0) {
3521 u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
3522
3523 if ((*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, &ifflags, NULL, NULL)) {
3524 if ((filter->npf_flags & ifflags) == 0) {
3525 return false;
3526 }
3527 }
3528 }
3529
3530 if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) {
3531 struct nstat_procdetails *procdetails = gshad->gshad_procdetails;
3532 assert(procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
3533
3534 // Check details that we have readily to hand before asking the provider for descriptor items
3535 if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) &&
3536 (filter->npf_pid == procdetails->pdet_pid)) {
3537 return true;
3538 }
3539 if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) &&
3540 (memcmp(filter->npf_uuid, &procdetails->pdet_uuid, sizeof(filter->npf_uuid)) == 0)) {
3541 return true;
3542 }
3543 if ((filter->npf_flags & (NSTAT_FILTER_SPECIFIC_USER_BY_EPID | NSTAT_FILTER_SPECIFIC_USER_BY_EUUID)) != 0) {
3544 nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far?
3545 switch (gshad->gshad_provider) {
3546 case NSTAT_PROVIDER_CONN_USERLAND:
3547 // Filtering by effective uuid or effective pid is currently not supported
3548 filter->npf_flags &= ~((uint64_t)(NSTAT_FILTER_SPECIFIC_USER_BY_EPID | NSTAT_FILTER_SPECIFIC_USER_BY_EUUID));
3549 NSTAT_LOG("attempt to filter conn provider by effective pid/uuid, not supported");
3550 return true;
3551
3552 case NSTAT_PROVIDER_UDP_SUBFLOW:
3553 if ((*gshad->gshad_getvals_fn)(gshad->gshad_provider_context, NULL, NULL, &udp_desc)) {
3554 if (check_reporting_for_user(filter, procdetails->pdet_pid, (pid_t)udp_desc.epid,
3555 &procdetails->pdet_uuid, &udp_desc.euuid)) {
3556 return true;
3557 }
3558 }
3559 break;
3560 default:
3561 break;
3562 }
3563 }
3564 return false;
3565 }
3566 return true;
3567 }
3568
3569 static size_t
nstat_generic_extensions(nstat_provider_cookie_t cookie,u_int32_t extension_id,void * __sized_by (len)buf,size_t len)3570 nstat_generic_extensions(nstat_provider_cookie_t cookie, u_int32_t extension_id, void *__sized_by(len)buf, size_t len)
3571 {
3572 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)cookie;
3573 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3574 assert(gshad->gshad_procdetails->pdet_magic == NSTAT_PROCDETAILS_MAGIC);
3575
3576 if (gshad->gshad_getextensions_fn == NULL) {
3577 return 0;
3578 }
3579 return gshad->gshad_getextensions_fn(gshad->gshad_provider_context, extension_id, buf, len);
3580 }
3581
3582 static errno_t
nstat_generic_provider_add_watcher(nstat_client * client,nstat_msg_add_all_srcs * req)3583 nstat_generic_provider_add_watcher(
3584 nstat_client *client,
3585 nstat_msg_add_all_srcs *req)
3586 {
3587 errno_t result;
3588 nstat_provider_id_t provider_id = req->provider;
3589 nstat_provider *provider;
3590
3591 switch (provider_id) {
3592 case NSTAT_PROVIDER_CONN_USERLAND:
3593 provider = &nstat_userland_conn_provider;
3594 break;
3595 case NSTAT_PROVIDER_UDP_SUBFLOW:
3596 provider = &nstat_udp_subflow_provider;
3597 break;
3598 default:
3599 return ENOTSUP;
3600 }
3601
3602 NSTAT_LOCK_EXCLUSIVE();
3603 result = nstat_set_provider_filter(client, req);
3604
3605 if (result == 0) {
3606 struct nstat_generic_shadow *gshad;
3607 nstat_provider_filter *filter = &client->ntc_provider_filters[provider_id];
3608
3609 OSIncrementAtomic(&nstat_generic_provider_watchers[provider_id]);
3610
3611 TAILQ_FOREACH(gshad, &nstat_gshad_head, gshad_link) {
3612 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3613
3614 if (gshad->gshad_provider == provider_id) {
3615 if (filter->npf_flags & NSTAT_FILTER_INITIAL_PROPERTIES) {
3616 u_int64_t npf_flags = filter->npf_flags & NSTAT_FILTER_IFNET_AND_CONN_FLAGS;
3617 if ((npf_flags != 0) && ((npf_flags & gshad->gshad_properties) == 0)) {
3618 // Skip this one
3619 // Note - no filtering by pid or UUID supported at this point, for simplicity
3620 continue;
3621 }
3622 }
3623 nstat_retain_gshad(gshad);
3624 result = nstat_client_source_add(0, client, provider, gshad, NSTAT_LOCK_HELD);
3625 if (result != 0) {
3626 NSTAT_LOG_ERROR("nstat_client_source_add returned %d for "
3627 "provider type: %d", result, provider_id);
3628 nstat_release_gshad(gshad);
3629 break;
3630 }
3631 }
3632 }
3633 }
3634 NSTAT_UNLOCK_EXCLUSIVE();
3635
3636 return result;
3637 }
3638
3639 static void
nstat_userland_conn_remove_watcher(__unused nstat_client * client)3640 nstat_userland_conn_remove_watcher(
3641 __unused nstat_client *client)
3642 {
3643 OSDecrementAtomic(&nstat_generic_provider_watchers[NSTAT_PROVIDER_CONN_USERLAND]);
3644 }
3645
3646 static void
nstat_udp_subflow_remove_watcher(__unused nstat_client * client)3647 nstat_udp_subflow_remove_watcher(
3648 __unused nstat_client *client)
3649 {
3650 OSDecrementAtomic(&nstat_generic_provider_watchers[NSTAT_PROVIDER_UDP_SUBFLOW]);
3651 }
3652
3653 static void
nstat_init_userland_conn_provider(void)3654 nstat_init_userland_conn_provider(void)
3655 {
3656 bzero(&nstat_userland_conn_provider, sizeof(nstat_userland_conn_provider));
3657 nstat_userland_conn_provider.nstat_descriptor_length = sizeof(nstat_connection_descriptor);
3658 nstat_userland_conn_provider.nstat_provider_id = NSTAT_PROVIDER_CONN_USERLAND;
3659 nstat_userland_conn_provider.nstat_lookup = nstat_generic_provider_lookup;
3660 nstat_userland_conn_provider.nstat_gone = nstat_generic_provider_gone;
3661 nstat_userland_conn_provider.nstat_counts = nstat_generic_provider_counts;
3662 nstat_userland_conn_provider.nstat_release = nstat_generic_provider_release;
3663 nstat_userland_conn_provider.nstat_watcher_add = nstat_generic_provider_add_watcher;
3664 nstat_userland_conn_provider.nstat_watcher_remove = nstat_userland_conn_remove_watcher;
3665 nstat_userland_conn_provider.nstat_copy_descriptor = nstat_generic_provider_copy_descriptor;
3666 nstat_userland_conn_provider.nstat_reporting_allowed = nstat_generic_provider_reporting_allowed;
3667 nstat_userland_conn_provider.nstat_copy_extension = nstat_generic_extensions;
3668 nstat_userland_conn_provider.next = nstat_providers;
3669 nstat_providers = &nstat_userland_conn_provider;
3670 }
3671
3672 static void
nstat_init_udp_subflow_provider(void)3673 nstat_init_udp_subflow_provider(void)
3674 {
3675 bzero(&nstat_udp_subflow_provider, sizeof(nstat_udp_subflow_provider));
3676 nstat_udp_subflow_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor);
3677 nstat_udp_subflow_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_SUBFLOW;
3678 nstat_udp_subflow_provider.nstat_lookup = nstat_generic_provider_lookup;
3679 nstat_udp_subflow_provider.nstat_gone = nstat_generic_provider_gone;
3680 nstat_udp_subflow_provider.nstat_counts = nstat_generic_provider_counts;
3681 nstat_udp_subflow_provider.nstat_release = nstat_generic_provider_release;
3682 nstat_udp_subflow_provider.nstat_watcher_add = nstat_generic_provider_add_watcher;
3683 nstat_udp_subflow_provider.nstat_watcher_remove = nstat_udp_subflow_remove_watcher;
3684 nstat_udp_subflow_provider.nstat_copy_descriptor = nstat_generic_provider_copy_descriptor;
3685 nstat_udp_subflow_provider.nstat_reporting_allowed = nstat_generic_provider_reporting_allowed;
3686 nstat_udp_subflow_provider.nstat_copy_extension = nstat_generic_extensions;
3687 nstat_udp_subflow_provider.next = nstat_providers;
3688 nstat_providers = &nstat_udp_subflow_provider;
3689 }
3690
3691 // Things get started with a call from the provider to netstats to say that there’s a new source
3692 __private_extern__ nstat_context
nstat_provider_stats_open(nstat_provider_context ctx,int provider_id,u_int64_t properties,nstat_provider_request_vals_fn req_fn,nstat_provider_request_extensions_fn req_extensions_fn)3693 nstat_provider_stats_open(nstat_provider_context ctx,
3694 int provider_id,
3695 u_int64_t properties,
3696 nstat_provider_request_vals_fn req_fn,
3697 nstat_provider_request_extensions_fn req_extensions_fn)
3698 {
3699 struct nstat_generic_shadow *gshad;
3700 struct nstat_procdetails *procdetails;
3701 nstat_provider *provider = nstat_find_provider_by_id(provider_id);
3702
3703 gshad = kalloc_type(struct nstat_generic_shadow, Z_WAITOK | Z_NOFAIL);
3704
3705 procdetails = nstat_retain_curprocdetails();
3706
3707 if (procdetails == NULL) {
3708 kfree_type(struct nstat_generic_shadow, gshad);
3709 return NULL;
3710 }
3711
3712 gshad->gshad_getvals_fn = req_fn;
3713 gshad->gshad_getextensions_fn = req_extensions_fn;
3714 gshad->gshad_provider_context = ctx;
3715 gshad->gshad_properties = properties;
3716 gshad->gshad_procdetails = procdetails;
3717 gshad->gshad_provider = provider_id;
3718 gshad->gshad_start_timestamp = mach_continuous_time();
3719 gshad->gshad_refcnt = 0;
3720 gshad->gshad_magic = NSTAT_GENERIC_SHADOW_MAGIC;
3721 nstat_retain_gshad(gshad);
3722 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_gshad_allocs);
3723 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_gshad_current, nstat_global_gshad_max);
3724
3725 NSTAT_LOCK_EXCLUSIVE();
3726 nstat_client *client;
3727
3728 // Even if there are no watchers, we save the shadow structure
3729 TAILQ_INSERT_HEAD(&nstat_gshad_head, gshad, gshad_link);
3730
3731 for (client = nstat_clients; client; client = client->ntc_next) {
3732 if ((client->ntc_watching & (1 << provider_id)) != 0) {
3733 // Does this client want an initial filtering to be made?
3734 u_int64_t npf_flags = client->ntc_provider_filters[provider->nstat_provider_id].npf_flags;
3735 if (npf_flags & NSTAT_FILTER_INITIAL_PROPERTIES) {
3736 npf_flags &= NSTAT_FILTER_IFNET_AND_CONN_FLAGS;
3737 if ((npf_flags != 0) && ((npf_flags & properties) == 0)) {
3738 // Skip this one
3739 // Note - no filtering by pid or UUID supported at this point, for simplicity
3740 continue;
3741 }
3742 }
3743 // this client is watching, so link to it.
3744 nstat_retain_gshad(gshad);
3745 int result = nstat_client_source_add(0, client, provider, gshad, NSTAT_LOCK_HELD);
3746 if (result != 0) {
3747 // There should be some kind of statistics for failures like this.
3748 // <rdar://problem/31377195> The kernel ntstat component should keep some
3749 // internal counters reflecting operational state for eventual AWD reporting
3750 nstat_release_gshad(gshad);
3751 }
3752 }
3753 }
3754 NSTAT_UNLOCK_EXCLUSIVE();
3755
3756 return (nstat_context) gshad;
3757 }
3758
3759
3760 // When the source is closed, netstats will make one last call on the request functions to retrieve final values
3761 __private_extern__ void
nstat_provider_stats_close(nstat_context nstat_ctx)3762 nstat_provider_stats_close(nstat_context nstat_ctx)
3763 {
3764 tailq_head_nstat_src dead_list;
3765 nstat_src *src;
3766 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)nstat_ctx;
3767
3768 if (gshad == NULL) {
3769 NSTAT_LOG_ERROR("called with null reference");
3770 return;
3771 }
3772
3773 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3774
3775 if (gshad->gshad_magic != NSTAT_GENERIC_SHADOW_MAGIC) {
3776 NSTAT_LOG_ERROR("called with incorrect shadow magic 0x%x", gshad->gshad_magic);
3777 }
3778
3779 TAILQ_INIT(&dead_list);
3780
3781 NSTAT_LOCK_EXCLUSIVE();
3782
3783 TAILQ_REMOVE(&nstat_gshad_head, gshad, gshad_link);
3784
3785 int32_t num_srcs = gshad->gshad_refcnt - 1;
3786 if ((nstat_generic_provider_watchers[gshad->gshad_provider] != 0) && (num_srcs > 0)) {
3787 nstat_client *client;
3788 errno_t result;
3789
3790 for (client = nstat_clients; client; client = client->ntc_next) {
3791 // Only scan further if this client is watching
3792 if ((client->ntc_watching & (1 << gshad->gshad_provider)) != 0) {
3793 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
3794 {
3795 if ((gshad == (struct nstat_generic_shadow *)src->nts_cookie) &&
3796 (gshad->gshad_provider == src->nts_provider->nstat_provider_id)) {
3797 break;
3798 }
3799 }
3800 if (src) {
3801 result = nstat_client_send_goodbye(client, src);
3802 // There is currently no recovery possible from failure to send,
3803 // so no need to check the return code.
3804 // rdar://28312774 (Scalability and resilience issues in ntstat.c)
3805
3806 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
3807 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
3808 --num_srcs;
3809 }
3810
3811 // Performance optimization, don't scan full lists if no chance of presence
3812 if (num_srcs == 0) {
3813 break;
3814 }
3815 }
3816 }
3817 }
3818 NSTAT_UNLOCK_EXCLUSIVE();
3819
3820 while ((src = TAILQ_FIRST(&dead_list))) {
3821 TAILQ_REMOVE(&dead_list, src, nts_client_link);
3822 nstat_client_cleanup_source(NULL, src, TRUE);
3823 }
3824 nstat_release_gshad(gshad);
3825 }
3826
3827 // Events that cause a significant change may be reported via a flags word
3828 void
nstat_provider_stats_event(__unused nstat_context nstat_ctx,__unused uint64_t event)3829 nstat_provider_stats_event(__unused nstat_context nstat_ctx, __unused uint64_t event)
3830 {
3831 nstat_src *src;
3832 struct nstat_generic_shadow *gshad = (struct nstat_generic_shadow *)nstat_ctx;
3833
3834 if (gshad == NULL) {
3835 NSTAT_LOG_ERROR("called with null reference");
3836 return;
3837 }
3838
3839 assert(gshad->gshad_magic == NSTAT_GENERIC_SHADOW_MAGIC);
3840
3841 if (gshad->gshad_magic != NSTAT_GENERIC_SHADOW_MAGIC) {
3842 NSTAT_LOG_ERROR("called with incorrect shadow magic 0x%x", gshad->gshad_magic);
3843 }
3844
3845 NSTAT_LOCK_EXCLUSIVE();
3846
3847 if (nstat_generic_provider_watchers[gshad->gshad_provider] != 0) {
3848 nstat_client *client;
3849 errno_t result;
3850 nstat_provider_id_t provider_id = gshad->gshad_provider;
3851
3852 for (client = nstat_clients; client; client = client->ntc_next) {
3853 // Only scan further if this client is watching and has interest in the event
3854 // or the client has requested "boring" unchanged status to be ignored
3855 if (((client->ntc_watching & (1 << provider_id)) != 0) &&
3856 (((client->ntc_provider_filters[provider_id].npf_events & event) != 0) ||
3857 ((client->ntc_provider_filters[provider_id].npf_flags & NSTAT_FILTER_SUPPRESS_BORING_FLAGS) != 0))) {
3858 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
3859 {
3860 if (gshad == (struct nstat_generic_shadow *)src->nts_cookie) {
3861 break;
3862 }
3863 }
3864
3865 if (src) {
3866 src->nts_reported = false;
3867 if ((client->ntc_provider_filters[provider_id].npf_events & event) != 0) {
3868 result = nstat_client_send_event(client, src, event);
3869 // There is currently no recovery possible from failure to send,
3870 // so no need to check the return code.
3871 // rdar://28312774 (Scalability and resilience issues in ntstat.c)
3872 }
3873 }
3874 }
3875 }
3876 }
3877 NSTAT_UNLOCK_EXCLUSIVE();
3878 }
3879
3880 #endif /* SKYWALK */
3881
3882
3883 #pragma mark -- ifnet Provider --
3884
3885 static nstat_provider nstat_ifnet_provider;
3886
3887 /*
3888 * We store a pointer to the ifnet and the original threshold
3889 * requested by the client.
3890 */
3891 struct nstat_ifnet_cookie {
3892 struct ifnet *ifp;
3893 uint64_t threshold;
3894 };
3895
3896 static errno_t
nstat_ifnet_lookup(const void * data,u_int32_t length,nstat_provider_cookie_t * out_cookie)3897 nstat_ifnet_lookup(
3898 const void *data,
3899 u_int32_t length,
3900 nstat_provider_cookie_t *out_cookie)
3901 {
3902 const nstat_ifnet_add_param *param = (const nstat_ifnet_add_param *)data;
3903 struct ifnet *ifp;
3904 boolean_t changed = FALSE;
3905 nstat_client *client;
3906 nstat_src *src;
3907 struct nstat_ifnet_cookie *cookie;
3908
3909 if (length < sizeof(*param) || param->threshold < 1024 * 1024) {
3910 return EINVAL;
3911 }
3912 if (nstat_privcheck != 0) {
3913 errno_t result = priv_check_cred(kauth_cred_get(),
3914 PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0);
3915 if (result != 0) {
3916 return result;
3917 }
3918 }
3919 cookie = kalloc_type(struct nstat_ifnet_cookie,
3920 Z_WAITOK | Z_ZERO | Z_NOFAIL);
3921
3922 ifnet_head_lock_shared();
3923 TAILQ_FOREACH(ifp, &ifnet_head, if_link)
3924 {
3925 if (!ifnet_is_attached(ifp, 1)) {
3926 continue;
3927 }
3928 ifnet_lock_exclusive(ifp);
3929 if (ifp->if_index == param->ifindex) {
3930 cookie->ifp = ifp;
3931 cookie->threshold = param->threshold;
3932 *out_cookie = cookie;
3933 if (!ifp->if_data_threshold ||
3934 ifp->if_data_threshold > param->threshold) {
3935 changed = TRUE;
3936 ifp->if_data_threshold = param->threshold;
3937 }
3938 ifnet_lock_done(ifp);
3939 ifnet_reference(ifp);
3940 ifnet_decr_iorefcnt(ifp);
3941 break;
3942 }
3943 ifnet_lock_done(ifp);
3944 ifnet_decr_iorefcnt(ifp);
3945 }
3946 ifnet_head_done();
3947
3948 /*
3949 * When we change the threshold to something smaller, we notify
3950 * all of our clients with a description message.
3951 * We won't send a message to the client we are currently serving
3952 * because it has no `ifnet source' yet.
3953 */
3954 if (changed) {
3955 NSTAT_LOCK_EXCLUSIVE();
3956 for (client = nstat_clients; client; client = client->ntc_next) {
3957 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
3958 {
3959 if (src->nts_provider != &nstat_ifnet_provider) {
3960 continue;
3961 }
3962 nstat_client_send_description(client, src, 0, 0);
3963 }
3964 }
3965 NSTAT_UNLOCK_EXCLUSIVE();
3966 }
3967 if (cookie->ifp == NULL) {
3968 kfree_type(struct nstat_ifnet_cookie, cookie);
3969 }
3970
3971 return ifp ? 0 : EINVAL;
3972 }
3973
3974 static int
nstat_ifnet_gone(nstat_provider_cookie_t cookie)3975 nstat_ifnet_gone(
3976 nstat_provider_cookie_t cookie)
3977 {
3978 struct ifnet *ifp;
3979 struct nstat_ifnet_cookie *ifcookie =
3980 (struct nstat_ifnet_cookie *)cookie;
3981
3982 ifnet_head_lock_shared();
3983 TAILQ_FOREACH(ifp, &ifnet_head, if_link)
3984 {
3985 if (ifp == ifcookie->ifp) {
3986 break;
3987 }
3988 }
3989 ifnet_head_done();
3990
3991 return ifp ? 0 : 1;
3992 }
3993
3994 static errno_t
nstat_ifnet_counts(nstat_provider_cookie_t cookie,struct nstat_counts * out_counts,int * out_gone)3995 nstat_ifnet_counts(
3996 nstat_provider_cookie_t cookie,
3997 struct nstat_counts *out_counts,
3998 int *out_gone)
3999 {
4000 struct nstat_ifnet_cookie *ifcookie =
4001 (struct nstat_ifnet_cookie *)cookie;
4002 struct ifnet *ifp = ifcookie->ifp;
4003
4004 if (out_gone) {
4005 *out_gone = 0;
4006 }
4007
4008 // if the ifnet is gone, we should stop using it
4009 if (nstat_ifnet_gone(cookie)) {
4010 if (out_gone) {
4011 *out_gone = 1;
4012 }
4013 return EINVAL;
4014 }
4015
4016 bzero(out_counts, sizeof(*out_counts));
4017 out_counts->nstat_rxpackets = ifp->if_ipackets;
4018 out_counts->nstat_rxbytes = ifp->if_ibytes;
4019 out_counts->nstat_txpackets = ifp->if_opackets;
4020 out_counts->nstat_txbytes = ifp->if_obytes;
4021 out_counts->nstat_cell_rxbytes = out_counts->nstat_cell_txbytes = 0;
4022 return 0;
4023 }
4024
4025 static void
nstat_ifnet_release(nstat_provider_cookie_t cookie,__unused int locked)4026 nstat_ifnet_release(
4027 nstat_provider_cookie_t cookie,
4028 __unused int locked)
4029 {
4030 struct nstat_ifnet_cookie *ifcookie;
4031 struct ifnet *ifp;
4032 nstat_client *client;
4033 nstat_src *src;
4034 uint64_t minthreshold = UINT64_MAX;
4035
4036 /*
4037 * Find all the clients that requested a threshold
4038 * for this ifnet and re-calculate if_data_threshold.
4039 */
4040 NSTAT_LOCK_SHARED();
4041 for (client = nstat_clients; client; client = client->ntc_next) {
4042 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
4043 {
4044 /* Skip the provider we are about to detach. */
4045 if (src->nts_provider != &nstat_ifnet_provider ||
4046 src->nts_cookie == cookie) {
4047 continue;
4048 }
4049 ifcookie = (struct nstat_ifnet_cookie *)src->nts_cookie;
4050 if (ifcookie->threshold < minthreshold) {
4051 minthreshold = ifcookie->threshold;
4052 }
4053 }
4054 }
4055 NSTAT_UNLOCK_SHARED();
4056 /*
4057 * Reset if_data_threshold or disable it.
4058 */
4059 ifcookie = (struct nstat_ifnet_cookie *)cookie;
4060 ifp = ifcookie->ifp;
4061 if (ifnet_is_attached(ifp, 1)) {
4062 ifnet_lock_exclusive(ifp);
4063 if (minthreshold == UINT64_MAX) {
4064 ifp->if_data_threshold = 0;
4065 } else {
4066 ifp->if_data_threshold = minthreshold;
4067 }
4068 ifnet_lock_done(ifp);
4069 ifnet_decr_iorefcnt(ifp);
4070 }
4071 ifnet_release(ifp);
4072 kfree_type(struct nstat_ifnet_cookie, ifcookie);
4073 }
4074
4075 static void
nstat_ifnet_copy_link_status(struct ifnet * ifp,struct nstat_ifnet_descriptor * desc)4076 nstat_ifnet_copy_link_status(
4077 struct ifnet *ifp,
4078 struct nstat_ifnet_descriptor *desc)
4079 {
4080 struct if_link_status *ifsr = ifp->if_link_status;
4081 nstat_ifnet_desc_link_status *link_status = &desc->link_status;
4082
4083 link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_NONE;
4084 if (ifsr == NULL) {
4085 return;
4086 }
4087
4088 lck_rw_lock_shared(&ifp->if_link_status_lock);
4089
4090 if (ifp->if_type == IFT_CELLULAR) {
4091 nstat_ifnet_desc_cellular_status *cell_status = &link_status->u.cellular;
4092 struct if_cellular_status_v1 *if_cell_sr =
4093 &ifsr->ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
4094
4095 if (ifsr->ifsr_version != IF_CELLULAR_STATUS_REPORT_VERSION_1) {
4096 goto done;
4097 }
4098
4099 link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR;
4100
4101 if (if_cell_sr->valid_bitmask & IF_CELL_LINK_QUALITY_METRIC_VALID) {
4102 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_LINK_QUALITY_METRIC_VALID;
4103 cell_status->link_quality_metric = if_cell_sr->link_quality_metric;
4104 }
4105 if (if_cell_sr->valid_bitmask & IF_CELL_UL_EFFECTIVE_BANDWIDTH_VALID) {
4106 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_EFFECTIVE_BANDWIDTH_VALID;
4107 cell_status->ul_effective_bandwidth = if_cell_sr->ul_effective_bandwidth;
4108 }
4109 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_BANDWIDTH_VALID) {
4110 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_BANDWIDTH_VALID;
4111 cell_status->ul_max_bandwidth = if_cell_sr->ul_max_bandwidth;
4112 }
4113 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MIN_LATENCY_VALID) {
4114 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MIN_LATENCY_VALID;
4115 cell_status->ul_min_latency = if_cell_sr->ul_min_latency;
4116 }
4117 if (if_cell_sr->valid_bitmask & IF_CELL_UL_EFFECTIVE_LATENCY_VALID) {
4118 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_EFFECTIVE_LATENCY_VALID;
4119 cell_status->ul_effective_latency = if_cell_sr->ul_effective_latency;
4120 }
4121 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_LATENCY_VALID) {
4122 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_LATENCY_VALID;
4123 cell_status->ul_max_latency = if_cell_sr->ul_max_latency;
4124 }
4125 if (if_cell_sr->valid_bitmask & IF_CELL_UL_RETXT_LEVEL_VALID) {
4126 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_VALID;
4127 if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_NONE) {
4128 cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_NONE;
4129 } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_LOW) {
4130 cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_LOW;
4131 } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_MEDIUM) {
4132 cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_MEDIUM;
4133 } else if (if_cell_sr->ul_retxt_level == IF_CELL_UL_RETXT_LEVEL_HIGH) {
4134 cell_status->ul_retxt_level = NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_HIGH;
4135 } else {
4136 cell_status->valid_bitmask &= ~NSTAT_IFNET_DESC_CELL_UL_RETXT_LEVEL_VALID;
4137 }
4138 }
4139 if (if_cell_sr->valid_bitmask & IF_CELL_UL_BYTES_LOST_VALID) {
4140 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_BYTES_LOST_VALID;
4141 cell_status->ul_bytes_lost = if_cell_sr->ul_bytes_lost;
4142 }
4143 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MIN_QUEUE_SIZE_VALID) {
4144 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MIN_QUEUE_SIZE_VALID;
4145 cell_status->ul_min_queue_size = if_cell_sr->ul_min_queue_size;
4146 }
4147 if (if_cell_sr->valid_bitmask & IF_CELL_UL_AVG_QUEUE_SIZE_VALID) {
4148 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_AVG_QUEUE_SIZE_VALID;
4149 cell_status->ul_avg_queue_size = if_cell_sr->ul_avg_queue_size;
4150 }
4151 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MAX_QUEUE_SIZE_VALID) {
4152 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_UL_MAX_QUEUE_SIZE_VALID;
4153 cell_status->ul_max_queue_size = if_cell_sr->ul_max_queue_size;
4154 }
4155 if (if_cell_sr->valid_bitmask & IF_CELL_DL_EFFECTIVE_BANDWIDTH_VALID) {
4156 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_DL_EFFECTIVE_BANDWIDTH_VALID;
4157 cell_status->dl_effective_bandwidth = if_cell_sr->dl_effective_bandwidth;
4158 }
4159 if (if_cell_sr->valid_bitmask & IF_CELL_DL_MAX_BANDWIDTH_VALID) {
4160 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_DL_MAX_BANDWIDTH_VALID;
4161 cell_status->dl_max_bandwidth = if_cell_sr->dl_max_bandwidth;
4162 }
4163 if (if_cell_sr->valid_bitmask & IF_CELL_CONFIG_INACTIVITY_TIME_VALID) {
4164 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_CONFIG_INACTIVITY_TIME_VALID;
4165 cell_status->config_inactivity_time = if_cell_sr->config_inactivity_time;
4166 }
4167 if (if_cell_sr->valid_bitmask & IF_CELL_CONFIG_BACKOFF_TIME_VALID) {
4168 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_CONFIG_BACKOFF_TIME_VALID;
4169 cell_status->config_backoff_time = if_cell_sr->config_backoff_time;
4170 }
4171 if (if_cell_sr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) {
4172 cell_status->valid_bitmask |= NSTAT_IFNET_DESC_CELL_MSS_RECOMMENDED_VALID;
4173 cell_status->mss_recommended = if_cell_sr->mss_recommended;
4174 }
4175 } else if (IFNET_IS_WIFI(ifp)) {
4176 nstat_ifnet_desc_wifi_status *wifi_status = &link_status->u.wifi;
4177 struct if_wifi_status_v1 *if_wifi_sr =
4178 &ifsr->ifsr_u.ifsr_wifi.if_wifi_u.if_status_v1;
4179
4180 if (ifsr->ifsr_version != IF_WIFI_STATUS_REPORT_VERSION_1) {
4181 goto done;
4182 }
4183
4184 link_status->link_status_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI;
4185
4186 if (if_wifi_sr->valid_bitmask & IF_WIFI_LINK_QUALITY_METRIC_VALID) {
4187 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_LINK_QUALITY_METRIC_VALID;
4188 wifi_status->link_quality_metric = if_wifi_sr->link_quality_metric;
4189 }
4190 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_EFFECTIVE_BANDWIDTH_VALID) {
4191 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_EFFECTIVE_BANDWIDTH_VALID;
4192 wifi_status->ul_effective_bandwidth = if_wifi_sr->ul_effective_bandwidth;
4193 }
4194 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MAX_BANDWIDTH_VALID) {
4195 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MAX_BANDWIDTH_VALID;
4196 wifi_status->ul_max_bandwidth = if_wifi_sr->ul_max_bandwidth;
4197 }
4198 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MIN_LATENCY_VALID) {
4199 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MIN_LATENCY_VALID;
4200 wifi_status->ul_min_latency = if_wifi_sr->ul_min_latency;
4201 }
4202 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_EFFECTIVE_LATENCY_VALID) {
4203 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_EFFECTIVE_LATENCY_VALID;
4204 wifi_status->ul_effective_latency = if_wifi_sr->ul_effective_latency;
4205 }
4206 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_MAX_LATENCY_VALID) {
4207 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_MAX_LATENCY_VALID;
4208 wifi_status->ul_max_latency = if_wifi_sr->ul_max_latency;
4209 }
4210 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_RETXT_LEVEL_VALID) {
4211 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_VALID;
4212 if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_NONE) {
4213 wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_NONE;
4214 } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_LOW) {
4215 wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_LOW;
4216 } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_MEDIUM) {
4217 wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_MEDIUM;
4218 } else if (if_wifi_sr->ul_retxt_level == IF_WIFI_UL_RETXT_LEVEL_HIGH) {
4219 wifi_status->ul_retxt_level = NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_HIGH;
4220 } else {
4221 wifi_status->valid_bitmask &= ~NSTAT_IFNET_DESC_WIFI_UL_RETXT_LEVEL_VALID;
4222 }
4223 }
4224 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_BYTES_LOST_VALID) {
4225 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_BYTES_LOST_VALID;
4226 wifi_status->ul_bytes_lost = if_wifi_sr->ul_bytes_lost;
4227 }
4228 if (if_wifi_sr->valid_bitmask & IF_WIFI_UL_ERROR_RATE_VALID) {
4229 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_UL_ERROR_RATE_VALID;
4230 wifi_status->ul_error_rate = if_wifi_sr->ul_error_rate;
4231 }
4232 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_EFFECTIVE_BANDWIDTH_VALID) {
4233 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_EFFECTIVE_BANDWIDTH_VALID;
4234 wifi_status->dl_effective_bandwidth = if_wifi_sr->dl_effective_bandwidth;
4235 }
4236 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MAX_BANDWIDTH_VALID) {
4237 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MAX_BANDWIDTH_VALID;
4238 wifi_status->dl_max_bandwidth = if_wifi_sr->dl_max_bandwidth;
4239 }
4240 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MIN_LATENCY_VALID) {
4241 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MIN_LATENCY_VALID;
4242 wifi_status->dl_min_latency = if_wifi_sr->dl_min_latency;
4243 }
4244 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_EFFECTIVE_LATENCY_VALID) {
4245 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_EFFECTIVE_LATENCY_VALID;
4246 wifi_status->dl_effective_latency = if_wifi_sr->dl_effective_latency;
4247 }
4248 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_MAX_LATENCY_VALID) {
4249 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_MAX_LATENCY_VALID;
4250 wifi_status->dl_max_latency = if_wifi_sr->dl_max_latency;
4251 }
4252 if (if_wifi_sr->valid_bitmask & IF_WIFI_DL_ERROR_RATE_VALID) {
4253 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_DL_ERROR_RATE_VALID;
4254 wifi_status->dl_error_rate = if_wifi_sr->dl_error_rate;
4255 }
4256 if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_FREQUENCY_VALID) {
4257 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_VALID;
4258 if (if_wifi_sr->config_frequency == IF_WIFI_CONFIG_FREQUENCY_2_4_GHZ) {
4259 wifi_status->config_frequency = NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_2_4_GHZ;
4260 } else if (if_wifi_sr->config_frequency == IF_WIFI_CONFIG_FREQUENCY_5_0_GHZ) {
4261 wifi_status->config_frequency = NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_5_0_GHZ;
4262 } else {
4263 wifi_status->valid_bitmask &= ~NSTAT_IFNET_DESC_WIFI_CONFIG_FREQUENCY_VALID;
4264 }
4265 }
4266 if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_MULTICAST_RATE_VALID) {
4267 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_MULTICAST_RATE_VALID;
4268 wifi_status->config_multicast_rate = if_wifi_sr->config_multicast_rate;
4269 }
4270 if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_SCAN_COUNT_VALID) {
4271 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_SCAN_COUNT_VALID;
4272 wifi_status->scan_count = if_wifi_sr->scan_count;
4273 }
4274 if (if_wifi_sr->valid_bitmask & IF_WIFI_CONFIG_SCAN_DURATION_VALID) {
4275 wifi_status->valid_bitmask |= NSTAT_IFNET_DESC_WIFI_CONFIG_SCAN_DURATION_VALID;
4276 wifi_status->scan_duration = if_wifi_sr->scan_duration;
4277 }
4278 }
4279
4280 done:
4281 lck_rw_done(&ifp->if_link_status_lock);
4282 }
4283
4284 static u_int64_t nstat_ifnet_last_report_time = 0;
4285 extern int tcp_report_stats_interval;
4286
4287 static void
nstat_ifnet_compute_percentages(struct if_tcp_ecn_perf_stat * ifst)4288 nstat_ifnet_compute_percentages(struct if_tcp_ecn_perf_stat *ifst)
4289 {
4290 /* Retransmit percentage */
4291 if (ifst->total_rxmitpkts > 0 && ifst->total_txpkts > 0) {
4292 /* shift by 10 for precision */
4293 ifst->rxmit_percent =
4294 ((ifst->total_rxmitpkts << 10) * 100) / ifst->total_txpkts;
4295 } else {
4296 ifst->rxmit_percent = 0;
4297 }
4298
4299 /* Out-of-order percentage */
4300 if (ifst->total_oopkts > 0 && ifst->total_rxpkts > 0) {
4301 /* shift by 10 for precision */
4302 ifst->oo_percent =
4303 ((ifst->total_oopkts << 10) * 100) / ifst->total_rxpkts;
4304 } else {
4305 ifst->oo_percent = 0;
4306 }
4307
4308 /* Reorder percentage */
4309 if (ifst->total_reorderpkts > 0 &&
4310 (ifst->total_txpkts + ifst->total_rxpkts) > 0) {
4311 /* shift by 10 for precision */
4312 ifst->reorder_percent =
4313 ((ifst->total_reorderpkts << 10) * 100) /
4314 (ifst->total_txpkts + ifst->total_rxpkts);
4315 } else {
4316 ifst->reorder_percent = 0;
4317 }
4318 }
4319
4320 static void
nstat_ifnet_normalize_counter(struct if_tcp_ecn_stat * if_st)4321 nstat_ifnet_normalize_counter(struct if_tcp_ecn_stat *if_st)
4322 {
4323 u_int64_t ecn_on_conn, ecn_off_conn;
4324
4325 if (if_st == NULL) {
4326 return;
4327 }
4328 ecn_on_conn = if_st->ecn_client_success +
4329 if_st->ecn_server_success;
4330 ecn_off_conn = if_st->ecn_off_conn +
4331 (if_st->ecn_client_setup - if_st->ecn_client_success) +
4332 (if_st->ecn_server_setup - if_st->ecn_server_success);
4333
4334 /*
4335 * report sack episodes, rst_drop and rxmit_drop
4336 * as a ratio per connection, shift by 10 for precision
4337 */
4338 if (ecn_on_conn > 0) {
4339 if_st->ecn_on.sack_episodes =
4340 (if_st->ecn_on.sack_episodes << 10) / ecn_on_conn;
4341 if_st->ecn_on.rst_drop =
4342 (if_st->ecn_on.rst_drop << 10) * 100 / ecn_on_conn;
4343 if_st->ecn_on.rxmit_drop =
4344 (if_st->ecn_on.rxmit_drop << 10) * 100 / ecn_on_conn;
4345 } else {
4346 /* set to zero, just in case */
4347 if_st->ecn_on.sack_episodes = 0;
4348 if_st->ecn_on.rst_drop = 0;
4349 if_st->ecn_on.rxmit_drop = 0;
4350 }
4351
4352 if (ecn_off_conn > 0) {
4353 if_st->ecn_off.sack_episodes =
4354 (if_st->ecn_off.sack_episodes << 10) / ecn_off_conn;
4355 if_st->ecn_off.rst_drop =
4356 (if_st->ecn_off.rst_drop << 10) * 100 / ecn_off_conn;
4357 if_st->ecn_off.rxmit_drop =
4358 (if_st->ecn_off.rxmit_drop << 10) * 100 / ecn_off_conn;
4359 } else {
4360 if_st->ecn_off.sack_episodes = 0;
4361 if_st->ecn_off.rst_drop = 0;
4362 if_st->ecn_off.rxmit_drop = 0;
4363 }
4364 if_st->ecn_total_conn = ecn_off_conn + ecn_on_conn;
4365 }
4366
4367 static void
nstat_ifnet_report_ecn_stats(void)4368 nstat_ifnet_report_ecn_stats(void)
4369 {
4370 u_int64_t uptime, last_report_time;
4371 struct nstat_sysinfo_data data;
4372 struct nstat_sysinfo_ifnet_ecn_stats *st;
4373 struct ifnet *ifp;
4374
4375 uptime = net_uptime();
4376
4377 if ((int)(uptime - nstat_ifnet_last_report_time) <
4378 tcp_report_stats_interval) {
4379 return;
4380 }
4381
4382 last_report_time = nstat_ifnet_last_report_time;
4383 nstat_ifnet_last_report_time = uptime;
4384 data.flags = NSTAT_SYSINFO_IFNET_ECN_STATS;
4385 st = &data.u.ifnet_ecn_stats;
4386
4387 ifnet_head_lock_shared();
4388 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
4389 if (ifp->if_ipv4_stat == NULL || ifp->if_ipv6_stat == NULL) {
4390 continue;
4391 }
4392
4393 if (!IF_FULLY_ATTACHED(ifp)) {
4394 continue;
4395 }
4396
4397 /* Limit reporting to Wifi, Ethernet and cellular. */
4398 if (!(IFNET_IS_ETHERNET(ifp) || IFNET_IS_CELLULAR(ifp))) {
4399 continue;
4400 }
4401
4402 bzero(st, sizeof(*st));
4403 if (IFNET_IS_CELLULAR(ifp)) {
4404 st->ifnet_type = NSTAT_IFNET_ECN_TYPE_CELLULAR;
4405 } else if (IFNET_IS_WIFI(ifp)) {
4406 st->ifnet_type = NSTAT_IFNET_ECN_TYPE_WIFI;
4407 } else {
4408 st->ifnet_type = NSTAT_IFNET_ECN_TYPE_ETHERNET;
4409 }
4410 data.unsent_data_cnt = ifp->if_unsent_data_cnt;
4411 /* skip if there was no update since last report */
4412 if (ifp->if_ipv4_stat->timestamp <= 0 ||
4413 ifp->if_ipv4_stat->timestamp < last_report_time) {
4414 goto v6;
4415 }
4416 st->ifnet_proto = NSTAT_IFNET_ECN_PROTO_IPV4;
4417 /* compute percentages using packet counts */
4418 nstat_ifnet_compute_percentages(&ifp->if_ipv4_stat->ecn_on);
4419 nstat_ifnet_compute_percentages(&ifp->if_ipv4_stat->ecn_off);
4420 nstat_ifnet_normalize_counter(ifp->if_ipv4_stat);
4421 bcopy(ifp->if_ipv4_stat, &st->ecn_stat,
4422 sizeof(st->ecn_stat));
4423 nstat_sysinfo_send_data(&data);
4424 bzero(ifp->if_ipv4_stat, sizeof(*ifp->if_ipv4_stat));
4425
4426 v6:
4427 /* skip if there was no update since last report */
4428 if (ifp->if_ipv6_stat->timestamp <= 0 ||
4429 ifp->if_ipv6_stat->timestamp < last_report_time) {
4430 continue;
4431 }
4432 st->ifnet_proto = NSTAT_IFNET_ECN_PROTO_IPV6;
4433
4434 /* compute percentages using packet counts */
4435 nstat_ifnet_compute_percentages(&ifp->if_ipv6_stat->ecn_on);
4436 nstat_ifnet_compute_percentages(&ifp->if_ipv6_stat->ecn_off);
4437 nstat_ifnet_normalize_counter(ifp->if_ipv6_stat);
4438 bcopy(ifp->if_ipv6_stat, &st->ecn_stat,
4439 sizeof(st->ecn_stat));
4440 nstat_sysinfo_send_data(&data);
4441
4442 /* Zero the stats in ifp */
4443 bzero(ifp->if_ipv6_stat, sizeof(*ifp->if_ipv6_stat));
4444 }
4445 ifnet_head_done();
4446 }
4447
4448 /* Some thresholds to determine Low Iternet mode */
4449 #define NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD 1000000 /* 1 Mbps */
4450 #define NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD 500000 /* 500 Kbps */
4451 #define NSTAT_LIM_UL_MIN_RTT_THRESHOLD 1000 /* 1 second */
4452 #define NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD (10 << 10) /* 10 percent connection timeouts */
4453 #define NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD (2 << 10) /* 2 percent packet loss rate */
4454
4455 static boolean_t
nstat_lim_activity_check(struct if_lim_perf_stat * st)4456 nstat_lim_activity_check(struct if_lim_perf_stat *st)
4457 {
4458 /* check that the current activity is enough to report stats */
4459 if (st->lim_total_txpkts < nstat_lim_min_tx_pkts ||
4460 st->lim_total_rxpkts < nstat_lim_min_rx_pkts ||
4461 st->lim_conn_attempts == 0) {
4462 return FALSE;
4463 }
4464
4465 /*
4466 * Compute percentages if there was enough activity. Use
4467 * shift-left by 10 to preserve precision.
4468 */
4469 st->lim_packet_loss_percent = ((st->lim_total_retxpkts << 10) /
4470 st->lim_total_txpkts) * 100;
4471
4472 st->lim_packet_ooo_percent = ((st->lim_total_oopkts << 10) /
4473 st->lim_total_rxpkts) * 100;
4474
4475 st->lim_conn_timeout_percent = ((st->lim_conn_timeouts << 10) /
4476 st->lim_conn_attempts) * 100;
4477
4478 /*
4479 * Is Low Internet detected? First order metrics are bandwidth
4480 * and RTT. If these metrics are below the minimum thresholds
4481 * defined then the network attachment can be classified as
4482 * having Low Internet capacity.
4483 *
4484 * High connection timeout rate also indicates Low Internet
4485 * capacity.
4486 */
4487 if (st->lim_dl_max_bandwidth > 0 &&
4488 st->lim_dl_max_bandwidth <= NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD) {
4489 st->lim_dl_detected = 1;
4490 }
4491
4492 if ((st->lim_ul_max_bandwidth > 0 &&
4493 st->lim_ul_max_bandwidth <= NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD) ||
4494 st->lim_rtt_min >= NSTAT_LIM_UL_MIN_RTT_THRESHOLD) {
4495 st->lim_ul_detected = 1;
4496 }
4497
4498 if (st->lim_conn_attempts > 20 &&
4499 st->lim_conn_timeout_percent >=
4500 NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD) {
4501 st->lim_ul_detected = 1;
4502 }
4503 /*
4504 * Second order metrics: If there was high packet loss even after
4505 * using delay based algorithms then we classify it as Low Internet
4506 * again
4507 */
4508 if (st->lim_bk_txpkts >= nstat_lim_min_tx_pkts &&
4509 st->lim_packet_loss_percent >=
4510 NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD) {
4511 st->lim_ul_detected = 1;
4512 }
4513 return TRUE;
4514 }
4515
4516 static u_int64_t nstat_lim_last_report_time = 0;
4517 static void
nstat_ifnet_report_lim_stats(void)4518 nstat_ifnet_report_lim_stats(void)
4519 {
4520 u_int64_t uptime;
4521 struct nstat_sysinfo_data data;
4522 struct nstat_sysinfo_lim_stats *st;
4523 struct ifnet *ifp;
4524 int err;
4525
4526 uptime = net_uptime();
4527
4528 if ((u_int32_t)(uptime - nstat_lim_last_report_time) <
4529 nstat_lim_interval) {
4530 return;
4531 }
4532
4533 nstat_lim_last_report_time = uptime;
4534 data.flags = NSTAT_SYSINFO_LIM_STATS;
4535 st = &data.u.lim_stats;
4536 data.unsent_data_cnt = 0;
4537
4538 ifnet_head_lock_shared();
4539 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
4540 if (!IF_FULLY_ATTACHED(ifp)) {
4541 continue;
4542 }
4543
4544 /* Limit reporting to Wifi, Ethernet and cellular */
4545 if (!(IFNET_IS_ETHERNET(ifp) || IFNET_IS_CELLULAR(ifp))) {
4546 continue;
4547 }
4548
4549 if (!nstat_lim_activity_check(&ifp->if_lim_stat)) {
4550 continue;
4551 }
4552
4553 bzero(st, sizeof(*st));
4554 st->ifnet_siglen = sizeof(st->ifnet_signature);
4555 err = ifnet_get_netsignature(ifp, AF_INET,
4556 (u_int8_t *)&st->ifnet_siglen, NULL,
4557 st->ifnet_signature);
4558 if (err != 0) {
4559 err = ifnet_get_netsignature(ifp, AF_INET6,
4560 (u_int8_t *)&st->ifnet_siglen, NULL,
4561 st->ifnet_signature);
4562 if (err != 0) {
4563 continue;
4564 }
4565 }
4566 ifnet_lock_shared(ifp);
4567 if (IFNET_IS_CELLULAR(ifp)) {
4568 st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR;
4569 } else if (IFNET_IS_WIFI(ifp)) {
4570 st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI;
4571 } else {
4572 st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET;
4573 }
4574 bcopy(&ifp->if_lim_stat, &st->lim_stat,
4575 sizeof(st->lim_stat));
4576
4577 /* Zero the stats in ifp */
4578 bzero(&ifp->if_lim_stat, sizeof(ifp->if_lim_stat));
4579 ifnet_lock_done(ifp);
4580 nstat_sysinfo_send_data(&data);
4581 }
4582 ifnet_head_done();
4583 }
4584
4585 static errno_t
nstat_ifnet_copy_descriptor(nstat_provider_cookie_t cookie,void * __sized_by (len)data,size_t len)4586 nstat_ifnet_copy_descriptor(
4587 nstat_provider_cookie_t cookie,
4588 void *__sized_by(len)data,
4589 size_t len)
4590 {
4591 nstat_ifnet_descriptor *desc = (nstat_ifnet_descriptor *)data;
4592 struct nstat_ifnet_cookie *ifcookie =
4593 (struct nstat_ifnet_cookie *)cookie;
4594 struct ifnet *ifp = ifcookie->ifp;
4595
4596 if (len < sizeof(nstat_ifnet_descriptor)) {
4597 return EINVAL;
4598 }
4599
4600 if (nstat_ifnet_gone(cookie)) {
4601 return EINVAL;
4602 }
4603
4604 bzero(desc, sizeof(*desc));
4605 ifnet_lock_shared(ifp);
4606 strlcpy(desc->name, ifp->if_xname, sizeof(desc->name));
4607 desc->ifindex = ifp->if_index;
4608 desc->threshold = ifp->if_data_threshold;
4609 desc->type = ifp->if_type;
4610 if (ifp->if_desc.ifd_len < sizeof(desc->description)) {
4611 memcpy(desc->description, ifp->if_desc.ifd_desc,
4612 sizeof(desc->description));
4613 }
4614 nstat_ifnet_copy_link_status(ifp, desc);
4615 ifnet_lock_done(ifp);
4616 return 0;
4617 }
4618
4619 static bool
nstat_ifnet_cookie_equal(nstat_provider_cookie_t cookie1,nstat_provider_cookie_t cookie2)4620 nstat_ifnet_cookie_equal(
4621 nstat_provider_cookie_t cookie1,
4622 nstat_provider_cookie_t cookie2)
4623 {
4624 struct nstat_ifnet_cookie *c1 = (struct nstat_ifnet_cookie *)cookie1;
4625 struct nstat_ifnet_cookie *c2 = (struct nstat_ifnet_cookie *)cookie2;
4626
4627 return (c1->ifp->if_index == c2->ifp->if_index) ? true : false;
4628 }
4629
4630 static void
nstat_init_ifnet_provider(void)4631 nstat_init_ifnet_provider(void)
4632 {
4633 bzero(&nstat_ifnet_provider, sizeof(nstat_ifnet_provider));
4634 nstat_ifnet_provider.nstat_provider_id = NSTAT_PROVIDER_IFNET;
4635 nstat_ifnet_provider.nstat_descriptor_length = sizeof(nstat_ifnet_descriptor);
4636 nstat_ifnet_provider.nstat_lookup = nstat_ifnet_lookup;
4637 nstat_ifnet_provider.nstat_gone = nstat_ifnet_gone;
4638 nstat_ifnet_provider.nstat_counts = nstat_ifnet_counts;
4639 nstat_ifnet_provider.nstat_watcher_add = NULL;
4640 nstat_ifnet_provider.nstat_watcher_remove = NULL;
4641 nstat_ifnet_provider.nstat_copy_descriptor = nstat_ifnet_copy_descriptor;
4642 nstat_ifnet_provider.nstat_cookie_equal = nstat_ifnet_cookie_equal;
4643 nstat_ifnet_provider.nstat_release = nstat_ifnet_release;
4644 nstat_ifnet_provider.next = nstat_providers;
4645 nstat_providers = &nstat_ifnet_provider;
4646 }
4647
4648 __private_extern__ void
nstat_ifnet_threshold_reached(unsigned int ifindex)4649 nstat_ifnet_threshold_reached(unsigned int ifindex)
4650 {
4651 nstat_client *client;
4652 nstat_src *src;
4653 struct ifnet *ifp;
4654 struct nstat_ifnet_cookie *ifcookie;
4655
4656 NSTAT_LOCK_EXCLUSIVE();
4657 for (client = nstat_clients; client; client = client->ntc_next) {
4658 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
4659 {
4660 if (src->nts_provider != &nstat_ifnet_provider) {
4661 continue;
4662 }
4663 ifcookie = (struct nstat_ifnet_cookie *)src->nts_cookie;
4664 ifp = ifcookie->ifp;
4665 if (ifp->if_index != ifindex) {
4666 continue;
4667 }
4668 nstat_client_send_counts(client, src, 0, 0, NULL);
4669 }
4670 }
4671 NSTAT_UNLOCK_EXCLUSIVE();
4672 }
4673
4674 #pragma mark -- Sysinfo --
4675 static void
nstat_set_keyval_scalar(nstat_sysinfo_keyval * kv,int key,u_int32_t val)4676 nstat_set_keyval_scalar(nstat_sysinfo_keyval *kv, int key, u_int32_t val)
4677 {
4678 kv->nstat_sysinfo_key = key;
4679 kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR;
4680 kv->u.nstat_sysinfo_scalar = val;
4681 kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar);
4682 }
4683
4684 static void
nstat_set_keyval_u64_scalar(nstat_sysinfo_keyval * kv,int key,u_int64_t val)4685 nstat_set_keyval_u64_scalar(nstat_sysinfo_keyval *kv, int key, u_int64_t val)
4686 {
4687 kv->nstat_sysinfo_key = key;
4688 kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR;
4689 kv->u.nstat_sysinfo_scalar = val;
4690 kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar);
4691 }
4692
4693 static void
nstat_set_keyval_string(nstat_sysinfo_keyval * kv,int key,u_int8_t * __counted_by (len)buf,u_int32_t len)4694 nstat_set_keyval_string(nstat_sysinfo_keyval *kv, int key, u_int8_t *__counted_by(len)buf,
4695 u_int32_t len)
4696 {
4697 kv->nstat_sysinfo_key = key;
4698 kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_STRING;
4699 kv->nstat_sysinfo_valsize = len;
4700 bcopy(buf, kv->u.nstat_sysinfo_string, kv->nstat_sysinfo_valsize);
4701 }
4702
4703 static void
nstat_sysinfo_send_data_internal(nstat_client * client,nstat_sysinfo_data * data)4704 nstat_sysinfo_send_data_internal(
4705 nstat_client *client,
4706 nstat_sysinfo_data *data)
4707 {
4708 nstat_msg_sysinfo_counts *syscnt = NULL;
4709 size_t allocsize = 0, countsize = 0, nkeyvals = 0, finalsize = 0;
4710 nstat_sysinfo_keyval *kv;
4711 errno_t result = 0;
4712 size_t i = 0;
4713
4714 allocsize = offsetof(nstat_msg_sysinfo_counts, counts);
4715 countsize = offsetof(nstat_sysinfo_counts, nstat_sysinfo_keyvals);
4716 finalsize = allocsize;
4717
4718 /* get number of key-vals for each kind of stat */
4719 switch (data->flags) {
4720 case NSTAT_SYSINFO_TCP_STATS:
4721 nkeyvals = NSTAT_SYSINFO_TCP_STATS_COUNT;
4722 break;
4723 case NSTAT_SYSINFO_IFNET_ECN_STATS:
4724 nkeyvals = (sizeof(struct if_tcp_ecn_stat) /
4725 sizeof(u_int64_t));
4726
4727 /* Two more keys for ifnet type and proto */
4728 nkeyvals += 2;
4729
4730 /* One key for unsent data. */
4731 nkeyvals++;
4732 break;
4733 case NSTAT_SYSINFO_LIM_STATS:
4734 nkeyvals = NSTAT_LIM_STAT_KEYVAL_COUNT;
4735 break;
4736 case NSTAT_SYSINFO_NET_API_STATS:
4737 nkeyvals = NSTAT_NET_API_STAT_KEYVAL_COUNT;
4738 break;
4739 default:
4740 return;
4741 }
4742 countsize += sizeof(nstat_sysinfo_keyval) * nkeyvals;
4743 allocsize += countsize;
4744
4745 syscnt = (nstat_msg_sysinfo_counts *) kalloc_data(allocsize,
4746 Z_WAITOK | Z_ZERO);
4747 if (syscnt == NULL) {
4748 return;
4749 }
4750
4751 kv = nstat_sysinfo_get_keyvals(syscnt);
4752
4753 switch (data->flags) {
4754 case NSTAT_SYSINFO_TCP_STATS:
4755 {
4756 nstat_set_keyval_scalar(&kv[i++],
4757 NSTAT_SYSINFO_KEY_IPV4_AVGRTT,
4758 data->u.tcp_stats.ipv4_avgrtt);
4759 nstat_set_keyval_scalar(&kv[i++],
4760 NSTAT_SYSINFO_KEY_IPV6_AVGRTT,
4761 data->u.tcp_stats.ipv6_avgrtt);
4762 nstat_set_keyval_scalar(&kv[i++],
4763 NSTAT_SYSINFO_KEY_SEND_PLR,
4764 data->u.tcp_stats.send_plr);
4765 nstat_set_keyval_scalar(&kv[i++],
4766 NSTAT_SYSINFO_KEY_RECV_PLR,
4767 data->u.tcp_stats.recv_plr);
4768 nstat_set_keyval_scalar(&kv[i++],
4769 NSTAT_SYSINFO_KEY_SEND_TLRTO,
4770 data->u.tcp_stats.send_tlrto_rate);
4771 nstat_set_keyval_scalar(&kv[i++],
4772 NSTAT_SYSINFO_KEY_SEND_REORDERRATE,
4773 data->u.tcp_stats.send_reorder_rate);
4774 nstat_set_keyval_scalar(&kv[i++],
4775 NSTAT_SYSINFO_CONNECTION_ATTEMPTS,
4776 data->u.tcp_stats.connection_attempts);
4777 nstat_set_keyval_scalar(&kv[i++],
4778 NSTAT_SYSINFO_CONNECTION_ACCEPTS,
4779 data->u.tcp_stats.connection_accepts);
4780 nstat_set_keyval_scalar(&kv[i++],
4781 NSTAT_SYSINFO_ECN_CLIENT_ENABLED,
4782 data->u.tcp_stats.ecn_client_enabled);
4783 nstat_set_keyval_scalar(&kv[i++],
4784 NSTAT_SYSINFO_ECN_SERVER_ENABLED,
4785 data->u.tcp_stats.ecn_server_enabled);
4786 nstat_set_keyval_scalar(&kv[i++],
4787 NSTAT_SYSINFO_ECN_CLIENT_SETUP,
4788 data->u.tcp_stats.ecn_client_setup);
4789 nstat_set_keyval_scalar(&kv[i++],
4790 NSTAT_SYSINFO_ECN_SERVER_SETUP,
4791 data->u.tcp_stats.ecn_server_setup);
4792 nstat_set_keyval_scalar(&kv[i++],
4793 NSTAT_SYSINFO_ECN_CLIENT_SUCCESS,
4794 data->u.tcp_stats.ecn_client_success);
4795 nstat_set_keyval_scalar(&kv[i++],
4796 NSTAT_SYSINFO_ECN_SERVER_SUCCESS,
4797 data->u.tcp_stats.ecn_server_success);
4798 nstat_set_keyval_scalar(&kv[i++],
4799 NSTAT_SYSINFO_ECN_NOT_SUPPORTED,
4800 data->u.tcp_stats.ecn_not_supported);
4801 nstat_set_keyval_scalar(&kv[i++],
4802 NSTAT_SYSINFO_ECN_LOST_SYN,
4803 data->u.tcp_stats.ecn_lost_syn);
4804 nstat_set_keyval_scalar(&kv[i++],
4805 NSTAT_SYSINFO_ECN_LOST_SYNACK,
4806 data->u.tcp_stats.ecn_lost_synack);
4807 nstat_set_keyval_scalar(&kv[i++],
4808 NSTAT_SYSINFO_ECN_RECV_CE,
4809 data->u.tcp_stats.ecn_recv_ce);
4810 nstat_set_keyval_scalar(&kv[i++],
4811 NSTAT_SYSINFO_ECN_RECV_ECE,
4812 data->u.tcp_stats.ecn_recv_ece);
4813 nstat_set_keyval_scalar(&kv[i++],
4814 NSTAT_SYSINFO_ECN_SENT_ECE,
4815 data->u.tcp_stats.ecn_sent_ece);
4816 nstat_set_keyval_scalar(&kv[i++],
4817 NSTAT_SYSINFO_ECN_CONN_RECV_CE,
4818 data->u.tcp_stats.ecn_conn_recv_ce);
4819 nstat_set_keyval_scalar(&kv[i++],
4820 NSTAT_SYSINFO_ECN_CONN_RECV_ECE,
4821 data->u.tcp_stats.ecn_conn_recv_ece);
4822 nstat_set_keyval_scalar(&kv[i++],
4823 NSTAT_SYSINFO_ECN_CONN_PLNOCE,
4824 data->u.tcp_stats.ecn_conn_plnoce);
4825 nstat_set_keyval_scalar(&kv[i++],
4826 NSTAT_SYSINFO_ECN_CONN_PL_CE,
4827 data->u.tcp_stats.ecn_conn_pl_ce);
4828 nstat_set_keyval_scalar(&kv[i++],
4829 NSTAT_SYSINFO_ECN_CONN_NOPL_CE,
4830 data->u.tcp_stats.ecn_conn_nopl_ce);
4831 nstat_set_keyval_scalar(&kv[i++],
4832 NSTAT_SYSINFO_ECN_FALLBACK_SYNLOSS,
4833 data->u.tcp_stats.ecn_fallback_synloss);
4834 nstat_set_keyval_scalar(&kv[i++],
4835 NSTAT_SYSINFO_ECN_FALLBACK_REORDER,
4836 data->u.tcp_stats.ecn_fallback_reorder);
4837 nstat_set_keyval_scalar(&kv[i++],
4838 NSTAT_SYSINFO_ECN_FALLBACK_CE,
4839 data->u.tcp_stats.ecn_fallback_ce);
4840 nstat_set_keyval_scalar(&kv[i++],
4841 NSTAT_SYSINFO_TFO_SYN_DATA_RCV,
4842 data->u.tcp_stats.tfo_syn_data_rcv);
4843 nstat_set_keyval_scalar(&kv[i++],
4844 NSTAT_SYSINFO_TFO_COOKIE_REQ_RCV,
4845 data->u.tcp_stats.tfo_cookie_req_rcv);
4846 nstat_set_keyval_scalar(&kv[i++],
4847 NSTAT_SYSINFO_TFO_COOKIE_SENT,
4848 data->u.tcp_stats.tfo_cookie_sent);
4849 nstat_set_keyval_scalar(&kv[i++],
4850 NSTAT_SYSINFO_TFO_COOKIE_INVALID,
4851 data->u.tcp_stats.tfo_cookie_invalid);
4852 nstat_set_keyval_scalar(&kv[i++],
4853 NSTAT_SYSINFO_TFO_COOKIE_REQ,
4854 data->u.tcp_stats.tfo_cookie_req);
4855 nstat_set_keyval_scalar(&kv[i++],
4856 NSTAT_SYSINFO_TFO_COOKIE_RCV,
4857 data->u.tcp_stats.tfo_cookie_rcv);
4858 nstat_set_keyval_scalar(&kv[i++],
4859 NSTAT_SYSINFO_TFO_SYN_DATA_SENT,
4860 data->u.tcp_stats.tfo_syn_data_sent);
4861 nstat_set_keyval_scalar(&kv[i++],
4862 NSTAT_SYSINFO_TFO_SYN_DATA_ACKED,
4863 data->u.tcp_stats.tfo_syn_data_acked);
4864 nstat_set_keyval_scalar(&kv[i++],
4865 NSTAT_SYSINFO_TFO_SYN_LOSS,
4866 data->u.tcp_stats.tfo_syn_loss);
4867 nstat_set_keyval_scalar(&kv[i++],
4868 NSTAT_SYSINFO_TFO_BLACKHOLE,
4869 data->u.tcp_stats.tfo_blackhole);
4870 nstat_set_keyval_scalar(&kv[i++],
4871 NSTAT_SYSINFO_TFO_COOKIE_WRONG,
4872 data->u.tcp_stats.tfo_cookie_wrong);
4873 nstat_set_keyval_scalar(&kv[i++],
4874 NSTAT_SYSINFO_TFO_NO_COOKIE_RCV,
4875 data->u.tcp_stats.tfo_no_cookie_rcv);
4876 nstat_set_keyval_scalar(&kv[i++],
4877 NSTAT_SYSINFO_TFO_HEURISTICS_DISABLE,
4878 data->u.tcp_stats.tfo_heuristics_disable);
4879 nstat_set_keyval_scalar(&kv[i++],
4880 NSTAT_SYSINFO_TFO_SEND_BLACKHOLE,
4881 data->u.tcp_stats.tfo_sndblackhole);
4882 nstat_set_keyval_scalar(&kv[i++],
4883 NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT,
4884 data->u.tcp_stats.mptcp_handover_attempt);
4885 nstat_set_keyval_scalar(&kv[i++],
4886 NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT,
4887 data->u.tcp_stats.mptcp_interactive_attempt);
4888 nstat_set_keyval_scalar(&kv[i++],
4889 NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT,
4890 data->u.tcp_stats.mptcp_aggregate_attempt);
4891 nstat_set_keyval_scalar(&kv[i++],
4892 NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT,
4893 data->u.tcp_stats.mptcp_fp_handover_attempt);
4894 nstat_set_keyval_scalar(&kv[i++],
4895 NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT,
4896 data->u.tcp_stats.mptcp_fp_interactive_attempt);
4897 nstat_set_keyval_scalar(&kv[i++],
4898 NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT,
4899 data->u.tcp_stats.mptcp_fp_aggregate_attempt);
4900 nstat_set_keyval_scalar(&kv[i++],
4901 NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK,
4902 data->u.tcp_stats.mptcp_heuristic_fallback);
4903 nstat_set_keyval_scalar(&kv[i++],
4904 NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK,
4905 data->u.tcp_stats.mptcp_fp_heuristic_fallback);
4906 nstat_set_keyval_scalar(&kv[i++],
4907 NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI,
4908 data->u.tcp_stats.mptcp_handover_success_wifi);
4909 nstat_set_keyval_scalar(&kv[i++],
4910 NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL,
4911 data->u.tcp_stats.mptcp_handover_success_cell);
4912 nstat_set_keyval_scalar(&kv[i++],
4913 NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS,
4914 data->u.tcp_stats.mptcp_interactive_success);
4915 nstat_set_keyval_scalar(&kv[i++],
4916 NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS,
4917 data->u.tcp_stats.mptcp_aggregate_success);
4918 nstat_set_keyval_scalar(&kv[i++],
4919 NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI,
4920 data->u.tcp_stats.mptcp_fp_handover_success_wifi);
4921 nstat_set_keyval_scalar(&kv[i++],
4922 NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL,
4923 data->u.tcp_stats.mptcp_fp_handover_success_cell);
4924 nstat_set_keyval_scalar(&kv[i++],
4925 NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS,
4926 data->u.tcp_stats.mptcp_fp_interactive_success);
4927 nstat_set_keyval_scalar(&kv[i++],
4928 NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS,
4929 data->u.tcp_stats.mptcp_fp_aggregate_success);
4930 nstat_set_keyval_scalar(&kv[i++],
4931 NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI,
4932 data->u.tcp_stats.mptcp_handover_cell_from_wifi);
4933 nstat_set_keyval_scalar(&kv[i++],
4934 NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL,
4935 data->u.tcp_stats.mptcp_handover_wifi_from_cell);
4936 nstat_set_keyval_scalar(&kv[i++],
4937 NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI,
4938 data->u.tcp_stats.mptcp_interactive_cell_from_wifi);
4939 nstat_set_keyval_u64_scalar(&kv[i++],
4940 NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES,
4941 data->u.tcp_stats.mptcp_handover_cell_bytes);
4942 nstat_set_keyval_u64_scalar(&kv[i++],
4943 NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES,
4944 data->u.tcp_stats.mptcp_interactive_cell_bytes);
4945 nstat_set_keyval_u64_scalar(&kv[i++],
4946 NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES,
4947 data->u.tcp_stats.mptcp_aggregate_cell_bytes);
4948 nstat_set_keyval_u64_scalar(&kv[i++],
4949 NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES,
4950 data->u.tcp_stats.mptcp_handover_all_bytes);
4951 nstat_set_keyval_u64_scalar(&kv[i++],
4952 NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES,
4953 data->u.tcp_stats.mptcp_interactive_all_bytes);
4954 nstat_set_keyval_u64_scalar(&kv[i++],
4955 NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES,
4956 data->u.tcp_stats.mptcp_aggregate_all_bytes);
4957 nstat_set_keyval_scalar(&kv[i++],
4958 NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI,
4959 data->u.tcp_stats.mptcp_back_to_wifi);
4960 nstat_set_keyval_scalar(&kv[i++],
4961 NSTAT_SYSINFO_MPTCP_WIFI_PROXY,
4962 data->u.tcp_stats.mptcp_wifi_proxy);
4963 nstat_set_keyval_scalar(&kv[i++],
4964 NSTAT_SYSINFO_MPTCP_CELL_PROXY,
4965 data->u.tcp_stats.mptcp_cell_proxy);
4966 nstat_set_keyval_scalar(&kv[i++],
4967 NSTAT_SYSINFO_MPTCP_TRIGGERED_CELL,
4968 data->u.tcp_stats.mptcp_triggered_cell);
4969 VERIFY(i == nkeyvals);
4970 break;
4971 }
4972 case NSTAT_SYSINFO_IFNET_ECN_STATS:
4973 {
4974 nstat_set_keyval_scalar(&kv[i++],
4975 NSTAT_SYSINFO_ECN_IFNET_TYPE,
4976 data->u.ifnet_ecn_stats.ifnet_type);
4977 nstat_set_keyval_scalar(&kv[i++],
4978 NSTAT_SYSINFO_ECN_IFNET_PROTO,
4979 data->u.ifnet_ecn_stats.ifnet_proto);
4980 nstat_set_keyval_u64_scalar(&kv[i++],
4981 NSTAT_SYSINFO_ECN_IFNET_CLIENT_SETUP,
4982 data->u.ifnet_ecn_stats.ecn_stat.ecn_client_setup);
4983 nstat_set_keyval_u64_scalar(&kv[i++],
4984 NSTAT_SYSINFO_ECN_IFNET_SERVER_SETUP,
4985 data->u.ifnet_ecn_stats.ecn_stat.ecn_server_setup);
4986 nstat_set_keyval_u64_scalar(&kv[i++],
4987 NSTAT_SYSINFO_ECN_IFNET_CLIENT_SUCCESS,
4988 data->u.ifnet_ecn_stats.ecn_stat.ecn_client_success);
4989 nstat_set_keyval_u64_scalar(&kv[i++],
4990 NSTAT_SYSINFO_ECN_IFNET_SERVER_SUCCESS,
4991 data->u.ifnet_ecn_stats.ecn_stat.ecn_server_success);
4992 nstat_set_keyval_u64_scalar(&kv[i++],
4993 NSTAT_SYSINFO_ECN_IFNET_PEER_NOSUPPORT,
4994 data->u.ifnet_ecn_stats.ecn_stat.ecn_peer_nosupport);
4995 nstat_set_keyval_u64_scalar(&kv[i++],
4996 NSTAT_SYSINFO_ECN_IFNET_SYN_LOST,
4997 data->u.ifnet_ecn_stats.ecn_stat.ecn_syn_lost);
4998 nstat_set_keyval_u64_scalar(&kv[i++],
4999 NSTAT_SYSINFO_ECN_IFNET_SYNACK_LOST,
5000 data->u.ifnet_ecn_stats.ecn_stat.ecn_synack_lost);
5001 nstat_set_keyval_u64_scalar(&kv[i++],
5002 NSTAT_SYSINFO_ECN_IFNET_RECV_CE,
5003 data->u.ifnet_ecn_stats.ecn_stat.ecn_recv_ce);
5004 nstat_set_keyval_u64_scalar(&kv[i++],
5005 NSTAT_SYSINFO_ECN_IFNET_RECV_ECE,
5006 data->u.ifnet_ecn_stats.ecn_stat.ecn_recv_ece);
5007 nstat_set_keyval_u64_scalar(&kv[i++],
5008 NSTAT_SYSINFO_ECN_IFNET_CONN_RECV_CE,
5009 data->u.ifnet_ecn_stats.ecn_stat.ecn_conn_recv_ce);
5010 nstat_set_keyval_u64_scalar(&kv[i++],
5011 NSTAT_SYSINFO_ECN_IFNET_CONN_RECV_ECE,
5012 data->u.ifnet_ecn_stats.ecn_stat.ecn_conn_recv_ece);
5013 nstat_set_keyval_u64_scalar(&kv[i++],
5014 NSTAT_SYSINFO_ECN_IFNET_CONN_PLNOCE,
5015 data->u.ifnet_ecn_stats.ecn_stat.ecn_conn_plnoce);
5016 nstat_set_keyval_u64_scalar(&kv[i++],
5017 NSTAT_SYSINFO_ECN_IFNET_CONN_PLCE,
5018 data->u.ifnet_ecn_stats.ecn_stat.ecn_conn_plce);
5019 nstat_set_keyval_u64_scalar(&kv[i++],
5020 NSTAT_SYSINFO_ECN_IFNET_CONN_NOPLCE,
5021 data->u.ifnet_ecn_stats.ecn_stat.ecn_conn_noplce);
5022 nstat_set_keyval_u64_scalar(&kv[i++],
5023 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNLOSS,
5024 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_synloss);
5025 nstat_set_keyval_u64_scalar(&kv[i++],
5026 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_REORDER,
5027 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_reorder);
5028 nstat_set_keyval_u64_scalar(&kv[i++],
5029 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_CE,
5030 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_ce);
5031 nstat_set_keyval_u64_scalar(&kv[i++],
5032 NSTAT_SYSINFO_ECN_IFNET_ON_RTT_AVG,
5033 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.rtt_avg);
5034 nstat_set_keyval_u64_scalar(&kv[i++],
5035 NSTAT_SYSINFO_ECN_IFNET_ON_RTT_VAR,
5036 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.rtt_var);
5037 nstat_set_keyval_u64_scalar(&kv[i++],
5038 NSTAT_SYSINFO_ECN_IFNET_ON_OOPERCENT,
5039 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.oo_percent);
5040 nstat_set_keyval_u64_scalar(&kv[i++],
5041 NSTAT_SYSINFO_ECN_IFNET_ON_SACK_EPISODE,
5042 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.sack_episodes);
5043 nstat_set_keyval_u64_scalar(&kv[i++],
5044 NSTAT_SYSINFO_ECN_IFNET_ON_REORDER_PERCENT,
5045 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.reorder_percent);
5046 nstat_set_keyval_u64_scalar(&kv[i++],
5047 NSTAT_SYSINFO_ECN_IFNET_ON_RXMIT_PERCENT,
5048 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.rxmit_percent);
5049 nstat_set_keyval_u64_scalar(&kv[i++],
5050 NSTAT_SYSINFO_ECN_IFNET_ON_RXMIT_DROP,
5051 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.rxmit_drop);
5052 nstat_set_keyval_u64_scalar(&kv[i++],
5053 NSTAT_SYSINFO_ECN_IFNET_OFF_RTT_AVG,
5054 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.rtt_avg);
5055 nstat_set_keyval_u64_scalar(&kv[i++],
5056 NSTAT_SYSINFO_ECN_IFNET_OFF_RTT_VAR,
5057 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.rtt_var);
5058 nstat_set_keyval_u64_scalar(&kv[i++],
5059 NSTAT_SYSINFO_ECN_IFNET_OFF_OOPERCENT,
5060 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.oo_percent);
5061 nstat_set_keyval_u64_scalar(&kv[i++],
5062 NSTAT_SYSINFO_ECN_IFNET_OFF_SACK_EPISODE,
5063 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.sack_episodes);
5064 nstat_set_keyval_u64_scalar(&kv[i++],
5065 NSTAT_SYSINFO_ECN_IFNET_OFF_REORDER_PERCENT,
5066 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.reorder_percent);
5067 nstat_set_keyval_u64_scalar(&kv[i++],
5068 NSTAT_SYSINFO_ECN_IFNET_OFF_RXMIT_PERCENT,
5069 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.rxmit_percent);
5070 nstat_set_keyval_u64_scalar(&kv[i++],
5071 NSTAT_SYSINFO_ECN_IFNET_OFF_RXMIT_DROP,
5072 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.rxmit_drop);
5073 nstat_set_keyval_u64_scalar(&kv[i++],
5074 NSTAT_SYSINFO_ECN_IFNET_ON_TOTAL_TXPKTS,
5075 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.total_txpkts);
5076 nstat_set_keyval_u64_scalar(&kv[i++],
5077 NSTAT_SYSINFO_ECN_IFNET_ON_TOTAL_RXMTPKTS,
5078 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.total_rxmitpkts);
5079 nstat_set_keyval_u64_scalar(&kv[i++],
5080 NSTAT_SYSINFO_ECN_IFNET_ON_TOTAL_RXPKTS,
5081 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.total_rxpkts);
5082 nstat_set_keyval_u64_scalar(&kv[i++],
5083 NSTAT_SYSINFO_ECN_IFNET_ON_TOTAL_OOPKTS,
5084 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.total_oopkts);
5085 nstat_set_keyval_u64_scalar(&kv[i++],
5086 NSTAT_SYSINFO_ECN_IFNET_ON_DROP_RST,
5087 data->u.ifnet_ecn_stats.ecn_stat.ecn_on.rst_drop);
5088 nstat_set_keyval_u64_scalar(&kv[i++],
5089 NSTAT_SYSINFO_ECN_IFNET_OFF_TOTAL_TXPKTS,
5090 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.total_txpkts);
5091 nstat_set_keyval_u64_scalar(&kv[i++],
5092 NSTAT_SYSINFO_ECN_IFNET_OFF_TOTAL_RXMTPKTS,
5093 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.total_rxmitpkts);
5094 nstat_set_keyval_u64_scalar(&kv[i++],
5095 NSTAT_SYSINFO_ECN_IFNET_OFF_TOTAL_RXPKTS,
5096 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.total_rxpkts);
5097 nstat_set_keyval_u64_scalar(&kv[i++],
5098 NSTAT_SYSINFO_ECN_IFNET_OFF_TOTAL_OOPKTS,
5099 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.total_oopkts);
5100 nstat_set_keyval_u64_scalar(&kv[i++],
5101 NSTAT_SYSINFO_ECN_IFNET_OFF_DROP_RST,
5102 data->u.ifnet_ecn_stats.ecn_stat.ecn_off.rst_drop);
5103 nstat_set_keyval_u64_scalar(&kv[i++],
5104 NSTAT_SYSINFO_ECN_IFNET_TOTAL_CONN,
5105 data->u.ifnet_ecn_stats.ecn_stat.ecn_total_conn);
5106 nstat_set_keyval_scalar(&kv[i++],
5107 NSTAT_SYSINFO_IFNET_UNSENT_DATA,
5108 data->unsent_data_cnt);
5109 nstat_set_keyval_u64_scalar(&kv[i++],
5110 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRST,
5111 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_droprst);
5112 nstat_set_keyval_u64_scalar(&kv[i++],
5113 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRXMT,
5114 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_droprxmt);
5115 nstat_set_keyval_u64_scalar(&kv[i++],
5116 NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNRST,
5117 data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_synrst);
5118 break;
5119 }
5120 case NSTAT_SYSINFO_LIM_STATS:
5121 {
5122 nstat_set_keyval_string(&kv[i++],
5123 NSTAT_SYSINFO_LIM_IFNET_SIGNATURE,
5124 data->u.lim_stats.ifnet_signature,
5125 min(data->u.lim_stats.ifnet_siglen, NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE));
5126 nstat_set_keyval_u64_scalar(&kv[i++],
5127 NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH,
5128 data->u.lim_stats.lim_stat.lim_dl_max_bandwidth);
5129 nstat_set_keyval_u64_scalar(&kv[i++],
5130 NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH,
5131 data->u.lim_stats.lim_stat.lim_ul_max_bandwidth);
5132 nstat_set_keyval_u64_scalar(&kv[i++],
5133 NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT,
5134 data->u.lim_stats.lim_stat.lim_packet_loss_percent);
5135 nstat_set_keyval_u64_scalar(&kv[i++],
5136 NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT,
5137 data->u.lim_stats.lim_stat.lim_packet_ooo_percent);
5138 nstat_set_keyval_u64_scalar(&kv[i++],
5139 NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE,
5140 data->u.lim_stats.lim_stat.lim_rtt_variance);
5141 nstat_set_keyval_u64_scalar(&kv[i++],
5142 NSTAT_SYSINFO_LIM_IFNET_RTT_MIN,
5143 data->u.lim_stats.lim_stat.lim_rtt_min);
5144 nstat_set_keyval_u64_scalar(&kv[i++],
5145 NSTAT_SYSINFO_LIM_IFNET_RTT_AVG,
5146 data->u.lim_stats.lim_stat.lim_rtt_average);
5147 nstat_set_keyval_u64_scalar(&kv[i++],
5148 NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT,
5149 data->u.lim_stats.lim_stat.lim_conn_timeout_percent);
5150 nstat_set_keyval_scalar(&kv[i++],
5151 NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED,
5152 data->u.lim_stats.lim_stat.lim_dl_detected);
5153 nstat_set_keyval_scalar(&kv[i++],
5154 NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED,
5155 data->u.lim_stats.lim_stat.lim_ul_detected);
5156 nstat_set_keyval_scalar(&kv[i++],
5157 NSTAT_SYSINFO_LIM_IFNET_TYPE,
5158 data->u.lim_stats.ifnet_type);
5159 break;
5160 }
5161 case NSTAT_SYSINFO_NET_API_STATS:
5162 {
5163 nstat_set_keyval_u64_scalar(&kv[i++],
5164 NSTAT_SYSINFO_API_IF_FLTR_ATTACH,
5165 data->u.net_api_stats.net_api_stats.nas_iflt_attach_total);
5166 nstat_set_keyval_u64_scalar(&kv[i++],
5167 NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS,
5168 data->u.net_api_stats.net_api_stats.nas_iflt_attach_os_total);
5169 nstat_set_keyval_u64_scalar(&kv[i++],
5170 NSTAT_SYSINFO_API_IP_FLTR_ADD,
5171 data->u.net_api_stats.net_api_stats.nas_ipf_add_total);
5172 nstat_set_keyval_u64_scalar(&kv[i++],
5173 NSTAT_SYSINFO_API_IP_FLTR_ADD_OS,
5174 data->u.net_api_stats.net_api_stats.nas_ipf_add_os_total);
5175 nstat_set_keyval_u64_scalar(&kv[i++],
5176 NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH,
5177 data->u.net_api_stats.net_api_stats.nas_sfltr_register_total);
5178 nstat_set_keyval_u64_scalar(&kv[i++],
5179 NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS,
5180 data->u.net_api_stats.net_api_stats.nas_sfltr_register_os_total);
5181
5182
5183 nstat_set_keyval_u64_scalar(&kv[i++],
5184 NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL,
5185 data->u.net_api_stats.net_api_stats.nas_socket_alloc_total);
5186 nstat_set_keyval_u64_scalar(&kv[i++],
5187 NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL,
5188 data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_total);
5189 nstat_set_keyval_u64_scalar(&kv[i++],
5190 NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS,
5191 data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_os_total);
5192 nstat_set_keyval_u64_scalar(&kv[i++],
5193 NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID,
5194 data->u.net_api_stats.net_api_stats.nas_socket_necp_clientuuid_total);
5195
5196 nstat_set_keyval_u64_scalar(&kv[i++],
5197 NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL,
5198 data->u.net_api_stats.net_api_stats.nas_socket_domain_local_total);
5199 nstat_set_keyval_u64_scalar(&kv[i++],
5200 NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE,
5201 data->u.net_api_stats.net_api_stats.nas_socket_domain_route_total);
5202 nstat_set_keyval_u64_scalar(&kv[i++],
5203 NSTAT_SYSINFO_API_SOCK_DOMAIN_INET,
5204 data->u.net_api_stats.net_api_stats.nas_socket_domain_inet_total);
5205 nstat_set_keyval_u64_scalar(&kv[i++],
5206 NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6,
5207 data->u.net_api_stats.net_api_stats.nas_socket_domain_inet6_total);
5208 nstat_set_keyval_u64_scalar(&kv[i++],
5209 NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM,
5210 data->u.net_api_stats.net_api_stats.nas_socket_domain_system_total);
5211 nstat_set_keyval_u64_scalar(&kv[i++],
5212 NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH,
5213 data->u.net_api_stats.net_api_stats.nas_socket_domain_multipath_total);
5214 nstat_set_keyval_u64_scalar(&kv[i++],
5215 NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY,
5216 data->u.net_api_stats.net_api_stats.nas_socket_domain_key_total);
5217 nstat_set_keyval_u64_scalar(&kv[i++],
5218 NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV,
5219 data->u.net_api_stats.net_api_stats.nas_socket_domain_ndrv_total);
5220 nstat_set_keyval_u64_scalar(&kv[i++],
5221 NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER,
5222 data->u.net_api_stats.net_api_stats.nas_socket_domain_other_total);
5223
5224 nstat_set_keyval_u64_scalar(&kv[i++],
5225 NSTAT_SYSINFO_API_SOCK_INET_STREAM,
5226 data->u.net_api_stats.net_api_stats.nas_socket_inet_stream_total);
5227 nstat_set_keyval_u64_scalar(&kv[i++],
5228 NSTAT_SYSINFO_API_SOCK_INET_DGRAM,
5229 data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_total);
5230 nstat_set_keyval_u64_scalar(&kv[i++],
5231 NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED,
5232 data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_connected);
5233 nstat_set_keyval_u64_scalar(&kv[i++],
5234 NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS,
5235 data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_dns);
5236 nstat_set_keyval_u64_scalar(&kv[i++],
5237 NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA,
5238 data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_no_data);
5239
5240 nstat_set_keyval_u64_scalar(&kv[i++],
5241 NSTAT_SYSINFO_API_SOCK_INET6_STREAM,
5242 data->u.net_api_stats.net_api_stats.nas_socket_inet6_stream_total);
5243 nstat_set_keyval_u64_scalar(&kv[i++],
5244 NSTAT_SYSINFO_API_SOCK_INET6_DGRAM,
5245 data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_total);
5246 nstat_set_keyval_u64_scalar(&kv[i++],
5247 NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED,
5248 data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_connected);
5249 nstat_set_keyval_u64_scalar(&kv[i++],
5250 NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS,
5251 data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_dns);
5252 nstat_set_keyval_u64_scalar(&kv[i++],
5253 NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA,
5254 data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_no_data);
5255
5256 nstat_set_keyval_u64_scalar(&kv[i++],
5257 NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN,
5258 data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_total);
5259 nstat_set_keyval_u64_scalar(&kv[i++],
5260 NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS,
5261 data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_os_total);
5262
5263 nstat_set_keyval_u64_scalar(&kv[i++],
5264 NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM,
5265 data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_stream_total);
5266 nstat_set_keyval_u64_scalar(&kv[i++],
5267 NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM,
5268 data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_dgram_total);
5269
5270 nstat_set_keyval_u64_scalar(&kv[i++],
5271 NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM,
5272 data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_stream_total);
5273 nstat_set_keyval_u64_scalar(&kv[i++],
5274 NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM,
5275 data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_dgram_total);
5276
5277 nstat_set_keyval_u64_scalar(&kv[i++],
5278 NSTAT_SYSINFO_API_IFNET_ALLOC,
5279 data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_total);
5280 nstat_set_keyval_u64_scalar(&kv[i++],
5281 NSTAT_SYSINFO_API_IFNET_ALLOC_OS,
5282 data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_os_total);
5283
5284 nstat_set_keyval_u64_scalar(&kv[i++],
5285 NSTAT_SYSINFO_API_PF_ADDRULE,
5286 data->u.net_api_stats.net_api_stats.nas_pf_addrule_total);
5287 nstat_set_keyval_u64_scalar(&kv[i++],
5288 NSTAT_SYSINFO_API_PF_ADDRULE_OS,
5289 data->u.net_api_stats.net_api_stats.nas_pf_addrule_os);
5290
5291 nstat_set_keyval_u64_scalar(&kv[i++],
5292 NSTAT_SYSINFO_API_VMNET_START,
5293 data->u.net_api_stats.net_api_stats.nas_vmnet_total);
5294
5295 #if SKYWALK
5296 nstat_set_keyval_scalar(&kv[i++],
5297 NSTAT_SYSINFO_API_IF_NETAGENT_ENABLED,
5298 if_is_fsw_transport_netagent_enabled());
5299 #endif /* SKYWALK */
5300
5301 nstat_set_keyval_scalar(&kv[i++],
5302 NSTAT_SYSINFO_API_REPORT_INTERVAL,
5303 data->u.net_api_stats.report_interval);
5304
5305 break;
5306 }
5307 }
5308 if (syscnt != NULL) {
5309 VERIFY(i > 0 && i <= nkeyvals);
5310 countsize = offsetof(nstat_sysinfo_counts,
5311 nstat_sysinfo_keyvals) +
5312 sizeof(nstat_sysinfo_keyval) * i;
5313 finalsize += countsize;
5314 syscnt->hdr.type = NSTAT_MSG_TYPE_SYSINFO_COUNTS;
5315 assert(finalsize <= MAX_NSTAT_MSG_HDR_LENGTH);
5316 syscnt->hdr.length = (u_int16_t)finalsize;
5317 syscnt->counts.nstat_sysinfo_len = (u_int32_t)countsize;
5318
5319 result = ctl_enqueuedata(client->ntc_kctl,
5320 client->ntc_unit, syscnt, finalsize, CTL_DATA_EOR);
5321 if (result != 0) {
5322 nstat_stats.nstat_sysinfofailures += 1;
5323 }
5324 kfree_data(syscnt, allocsize);
5325 }
5326 return;
5327 }
5328
5329 __private_extern__ void
nstat_sysinfo_send_data(nstat_sysinfo_data * data)5330 nstat_sysinfo_send_data(
5331 nstat_sysinfo_data *data)
5332 {
5333 nstat_client *client;
5334
5335 NSTAT_LOCK_EXCLUSIVE();
5336 for (client = nstat_clients; client; client = client->ntc_next) {
5337 if ((client->ntc_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) {
5338 nstat_sysinfo_send_data_internal(client, data);
5339 }
5340 }
5341 NSTAT_UNLOCK_EXCLUSIVE();
5342 }
5343
5344 static void
nstat_sysinfo_generate_report(void)5345 nstat_sysinfo_generate_report(void)
5346 {
5347 tcp_report_stats();
5348 nstat_ifnet_report_ecn_stats();
5349 nstat_ifnet_report_lim_stats();
5350 nstat_net_api_report_stats();
5351 }
5352
5353 #pragma mark -- net_api --
5354
5355 static struct net_api_stats net_api_stats_before;
5356 static u_int64_t net_api_stats_last_report_time;
5357
5358 static void
nstat_net_api_report_stats(void)5359 nstat_net_api_report_stats(void)
5360 {
5361 struct nstat_sysinfo_data data;
5362 struct nstat_sysinfo_net_api_stats *st = &data.u.net_api_stats;
5363 u_int64_t uptime;
5364
5365 uptime = net_uptime();
5366
5367 if ((u_int32_t)(uptime - net_api_stats_last_report_time) <
5368 net_api_stats_report_interval) {
5369 return;
5370 }
5371
5372 st->report_interval = (u_int32_t)(uptime - net_api_stats_last_report_time);
5373 net_api_stats_last_report_time = uptime;
5374
5375 data.flags = NSTAT_SYSINFO_NET_API_STATS;
5376 data.unsent_data_cnt = 0;
5377
5378 /*
5379 * Some of the fields in the report are the current value and
5380 * other fields are the delta from the last report:
5381 * - Report difference for the per flow counters as they increase
5382 * with time
5383 * - Report current value for other counters as they tend not to change
5384 * much with time
5385 */
5386 #define STATCOPY(f) \
5387 (st->net_api_stats.f = net_api_stats.f)
5388 #define STATDIFF(f) \
5389 (st->net_api_stats.f = net_api_stats.f - net_api_stats_before.f)
5390
5391 STATCOPY(nas_iflt_attach_count);
5392 STATCOPY(nas_iflt_attach_total);
5393 STATCOPY(nas_iflt_attach_os_total);
5394
5395 STATCOPY(nas_ipf_add_count);
5396 STATCOPY(nas_ipf_add_total);
5397 STATCOPY(nas_ipf_add_os_total);
5398
5399 STATCOPY(nas_sfltr_register_count);
5400 STATCOPY(nas_sfltr_register_total);
5401 STATCOPY(nas_sfltr_register_os_total);
5402
5403 STATDIFF(nas_socket_alloc_total);
5404 STATDIFF(nas_socket_in_kernel_total);
5405 STATDIFF(nas_socket_in_kernel_os_total);
5406 STATDIFF(nas_socket_necp_clientuuid_total);
5407
5408 STATDIFF(nas_socket_domain_local_total);
5409 STATDIFF(nas_socket_domain_route_total);
5410 STATDIFF(nas_socket_domain_inet_total);
5411 STATDIFF(nas_socket_domain_inet6_total);
5412 STATDIFF(nas_socket_domain_system_total);
5413 STATDIFF(nas_socket_domain_multipath_total);
5414 STATDIFF(nas_socket_domain_key_total);
5415 STATDIFF(nas_socket_domain_ndrv_total);
5416 STATDIFF(nas_socket_domain_other_total);
5417
5418 STATDIFF(nas_socket_inet_stream_total);
5419 STATDIFF(nas_socket_inet_dgram_total);
5420 STATDIFF(nas_socket_inet_dgram_connected);
5421 STATDIFF(nas_socket_inet_dgram_dns);
5422 STATDIFF(nas_socket_inet_dgram_no_data);
5423
5424 STATDIFF(nas_socket_inet6_stream_total);
5425 STATDIFF(nas_socket_inet6_dgram_total);
5426 STATDIFF(nas_socket_inet6_dgram_connected);
5427 STATDIFF(nas_socket_inet6_dgram_dns);
5428 STATDIFF(nas_socket_inet6_dgram_no_data);
5429
5430 STATDIFF(nas_socket_mcast_join_total);
5431 STATDIFF(nas_socket_mcast_join_os_total);
5432
5433 STATDIFF(nas_sock_inet6_stream_exthdr_in);
5434 STATDIFF(nas_sock_inet6_stream_exthdr_out);
5435 STATDIFF(nas_sock_inet6_dgram_exthdr_in);
5436 STATDIFF(nas_sock_inet6_dgram_exthdr_out);
5437
5438 STATDIFF(nas_nx_flow_inet_stream_total);
5439 STATDIFF(nas_nx_flow_inet_dgram_total);
5440
5441 STATDIFF(nas_nx_flow_inet6_stream_total);
5442 STATDIFF(nas_nx_flow_inet6_dgram_total);
5443
5444 STATCOPY(nas_ifnet_alloc_count);
5445 STATCOPY(nas_ifnet_alloc_total);
5446 STATCOPY(nas_ifnet_alloc_os_count);
5447 STATCOPY(nas_ifnet_alloc_os_total);
5448
5449 STATCOPY(nas_pf_addrule_total);
5450 STATCOPY(nas_pf_addrule_os);
5451
5452 STATCOPY(nas_vmnet_total);
5453
5454 #undef STATCOPY
5455 #undef STATDIFF
5456
5457 nstat_sysinfo_send_data(&data);
5458
5459 /*
5460 * Save a copy of the current fields so we can diff them the next time
5461 */
5462 memcpy(&net_api_stats_before, &net_api_stats,
5463 sizeof(struct net_api_stats));
5464 _CASSERT(sizeof(net_api_stats_before) == sizeof(net_api_stats));
5465 }
5466
5467
5468 #pragma mark -- Kernel Control Socket --
5469
5470 static kern_ctl_ref nstat_ctlref = NULL;
5471
5472 static errno_t nstat_client_connect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo);
5473 static errno_t nstat_client_disconnect(kern_ctl_ref kctl, u_int32_t unit, void *uinfo);
5474 static errno_t nstat_client_send(kern_ctl_ref kctl, u_int32_t unit, void *uinfo, mbuf_t m, int flags);
5475
5476 static errno_t
nstat_enqueue_success(uint64_t context,nstat_client * client,u_int16_t flags)5477 nstat_enqueue_success(
5478 uint64_t context,
5479 nstat_client *client,
5480 u_int16_t flags)
5481 {
5482 nstat_msg_hdr success;
5483 errno_t result;
5484
5485 bzero(&success, sizeof(success));
5486 success.context = context;
5487 success.type = NSTAT_MSG_TYPE_SUCCESS;
5488 success.length = sizeof(success);
5489 success.flags = flags;
5490 result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &success,
5491 sizeof(success), CTL_DATA_EOR | CTL_DATA_CRIT);
5492 if (result != 0) {
5493 if (nstat_debug != 0) {
5494 NSTAT_LOG_ERROR("could not enqueue success message %d", result);
5495 }
5496 nstat_stats.nstat_successmsgfailures += 1;
5497 }
5498 return result;
5499 }
5500
5501 static errno_t
nstat_client_send_event(nstat_client * client,nstat_src * src,u_int64_t event)5502 nstat_client_send_event(
5503 nstat_client *client,
5504 nstat_src *src,
5505 u_int64_t event)
5506 {
5507 errno_t result = ENOTSUP;
5508 int failed = 0;
5509
5510 if (nstat_client_reporting_allowed(client, src, 0)) {
5511 if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_UPDATES) != 0) {
5512 result = nstat_client_send_update(client, src, 0, event, 0, NULL);
5513 if (result != 0) {
5514 failed = 1;
5515 if (nstat_debug != 0) {
5516 NSTAT_LOG_ERROR("nstat_client_send_event() %d", result);
5517 }
5518 }
5519 } else {
5520 if (nstat_debug != 0) {
5521 NSTAT_LOG_ERROR("nstat_client_send_event() used when updates not supported");
5522 }
5523 }
5524 }
5525 return result;
5526 }
5527
5528 static errno_t
nstat_client_send_goodbye(nstat_client * client,nstat_src * src)5529 nstat_client_send_goodbye(
5530 nstat_client *client,
5531 nstat_src *src)
5532 {
5533 errno_t result = 0;
5534 int failed = 0;
5535 u_int16_t hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_FILTER;
5536
5537 if (nstat_client_reporting_allowed(client, src, (src->nts_reported)? NSTAT_FILTER_SUPPRESS_BORING_CLOSE: 0)) {
5538 hdr_flags = 0;
5539 if ((client->ntc_flags & NSTAT_FLAG_SUPPORTS_UPDATES) != 0) {
5540 result = nstat_client_send_update(client, src, 0, 0, NSTAT_MSG_HDR_FLAG_CLOSING, NULL);
5541 if (result != 0) {
5542 failed = 1;
5543 hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP;
5544 if (nstat_debug != 0) {
5545 NSTAT_LOG_ERROR("nstat_client_send_update() %d", result);
5546 }
5547 }
5548 } else {
5549 // send one last counts notification
5550 result = nstat_client_send_counts(client, src, 0, NSTAT_MSG_HDR_FLAG_CLOSING, NULL);
5551 if (result != 0) {
5552 failed = 1;
5553 hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP;
5554 if (nstat_debug != 0) {
5555 NSTAT_LOG_ERROR("nstat_client_send_counts() %d", result);
5556 }
5557 }
5558
5559 // send a last description
5560 result = nstat_client_send_description(client, src, 0, NSTAT_MSG_HDR_FLAG_CLOSING);
5561 if (result != 0) {
5562 failed = 1;
5563 hdr_flags = NSTAT_MSG_HDR_FLAG_CLOSED_AFTER_DROP;
5564 if (nstat_debug != 0) {
5565 NSTAT_LOG_ERROR("nstat_client_send_description() %d", result);
5566 }
5567 }
5568 }
5569 }
5570
5571 // send the source removed notification
5572 result = nstat_client_send_removed(client, src, hdr_flags);
5573 if (result != 0 && nstat_debug) {
5574 failed = 1;
5575 if (nstat_debug != 0) {
5576 NSTAT_LOG_ERROR("nstat_client_send_removed() %d", result);
5577 }
5578 }
5579
5580 if (failed != 0) {
5581 nstat_stats.nstat_control_send_goodbye_failures++;
5582 }
5583
5584
5585 return result;
5586 }
5587
5588 static errno_t
nstat_flush_accumulated_msgs(nstat_client * client)5589 nstat_flush_accumulated_msgs(
5590 nstat_client *client)
5591 {
5592 errno_t result = 0;
5593 if (client->ntc_accumulated != NULL && mbuf_len(client->ntc_accumulated) > 0) {
5594 mbuf_pkthdr_setlen(client->ntc_accumulated, mbuf_len(client->ntc_accumulated));
5595 result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, client->ntc_accumulated, CTL_DATA_EOR);
5596 if (result != 0) {
5597 nstat_stats.nstat_flush_accumulated_msgs_failures++;
5598 if (nstat_debug != 0) {
5599 NSTAT_LOG_ERROR("ctl_enqueuembuf failed: %d", result);
5600 }
5601 mbuf_freem(client->ntc_accumulated);
5602 }
5603 client->ntc_accumulated = NULL;
5604 }
5605 return result;
5606 }
5607
5608 static errno_t
nstat_accumulate_msg(nstat_client * client,uint8_t * __sized_by (length)msg,size_t length)5609 nstat_accumulate_msg(
5610 nstat_client *client,
5611 uint8_t *__sized_by(length) msg,
5612 size_t length)
5613 {
5614 assert(length <= MAX_NSTAT_MSG_HDR_LENGTH);
5615
5616 if (client->ntc_accumulated && mbuf_trailingspace(client->ntc_accumulated) < length) {
5617 // Will send the current mbuf
5618 nstat_flush_accumulated_msgs(client);
5619 }
5620
5621 errno_t result = 0;
5622
5623 if (client->ntc_accumulated == NULL) {
5624 unsigned int one = 1;
5625 if (mbuf_allocpacket(MBUF_DONTWAIT, NSTAT_MAX_MSG_SIZE, &one, &client->ntc_accumulated) != 0) {
5626 if (nstat_debug != 0) {
5627 NSTAT_LOG_ERROR("mbuf_allocpacket failed");
5628 }
5629 result = ENOMEM;
5630 } else {
5631 mbuf_setlen(client->ntc_accumulated, 0);
5632 }
5633 }
5634
5635 if (result == 0) {
5636 result = mbuf_copyback(client->ntc_accumulated, mbuf_len(client->ntc_accumulated),
5637 length, msg, MBUF_DONTWAIT);
5638 }
5639
5640 if (result != 0) {
5641 nstat_flush_accumulated_msgs(client);
5642 if (nstat_debug != 0) {
5643 NSTAT_LOG_ERROR("resorting to ctl_enqueuedata");
5644 }
5645 result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, msg, length, CTL_DATA_EOR);
5646 }
5647
5648 if (result != 0) {
5649 nstat_stats.nstat_accumulate_msg_failures++;
5650 }
5651
5652 return result;
5653 }
5654
5655 static void
nstat_idle_check(__unused thread_call_param_t p0,__unused thread_call_param_t p1)5656 nstat_idle_check(
5657 __unused thread_call_param_t p0,
5658 __unused thread_call_param_t p1)
5659 {
5660 nstat_client *client;
5661 nstat_src *src, *tmpsrc;
5662 tailq_head_nstat_src dead_list;
5663 TAILQ_INIT(&dead_list);
5664
5665 NSTAT_LOCK_EXCLUSIVE();
5666
5667 nstat_idle_time = 0;
5668
5669 for (client = nstat_clients; client; client = client->ntc_next) {
5670 TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc)
5671 {
5672 if (src->nts_provider->nstat_gone(src->nts_cookie)) {
5673 errno_t result;
5674
5675 // Pull it off the list
5676 NSTAT_NOTE_SRC(nstat_src_gone_idlecheck, client, src);
5677 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
5678
5679 result = nstat_client_send_goodbye(client, src);
5680
5681 // Put this on the list to release later
5682 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
5683 }
5684 }
5685 }
5686
5687 if (nstat_clients) {
5688 clock_interval_to_deadline(60, NSEC_PER_SEC, &nstat_idle_time);
5689 thread_call_func_delayed((thread_call_func_t)nstat_idle_check, NULL, nstat_idle_time);
5690 }
5691
5692 NSTAT_UNLOCK_EXCLUSIVE();
5693
5694 /* Generate any system level reports, if needed */
5695 nstat_sysinfo_generate_report();
5696
5697 // Release the sources now that we aren't holding lots of locks
5698 while ((src = TAILQ_FIRST(&dead_list))) {
5699 TAILQ_REMOVE(&dead_list, src, nts_client_link);
5700 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_src_idlecheck_gone);
5701 nstat_client_cleanup_source(NULL, src, FALSE);
5702 }
5703
5704 nstat_prune_procdetails();
5705 }
5706
5707 static void
nstat_client_register(void)5708 nstat_client_register(void)
5709 {
5710 // Register the control
5711 struct kern_ctl_reg nstat_control;
5712 bzero(&nstat_control, sizeof(nstat_control));
5713 strlcpy(nstat_control.ctl_name, NET_STAT_CONTROL_NAME, sizeof(nstat_control.ctl_name));
5714 nstat_control.ctl_flags = CTL_FLAG_REG_EXTENDED | CTL_FLAG_REG_CRIT;
5715 nstat_control.ctl_sendsize = nstat_sendspace;
5716 nstat_control.ctl_recvsize = nstat_recvspace;
5717 nstat_control.ctl_connect = nstat_client_connect;
5718 nstat_control.ctl_disconnect = nstat_client_disconnect;
5719 nstat_control.ctl_send = nstat_client_send;
5720
5721 ctl_register(&nstat_control, &nstat_ctlref);
5722 }
5723
5724 static void
nstat_client_cleanup_source(nstat_client * client,struct nstat_src * src,boolean_t locked)5725 nstat_client_cleanup_source(
5726 nstat_client *client,
5727 struct nstat_src *src,
5728 boolean_t locked)
5729 {
5730 errno_t result;
5731
5732 if (client) {
5733 result = nstat_client_send_removed(client, src, 0);
5734 if (result != 0) {
5735 nstat_stats.nstat_control_cleanup_source_failures++;
5736 if (nstat_debug != 0) {
5737 NSTAT_LOG_ERROR("nstat_client_send_removed() %d", result);
5738 }
5739 }
5740 }
5741 // Cleanup the source if we found it.
5742 src->nts_provider->nstat_release(src->nts_cookie, locked);
5743 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_src_current);
5744 kfree_type(struct nstat_src, src);
5745 }
5746
5747 static bool
nstat_client_reporting_allowed(nstat_client * client,nstat_src * src,u_int64_t suppression_flags)5748 nstat_client_reporting_allowed(
5749 nstat_client *client,
5750 nstat_src *src,
5751 u_int64_t suppression_flags)
5752 {
5753 if (src->nts_provider->nstat_reporting_allowed == NULL) {
5754 return TRUE;
5755 }
5756
5757 return src->nts_provider->nstat_reporting_allowed(src->nts_cookie,
5758 &client->ntc_provider_filters[src->nts_provider->nstat_provider_id], suppression_flags);
5759 }
5760
5761
5762 static errno_t
nstat_client_connect(kern_ctl_ref kctl,struct sockaddr_ctl * sac,void ** uinfo)5763 nstat_client_connect(
5764 kern_ctl_ref kctl,
5765 struct sockaddr_ctl *sac,
5766 void **uinfo)
5767 {
5768 nstat_client *client = kalloc_type(nstat_client,
5769 Z_WAITOK | Z_ZERO);
5770 if (client == NULL) {
5771 return ENOMEM;
5772 }
5773
5774 lck_mtx_init(&client->ntc_user_mtx, &nstat_lck_grp, NULL);
5775 client->ntc_kctl = kctl;
5776 client->ntc_unit = sac->sc_unit;
5777 client->ntc_flags = NSTAT_FLAG_REQCOUNTS;
5778 client->ntc_procdetails = nstat_retain_curprocdetails();
5779 *uinfo = client;
5780
5781 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_client_allocs);
5782 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_client_current, nstat_global_client_max);
5783 #if NSTAT_TRACE_ENABLED
5784 client->ntc_trace = (nstat_cyclic_trace *)kalloc_data(sizeof(nstat_cyclic_trace), Z_WAITOK | Z_ZERO);
5785 #endif
5786 NSTAT_LOCK_EXCLUSIVE();
5787 client->ntc_next = nstat_clients;
5788 nstat_clients = client;
5789
5790 if (nstat_idle_time == 0) {
5791 clock_interval_to_deadline(60, NSEC_PER_SEC, &nstat_idle_time);
5792 thread_call_func_delayed((thread_call_func_t)nstat_idle_check, NULL, nstat_idle_time);
5793 }
5794
5795 merge_current_event_filters();
5796 NSTAT_UNLOCK_EXCLUSIVE();
5797
5798 return 0;
5799 }
5800
5801 static errno_t
nstat_client_disconnect(__unused kern_ctl_ref kctl,__unused u_int32_t unit,void * uinfo)5802 nstat_client_disconnect(
5803 __unused kern_ctl_ref kctl,
5804 __unused u_int32_t unit,
5805 void *uinfo)
5806 {
5807 u_int32_t watching;
5808 nstat_client *client = (nstat_client*)uinfo;
5809 tailq_head_nstat_src cleanup_list;
5810 nstat_src *src;
5811
5812 TAILQ_INIT(&cleanup_list);
5813
5814 // pull it out of the global list of clients
5815 NSTAT_LOCK_EXCLUSIVE();
5816 nstat_client **clientpp;
5817 for (clientpp = &nstat_clients; *clientpp; clientpp = &(*clientpp)->ntc_next) {
5818 if (*clientpp == client) {
5819 *clientpp = client->ntc_next;
5820 break;
5821 }
5822 }
5823 merge_current_event_filters();
5824
5825 // Stop watching for sources
5826 nstat_provider *provider;
5827 watching = client->ntc_watching;
5828 client->ntc_watching = 0;
5829 for (provider = nstat_providers; provider && watching; provider = provider->next) {
5830 if ((watching & (1 << provider->nstat_provider_id)) != 0) {
5831 watching &= ~(1 << provider->nstat_provider_id);
5832 provider->nstat_watcher_remove(client);
5833 }
5834 }
5835
5836 // set cleanup flags
5837 client->ntc_flags |= NSTAT_FLAG_CLEANUP;
5838
5839 if (client->ntc_accumulated) {
5840 mbuf_freem(client->ntc_accumulated);
5841 client->ntc_accumulated = NULL;
5842 }
5843
5844 // Copy out the list of sources
5845 TAILQ_CONCAT(&cleanup_list, &client->ntc_src_queue, nts_client_link);
5846
5847 NSTAT_UNLOCK_EXCLUSIVE();
5848
5849 while ((src = TAILQ_FIRST(&cleanup_list))) {
5850 TAILQ_REMOVE(&cleanup_list, src, nts_client_link);
5851 nstat_client_cleanup_source(NULL, src, FALSE);
5852 }
5853
5854 lck_mtx_destroy(&client->ntc_user_mtx, &nstat_lck_grp);
5855 nstat_release_procdetails(client->ntc_procdetails);
5856 nstat_accumulate_client_metrics(client);
5857 #if NSTAT_TRACE_ENABLED
5858 if (client->ntc_trace != NULL) {
5859 kfree_data(client->ntc_trace, sizeof(nstat_cyclic_trace));
5860 }
5861 #endif
5862 NSTAT_GLOBAL_COUNT_DECREMENT(nstat_global_client_current);
5863 kfree_type(struct nstat_client, client);
5864
5865 return 0;
5866 }
5867
5868 static nstat_src_ref_t
nstat_client_next_src_ref(nstat_client * client)5869 nstat_client_next_src_ref(
5870 nstat_client *client)
5871 {
5872 return ++client->ntc_next_srcref;
5873 }
5874
5875 static errno_t
nstat_client_send_counts(nstat_client * client,nstat_src * src,unsigned long long context,u_int16_t hdr_flags,int * gone)5876 nstat_client_send_counts(
5877 nstat_client *client,
5878 nstat_src *src,
5879 unsigned long long context,
5880 u_int16_t hdr_flags,
5881 int *gone)
5882 {
5883 nstat_msg_src_counts counts;
5884 errno_t result = 0;
5885
5886 /* Some providers may not have any counts to send */
5887 if (src->nts_provider->nstat_counts == NULL) {
5888 return 0;
5889 }
5890
5891 bzero(&counts, sizeof(counts));
5892 counts.hdr.type = NSTAT_MSG_TYPE_SRC_COUNTS;
5893 counts.hdr.length = sizeof(counts);
5894 counts.hdr.flags = hdr_flags;
5895 counts.hdr.context = context;
5896 counts.srcref = src->nts_srcref;
5897 counts.event_flags = 0;
5898
5899 if (src->nts_provider->nstat_counts(src->nts_cookie, &counts.counts, gone) == 0) {
5900 if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) &&
5901 counts.counts.nstat_rxbytes == 0 &&
5902 counts.counts.nstat_txbytes == 0) {
5903 result = EAGAIN;
5904 } else {
5905 result = ctl_enqueuedata(client->ntc_kctl,
5906 client->ntc_unit, &counts, sizeof(counts),
5907 CTL_DATA_EOR);
5908 if (result != 0) {
5909 nstat_stats.nstat_sendcountfailures += 1;
5910 }
5911 }
5912 }
5913 return result;
5914 }
5915
5916 static errno_t
nstat_client_append_counts(nstat_client * client,nstat_src * src,int * gone)5917 nstat_client_append_counts(
5918 nstat_client *client,
5919 nstat_src *src,
5920 int *gone)
5921 {
5922 /* Some providers may not have any counts to send */
5923 if (!src->nts_provider->nstat_counts) {
5924 return 0;
5925 }
5926
5927 nstat_msg_src_counts counts;
5928 bzero(&counts, sizeof(counts));
5929 counts.hdr.type = NSTAT_MSG_TYPE_SRC_COUNTS;
5930 counts.hdr.length = sizeof(counts);
5931 counts.srcref = src->nts_srcref;
5932 counts.event_flags = 0;
5933
5934 errno_t result = 0;
5935 result = src->nts_provider->nstat_counts(src->nts_cookie, &counts.counts, gone);
5936 if (result != 0) {
5937 return result;
5938 }
5939
5940 if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES &&
5941 counts.counts.nstat_rxbytes == 0 && counts.counts.nstat_txbytes == 0) {
5942 return EAGAIN;
5943 }
5944
5945 return nstat_accumulate_msg(client, (uint8_t *)&counts, sizeof(counts));
5946 }
5947
5948 static int
nstat_client_send_description(nstat_client * client,nstat_src * src,u_int64_t context,u_int16_t hdr_flags)5949 nstat_client_send_description(
5950 nstat_client *client,
5951 nstat_src *src,
5952 u_int64_t context,
5953 u_int16_t hdr_flags)
5954 {
5955 // Provider doesn't support getting the descriptor? Done.
5956 if (src->nts_provider->nstat_descriptor_length == 0 ||
5957 src->nts_provider->nstat_copy_descriptor == NULL) {
5958 return EOPNOTSUPP;
5959 }
5960
5961 // Allocate storage for the descriptor message
5962 mbuf_ref_t msg;
5963 unsigned int one = 1;
5964 size_t size = offsetof(nstat_msg_src_description, data) + src->nts_provider->nstat_descriptor_length;
5965 assert(size <= MAX_NSTAT_MSG_HDR_LENGTH);
5966
5967 if (mbuf_allocpacket(MBUF_DONTWAIT, size, &one, &msg) != 0) {
5968 return ENOMEM;
5969 }
5970
5971
5972 bzero(m_mtod_current(msg), size);
5973 mbuf_setlen(msg, size);
5974 mbuf_pkthdr_setlen(msg, mbuf_len(msg));
5975
5976 // Query the provider for the provider specific bits
5977 nstat_msg_src_description *desc = mtod(msg, nstat_msg_src_description *);
5978 desc->hdr.context = context;
5979 desc->hdr.type = NSTAT_MSG_TYPE_SRC_DESC;
5980 desc->hdr.length = (u_int16_t)size;
5981 desc->hdr.flags = hdr_flags;
5982 desc->srcref = src->nts_srcref;
5983 desc->event_flags = 0;
5984 desc->provider = src->nts_provider->nstat_provider_id;
5985
5986 uint8_t *desc_data_ptr = nstat_get_data(desc);
5987 errno_t result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr, src->nts_provider->nstat_descriptor_length);
5988
5989 if (result != 0) {
5990 mbuf_freem(msg);
5991 return result;
5992 }
5993
5994 result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR);
5995 if (result != 0) {
5996 nstat_stats.nstat_descriptionfailures += 1;
5997 mbuf_freem(msg);
5998 }
5999
6000 return result;
6001 }
6002
6003 static errno_t
nstat_client_append_description(nstat_client * client,nstat_src * src)6004 nstat_client_append_description(
6005 nstat_client *client,
6006 nstat_src *src)
6007 {
6008 size_t size = offsetof(nstat_msg_src_description, data) + src->nts_provider->nstat_descriptor_length;
6009 if (size > 512 || src->nts_provider->nstat_descriptor_length == 0 ||
6010 src->nts_provider->nstat_copy_descriptor == NULL) {
6011 return EOPNOTSUPP;
6012 }
6013
6014 // Fill out a buffer on the stack, we will copy to the mbuf later
6015 u_int64_t buffer[size / sizeof(u_int64_t) + 1]; // u_int64_t to ensure alignment
6016 bzero(buffer, size);
6017
6018 nstat_msg_src_description *desc = (nstat_msg_src_description*)buffer;
6019 desc->hdr.type = NSTAT_MSG_TYPE_SRC_DESC;
6020 desc->hdr.length = (u_int16_t)size;
6021 desc->srcref = src->nts_srcref;
6022 desc->event_flags = 0;
6023 desc->provider = src->nts_provider->nstat_provider_id;
6024
6025 errno_t result = 0;
6026 // Fill in the description
6027 // Query the provider for the provider specific bits
6028 uint8_t *desc_data_ptr = nstat_get_data(desc);
6029 result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr,
6030 src->nts_provider->nstat_descriptor_length);
6031 if (result != 0) {
6032 return result;
6033 }
6034
6035 return nstat_accumulate_msg(client, (uint8_t *)buffer, size);
6036 }
6037
6038 static uint64_t
nstat_extension_flags_for_source(nstat_client * client,nstat_src * src)6039 nstat_extension_flags_for_source(
6040 nstat_client *client,
6041 nstat_src *src)
6042 {
6043 VERIFY(client != NULL & src != NULL);
6044 nstat_provider_id_t provider_id = src->nts_provider->nstat_provider_id;
6045
6046 return client->ntc_provider_filters[provider_id].npf_extensions;
6047 }
6048
6049 static int
nstat_client_send_update(nstat_client * client,nstat_src * src,u_int64_t context,u_int64_t event,u_int16_t hdr_flags,int * gone)6050 nstat_client_send_update(
6051 nstat_client *client,
6052 nstat_src *src,
6053 u_int64_t context,
6054 u_int64_t event,
6055 u_int16_t hdr_flags,
6056 int *gone)
6057 {
6058 // Provider doesn't support getting the descriptor or counts? Done.
6059 if ((src->nts_provider->nstat_descriptor_length == 0 ||
6060 src->nts_provider->nstat_copy_descriptor == NULL) &&
6061 src->nts_provider->nstat_counts == NULL) {
6062 return EOPNOTSUPP;
6063 }
6064
6065 // Allocate storage for the descriptor message
6066 mbuf_ref_t msg;
6067 unsigned int one = 1;
6068 size_t size = offsetof(nstat_msg_src_update, data) +
6069 src->nts_provider->nstat_descriptor_length;
6070 size_t total_extension_size = 0;
6071 u_int32_t num_extensions = 0;
6072 u_int64_t extension_mask = nstat_extension_flags_for_source(client, src);
6073
6074 if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) {
6075 uint32_t extension_id = 0;
6076 for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) {
6077 if ((extension_mask & (1ull << extension_id)) != 0) {
6078 size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0);
6079 if (extension_size == 0) {
6080 extension_mask &= ~(1ull << extension_id);
6081 } else {
6082 num_extensions++;
6083 total_extension_size += ROUNDUP64(extension_size);
6084 }
6085 }
6086 }
6087 size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions);
6088 }
6089 assert(size <= MAX_NSTAT_MSG_HDR_LENGTH);
6090
6091 /*
6092 * XXX Would be interesting to see how extended updates affect mbuf
6093 * allocations, given the max segments defined as 1, one may get
6094 * allocations with higher fragmentation.
6095 */
6096 if (mbuf_allocpacket(MBUF_DONTWAIT, size, &one, &msg) != 0) {
6097 return ENOMEM;
6098 }
6099
6100 /* zero out for nstat_msg_src_update */
6101 bzero(m_mtod_current(msg), size);
6102
6103 nstat_msg_src_update *desc = mtod(msg, nstat_msg_src_update *);
6104 desc->hdr.context = context;
6105 desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_UPDATE :
6106 NSTAT_MSG_TYPE_SRC_EXTENDED_UPDATE;
6107 desc->hdr.length = (u_int16_t)size;
6108 desc->hdr.flags = hdr_flags;
6109 desc->srcref = src->nts_srcref;
6110 desc->event_flags = event;
6111 desc->provider = src->nts_provider->nstat_provider_id;
6112
6113 /*
6114 * XXX The following two lines are only valid when max-segments is passed
6115 * as one.
6116 * Other computations with offset also depend on that being true.
6117 * Be aware of that before making any modifications that changes that
6118 * behavior.
6119 */
6120 mbuf_setlen(msg, size);
6121 mbuf_pkthdr_setlen(msg, mbuf_len(msg));
6122
6123 errno_t result = 0;
6124 if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) {
6125 // Query the provider for the provider specific bits
6126 u_int8_t *desc_data_ptr = nstat_get_data(desc);
6127 result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr,
6128 src->nts_provider->nstat_descriptor_length);
6129 if (result != 0) {
6130 mbuf_freem(msg);
6131 return result;
6132 }
6133 }
6134
6135 if (num_extensions > 0) {
6136 nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)mtodo(msg, sizeof(nstat_msg_src_update_hdr) + src->nts_provider->nstat_descriptor_length);
6137 uint32_t extension_id = 0;
6138
6139 bzero(p_extension_hdr, total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions));
6140
6141 for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) {
6142 if ((extension_mask & (1ull << extension_id)) != 0) {
6143 void *buf = (void *)(p_extension_hdr + 1);
6144 size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size);
6145 if ((extension_size == 0) || (extension_size > total_extension_size)) {
6146 // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused
6147 p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN;
6148 p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1));
6149 break;
6150 } else {
6151 // The extension may be of any size alignment, reported as such in the extension header,
6152 // but we pad to ensure that whatever comes next is suitably aligned
6153 p_extension_hdr->type = extension_id;
6154 p_extension_hdr->length = extension_size;
6155 extension_size = ROUNDUP64(extension_size);
6156 total_extension_size -= extension_size;
6157 p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size);
6158 num_extensions--;
6159 }
6160 }
6161 }
6162 }
6163
6164 if (src->nts_provider->nstat_counts) {
6165 result = src->nts_provider->nstat_counts(src->nts_cookie, &desc->counts, gone);
6166 if (result == 0) {
6167 if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES &&
6168 desc->counts.nstat_rxbytes == 0 && desc->counts.nstat_txbytes == 0) {
6169 result = EAGAIN;
6170 } else {
6171 result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR);
6172 }
6173 }
6174 }
6175
6176 if (result != 0) {
6177 nstat_stats.nstat_srcupatefailures += 1;
6178 mbuf_freem(msg);
6179 } else {
6180 src->nts_reported = true;
6181 }
6182
6183 return result;
6184 }
6185
6186 static errno_t
nstat_client_append_update(nstat_client * client,nstat_src * src,int * gone)6187 nstat_client_append_update(
6188 nstat_client *client,
6189 nstat_src *src,
6190 int *gone)
6191 {
6192 if ((src->nts_provider->nstat_descriptor_length == 0 ||
6193 src->nts_provider->nstat_copy_descriptor == NULL) &&
6194 src->nts_provider->nstat_counts == NULL) {
6195 return EOPNOTSUPP;
6196 }
6197
6198 size_t size = offsetof(nstat_msg_src_update, data) + src->nts_provider->nstat_descriptor_length;
6199 size_t total_extension_size = 0;
6200 u_int32_t num_extensions = 0;
6201 u_int64_t extension_mask = nstat_extension_flags_for_source(client, src);
6202
6203 if ((extension_mask != 0) && (src->nts_provider->nstat_copy_extension != NULL)) {
6204 uint32_t extension_id = 0;
6205 for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) {
6206 if ((extension_mask & (1ull << extension_id)) != 0) {
6207 size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, NULL, 0);
6208 if (extension_size == 0) {
6209 extension_mask &= ~(1ull << extension_id);
6210 } else {
6211 num_extensions++;
6212 total_extension_size += ROUNDUP64(extension_size);
6213 }
6214 }
6215 }
6216 size += total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions);
6217 }
6218
6219 /*
6220 * This kind of limits extensions.
6221 * The optimization is around being able to deliver multiple
6222 * of updates bundled together.
6223 * Increasing the size runs the risk of too much stack usage.
6224 * One could potentially changed the allocation below to be on heap.
6225 * For now limiting it to half of NSTAT_MAX_MSG_SIZE.
6226 */
6227 if (size > (NSTAT_MAX_MSG_SIZE >> 1)) {
6228 return EOPNOTSUPP;
6229 }
6230
6231 // Fill out a buffer on the stack, we will copy to the mbuf later
6232 u_int64_t buffer[size / sizeof(u_int64_t) + 1]; // u_int64_t to ensure alignment
6233 bzero(buffer, size);
6234
6235 nstat_msg_src_update *desc = (nstat_msg_src_update*)buffer;
6236 desc->hdr.type = (num_extensions == 0) ? NSTAT_MSG_TYPE_SRC_UPDATE :
6237 NSTAT_MSG_TYPE_SRC_EXTENDED_UPDATE;
6238 desc->hdr.length = (u_int16_t)size;
6239 desc->srcref = src->nts_srcref;
6240 desc->event_flags = 0;
6241 desc->provider = src->nts_provider->nstat_provider_id;
6242
6243 errno_t result = 0;
6244 // Fill in the description
6245 if (src->nts_provider->nstat_descriptor_length != 0 && src->nts_provider->nstat_copy_descriptor) {
6246 // Query the provider for the provider specific bits
6247 u_int8_t *desc_data_ptr = nstat_get_data(desc);
6248 result = src->nts_provider->nstat_copy_descriptor(src->nts_cookie, desc_data_ptr,
6249 src->nts_provider->nstat_descriptor_length);
6250 if (result != 0) {
6251 nstat_stats.nstat_copy_descriptor_failures++;
6252 if (nstat_debug != 0) {
6253 NSTAT_LOG_ERROR("src->nts_provider->nstat_copy_descriptor: %d", result);
6254 }
6255 return result;
6256 }
6257 }
6258
6259 if (num_extensions > 0) {
6260 nstat_msg_src_extended_item_hdr *p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buffer +
6261 sizeof(nstat_msg_src_update_hdr) + src->nts_provider->nstat_descriptor_length);
6262 uint32_t extension_id = 0;
6263 bzero(p_extension_hdr, total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * num_extensions));
6264
6265 for (extension_id = NSTAT_EXTENDED_UPDATE_TYPE_MIN; extension_id <= NSTAT_EXTENDED_UPDATE_TYPE_MAX; extension_id++) {
6266 if ((extension_mask & (1ull << extension_id)) != 0) {
6267 void *buf = (void *)(p_extension_hdr + 1);
6268 size_t extension_size = src->nts_provider->nstat_copy_extension(src->nts_cookie, extension_id, buf, total_extension_size);
6269 if ((extension_size == 0) || (extension_size > total_extension_size)) {
6270 // Something has gone wrong. Instead of attempting to wind back the excess buffer space, mark it as unused
6271 p_extension_hdr->type = NSTAT_EXTENDED_UPDATE_TYPE_UNKNOWN;
6272 p_extension_hdr->length = total_extension_size + (sizeof(nstat_msg_src_extended_item_hdr) * (num_extensions - 1));
6273 break;
6274 } else {
6275 extension_size = ROUNDUP64(extension_size);
6276 p_extension_hdr->type = extension_id;
6277 p_extension_hdr->length = extension_size;
6278 total_extension_size -= extension_size;
6279 p_extension_hdr = (nstat_msg_src_extended_item_hdr *)(void *)((char *)buf + extension_size);
6280 num_extensions--;
6281 }
6282 }
6283 }
6284 }
6285
6286 if (src->nts_provider->nstat_counts) {
6287 result = src->nts_provider->nstat_counts(src->nts_cookie, &desc->counts, gone);
6288 if (result != 0) {
6289 nstat_stats.nstat_provider_counts_failures++;
6290 if (nstat_debug != 0) {
6291 NSTAT_LOG_ERROR("src->nts_provider->nstat_counts: %d", result);
6292 }
6293 return result;
6294 }
6295
6296 if ((src->nts_filter & NSTAT_FILTER_NOZEROBYTES) == NSTAT_FILTER_NOZEROBYTES &&
6297 desc->counts.nstat_rxbytes == 0 && desc->counts.nstat_txbytes == 0) {
6298 return EAGAIN;
6299 }
6300 }
6301
6302 result = nstat_accumulate_msg(client, (uint8_t *)buffer, size);
6303 if (result == 0) {
6304 src->nts_reported = true;
6305 }
6306 return result;
6307 }
6308
6309 static errno_t
nstat_client_send_removed(nstat_client * client,nstat_src * src,u_int16_t hdr_flags)6310 nstat_client_send_removed(
6311 nstat_client *client,
6312 nstat_src *src,
6313 u_int16_t hdr_flags)
6314 {
6315 nstat_msg_src_removed removed;
6316 errno_t result;
6317
6318 bzero(&removed, sizeof(removed));
6319 removed.hdr.type = NSTAT_MSG_TYPE_SRC_REMOVED;
6320 removed.hdr.length = sizeof(removed);
6321 removed.hdr.context = 0;
6322 removed.hdr.flags = hdr_flags;
6323 removed.srcref = src->nts_srcref;
6324 result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &removed,
6325 sizeof(removed), CTL_DATA_EOR | CTL_DATA_CRIT);
6326 if (result != 0) {
6327 nstat_stats.nstat_msgremovedfailures += 1;
6328 }
6329
6330 return result;
6331 }
6332
6333 static errno_t
nstat_client_handle_add_request(nstat_client * client,mbuf_t m)6334 nstat_client_handle_add_request(
6335 nstat_client *client,
6336 mbuf_t m)
6337 {
6338 errno_t result;
6339
6340 // Verify the header fits in the first mbuf
6341 if (mbuf_len(m) < offsetof(nstat_msg_add_src_req, param)) {
6342 return EINVAL;
6343 }
6344
6345 // Calculate the length of the parameter field
6346 ssize_t paramlength = mbuf_pkthdr_len(m) - offsetof(nstat_msg_add_src_req, param);
6347 if (paramlength < 0 || paramlength > 2 * 1024) {
6348 return EINVAL;
6349 }
6350
6351 nstat_provider *__single provider = NULL;
6352 nstat_provider_cookie_t __single cookie = NULL;
6353 nstat_msg_add_src_req *req = mtod(m, nstat_msg_add_src_req *);
6354 if (mbuf_pkthdr_len(m) > mbuf_len(m)) {
6355 // parameter is too large, we need to make a contiguous copy
6356 void *data = (void *) kalloc_data(paramlength, Z_WAITOK);
6357
6358 if (!data) {
6359 return ENOMEM;
6360 }
6361 result = mbuf_copydata(m, offsetof(nstat_msg_add_src_req, param), paramlength, data);
6362 if (result == 0) {
6363 result = nstat_lookup_entry(req->provider, data, paramlength, &provider, &cookie);
6364 }
6365 kfree_data(data, paramlength);
6366 } else {
6367 uint8_t *req_param_ptr = nstat_get_data(req);
6368 result = nstat_lookup_entry(req->provider, req_param_ptr, paramlength, &provider, &cookie);
6369 }
6370
6371 if (result != 0) {
6372 return result;
6373 }
6374
6375 // sanitize cookie
6376 nstat_client_sanitize_cookie(client, provider->nstat_provider_id, cookie);
6377
6378 result = nstat_client_source_add(req->hdr.context, client, provider, cookie, NSTAT_LOCK_NOTHELD);
6379
6380 if (result != 0) {
6381 provider->nstat_release(cookie, 0);
6382 }
6383
6384 // Set the flag if a provider added a single source
6385 os_atomic_or(&client->ntc_added_src, (1 << provider->nstat_provider_id), relaxed);
6386
6387 return result;
6388 }
6389
6390 static errno_t
nstat_set_provider_filter(nstat_client * client,nstat_msg_add_all_srcs * req)6391 nstat_set_provider_filter(
6392 nstat_client *client,
6393 nstat_msg_add_all_srcs *req)
6394 {
6395 nstat_provider_id_t provider_id = req->provider;
6396
6397 u_int32_t prev_ntc_watching = os_atomic_or_orig(&client->ntc_watching, (1 << provider_id), relaxed);
6398
6399 // Reject it if the client is already watching all the sources.
6400 if ((prev_ntc_watching & (1 << provider_id)) != 0) {
6401 return EALREADY;
6402 }
6403
6404 // Reject it if any single source has already been added.
6405 u_int32_t ntc_added_src = os_atomic_load(&client->ntc_added_src, relaxed);
6406 if ((ntc_added_src & (1 << provider_id)) != 0) {
6407 return EALREADY;
6408 }
6409
6410 client->ntc_watching |= (1 << provider_id);
6411 client->ntc_provider_filters[provider_id].npf_events = req->events;
6412 client->ntc_provider_filters[provider_id].npf_flags = req->filter;
6413 client->ntc_provider_filters[provider_id].npf_pid = req->target_pid;
6414 uuid_copy(client->ntc_provider_filters[provider_id].npf_uuid, req->target_uuid);
6415
6416 // The extensions should be populated by a more direct mechanism
6417 // Using the top 32 bits of the filter flags reduces the namespace of both,
6418 // but is a convenient workaround that avoids ntstat.h changes that would require rebuild of all clients
6419 // Extensions give away additional privacy information and are subject to unconditional privilege check,
6420 // unconstrained by the value of nstat_privcheck
6421 if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) == 0) {
6422 client->ntc_provider_filters[provider_id].npf_extensions = (req->filter >> NSTAT_FILTER_ALLOWED_EXTENSIONS_SHIFT) & NSTAT_EXTENDED_UPDATE_FLAG_MASK;
6423 }
6424 return 0;
6425 }
6426
6427 static errno_t
nstat_client_handle_add_all(nstat_client * client,mbuf_t m)6428 nstat_client_handle_add_all(
6429 nstat_client *client,
6430 mbuf_t m)
6431 {
6432 errno_t result = 0;
6433
6434 // Verify the header fits in the first mbuf
6435 if (mbuf_len(m) < sizeof(nstat_msg_add_all_srcs)) {
6436 return EINVAL;
6437 }
6438
6439 nstat_msg_add_all_srcs *req = mtod(m, nstat_msg_add_all_srcs *);
6440 if (req->provider > NSTAT_PROVIDER_LAST) {
6441 return ENOENT;
6442 }
6443
6444 nstat_provider *provider = nstat_find_provider_by_id(req->provider);
6445
6446 if (!provider) {
6447 return ENOENT;
6448 }
6449 if (provider->nstat_watcher_add == NULL) {
6450 return ENOTSUP;
6451 }
6452
6453 // Traditionally the nstat_privcheck value allowed for easy access to ntstat on the Mac.
6454 // Keep backwards compatibility while being more stringent with recent providers
6455 if ((nstat_privcheck != 0) || (req->provider == NSTAT_PROVIDER_UDP_SUBFLOW) || (req->provider == NSTAT_PROVIDER_CONN_USERLAND)) {
6456 result = priv_check_cred(kauth_cred_get(),
6457 PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0);
6458 if (result != 0) {
6459 return result;
6460 }
6461 }
6462
6463 NSTAT_LOCK_EXCLUSIVE();
6464 if (req->filter & NSTAT_FILTER_SUPPRESS_SRC_ADDED) {
6465 // Suppression of source messages implicitly requires the use of update messages
6466 client->ntc_flags |= NSTAT_FLAG_SUPPORTS_UPDATES;
6467 }
6468 NSTAT_UNLOCK_EXCLUSIVE();
6469
6470 // rdar://problem/30301300 Different providers require different synchronization
6471 // to ensure that a new entry does not get double counted due to being added prior
6472 // to all current provider entries being added. Hence pass the provider the details
6473 // in the original request for this to be applied atomically
6474
6475 result = provider->nstat_watcher_add(client, req);
6476
6477 if (result == 0) {
6478 nstat_enqueue_success(req->hdr.context, client, 0);
6479 }
6480
6481 return result;
6482 }
6483
6484 static errno_t
nstat_client_source_add(u_int64_t context,nstat_client * client,nstat_provider * provider,nstat_provider_cookie_t cookie,nstat_lock_status lock_status)6485 nstat_client_source_add(
6486 u_int64_t context,
6487 nstat_client *client,
6488 nstat_provider *provider,
6489 nstat_provider_cookie_t cookie,
6490 nstat_lock_status lock_status)
6491 {
6492 if (lock_status == NSTAT_LOCK_NOTHELD) {
6493 NSTAT_LOCK_EXCLUSIVE();
6494 } else {
6495 NSTAT_ASSERT_LOCKED_EXCLUSIVE();
6496 }
6497
6498 // Fill out source added message if appropriate
6499 errno_t result = 0;
6500 mbuf_ref_t msg = NULL;
6501 nstat_src_ref_t *srcrefp = NULL;
6502
6503 u_int64_t provider_filter_flags = client->ntc_provider_filters[provider->nstat_provider_id].npf_flags;
6504 boolean_t tell_user = ((provider_filter_flags & NSTAT_FILTER_SUPPRESS_SRC_ADDED) == 0);
6505 u_int32_t src_filter = (provider_filter_flags & NSTAT_FILTER_PROVIDER_NOZEROBYTES)? NSTAT_FILTER_NOZEROBYTES : 0;
6506
6507 if (provider_filter_flags & NSTAT_FILTER_TCP_NO_EARLY_CLOSE) {
6508 src_filter |= NSTAT_FILTER_TCP_NO_EARLY_CLOSE;
6509 }
6510
6511 do {
6512 if (tell_user) {
6513 unsigned int one = 1;
6514
6515 if (mbuf_allocpacket(MBUF_DONTWAIT, sizeof(nstat_msg_src_added),
6516 &one, &msg) != 0) {
6517 NSTAT_NOTE_QUAL(nstat_src_add_no_buf, client, 0);
6518 result = ENOMEM;
6519 break;
6520 }
6521
6522 mbuf_setlen(msg, sizeof(nstat_msg_src_added));
6523 mbuf_pkthdr_setlen(msg, mbuf_len(msg));
6524 bzero(m_mtod_current(msg), sizeof(nstat_msg_src_added));
6525
6526 nstat_msg_src_added *add = mtod(msg, nstat_msg_src_added *);
6527 add->hdr.type = NSTAT_MSG_TYPE_SRC_ADDED;
6528 assert(mbuf_len(msg) <= MAX_NSTAT_MSG_HDR_LENGTH);
6529 add->hdr.length = (u_int16_t)mbuf_len(msg);
6530 add->hdr.context = context;
6531 add->provider = provider->nstat_provider_id;
6532 srcrefp = &add->srcref;
6533 }
6534
6535 // Allocate storage for the source
6536 nstat_src *src = kalloc_type(struct nstat_src, Z_WAITOK);
6537 if (src == NULL) {
6538 NSTAT_NOTE_QUAL(nstat_src_add_no_src_mem, client, 0);
6539 if (msg) {
6540 mbuf_freem(msg);
6541 }
6542 result = ENOMEM;
6543 break;
6544 }
6545
6546 // Fill in the source, including picking an unused source ref
6547
6548 src->nts_srcref = nstat_client_next_src_ref(client);
6549 if (srcrefp) {
6550 *srcrefp = src->nts_srcref;
6551 }
6552
6553 if (client->ntc_flags & NSTAT_FLAG_CLEANUP || src->nts_srcref == NSTAT_SRC_REF_INVALID) {
6554 NSTAT_NOTE_QUAL(nstat_src_add_while_cleanup, client, 0);
6555 kfree_type(struct nstat_src, src);
6556 if (msg) {
6557 mbuf_freem(msg);
6558 }
6559 result = EINVAL;
6560 break;
6561 }
6562 src->nts_provider = provider;
6563 src->nts_cookie = cookie;
6564 src->nts_filter = src_filter;
6565 src->nts_seq = 0;
6566
6567 if (msg) {
6568 // send the source added message if appropriate
6569 result = ctl_enqueuembuf(client->ntc_kctl, client->ntc_unit, msg, CTL_DATA_EOR);
6570 if (result != 0) {
6571 NSTAT_NOTE_SRC(nstat_src_add_send_err, client, src);
6572 nstat_stats.nstat_srcaddedfailures += 1;
6573 kfree_type(struct nstat_src, src);
6574 mbuf_freem(msg);
6575 break;
6576 }
6577 }
6578 // Put the source in the list
6579 TAILQ_INSERT_HEAD(&client->ntc_src_queue, src, nts_client_link);
6580 src->nts_client = client;
6581
6582 NSTAT_GLOBAL_COUNT_INCREMENT(nstat_global_src_allocs);
6583 NSTAT_GLOBAL_COUNT_INCREMENT_WITH_MAX(nstat_global_src_current, nstat_global_src_max);
6584
6585 client->ntc_metrics.nstat_src_current++;
6586 if (client->ntc_metrics.nstat_src_current > client->ntc_metrics.nstat_src_max) {
6587 client->ntc_metrics.nstat_src_max = client->ntc_metrics.nstat_src_current;
6588 }
6589
6590 NSTAT_NOTE_SRC(nstat_src_add_success, client, src);
6591 } while (0);
6592
6593 if (lock_status == NSTAT_LOCK_NOTHELD) {
6594 NSTAT_UNLOCK_EXCLUSIVE();
6595 }
6596 return result;
6597 }
6598
6599 static errno_t
nstat_client_handle_remove_request(nstat_client * client,mbuf_t m)6600 nstat_client_handle_remove_request(
6601 nstat_client *client,
6602 mbuf_t m)
6603 {
6604 nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID;
6605 nstat_src *src;
6606
6607 if (mbuf_copydata(m, offsetof(nstat_msg_rem_src_req, srcref), sizeof(srcref), &srcref) != 0) {
6608 return EINVAL;
6609 }
6610
6611 NSTAT_LOCK_EXCLUSIVE();
6612
6613 // Remove this source as we look for it
6614 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
6615 {
6616 if (src->nts_srcref == srcref) {
6617 break;
6618 }
6619 }
6620 if (src) {
6621 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
6622 }
6623
6624 NSTAT_UNLOCK_EXCLUSIVE();
6625
6626 if (src) {
6627 nstat_client_cleanup_source(client, src, FALSE);
6628 NSTAT_NOTE_QUAL(nstat_remove_src_found, client, srcref);
6629 } else {
6630 NSTAT_NOTE_QUAL(nstat_remove_src_missed, client, srcref);
6631 }
6632
6633 return src ? 0 : ENOENT;
6634 }
6635
6636 static errno_t
nstat_client_handle_query_request(nstat_client * client,mbuf_t m)6637 nstat_client_handle_query_request(
6638 nstat_client *client,
6639 mbuf_t m)
6640 {
6641 // TBD: handle this from another thread so we can enqueue a lot of data
6642 // As written, if a client requests query all, this function will be
6643 // called from their send of the request message. We will attempt to write
6644 // responses and succeed until the buffer fills up. Since the clients thread
6645 // is blocked on send, it won't be reading unless the client has two threads
6646 // using this socket, one for read and one for write. Two threads probably
6647 // won't work with this code anyhow since we don't have proper locking in
6648 // place yet.
6649 tailq_head_nstat_src dead_list;
6650 errno_t result = ENOENT;
6651 nstat_msg_query_src_req req;
6652
6653 if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) {
6654 return EINVAL;
6655 }
6656
6657 TAILQ_INIT(&dead_list);
6658 const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL);
6659
6660 if (all_srcs) {
6661 NSTAT_NOTE_QUAL(nstat_query_request_all, client, 0);
6662 } else {
6663 NSTAT_NOTE_QUAL(nstat_query_request_one, client, req.srcref);
6664 }
6665
6666 NSTAT_LOCK_SHARED();
6667
6668 if (all_srcs) {
6669 client->ntc_flags |= NSTAT_FLAG_REQCOUNTS;
6670 }
6671 nstat_src *src, *tmpsrc;
6672 u_int64_t src_count = 0;
6673 boolean_t partial = FALSE;
6674
6675 /*
6676 * Error handling policy and sequence number generation is folded into
6677 * nstat_client_begin_query.
6678 */
6679 partial = nstat_client_begin_query(client, &req.hdr);
6680
6681
6682 TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc)
6683 {
6684 int gone = 0;
6685
6686 // XXX ignore IFACE types?
6687 if (all_srcs || src->nts_srcref == req.srcref) {
6688 if (nstat_client_reporting_allowed(client, src, 0)
6689 && (!partial || !all_srcs || src->nts_seq != client->ntc_seq)) {
6690 if (all_srcs) {
6691 result = nstat_client_append_counts(client, src, &gone);
6692 } else {
6693 result = nstat_client_send_counts(client, src, req.hdr.context, 0, &gone);
6694 }
6695
6696 if (ENOMEM == result || ENOBUFS == result) {
6697 /*
6698 * If the counts message failed to
6699 * enqueue then we should clear our flag so
6700 * that a client doesn't miss anything on
6701 * idle cleanup. We skip the "gone"
6702 * processing in the hope that we may
6703 * catch it another time.
6704 */
6705 NSTAT_NOTE_SRC(nstat_query_request_nobuf, client, src);
6706 client->ntc_flags &= ~NSTAT_FLAG_REQCOUNTS;
6707 break;
6708 }
6709 if (partial) {
6710 /*
6711 * We skip over hard errors and
6712 * filtered sources.
6713 */
6714 src->nts_seq = client->ntc_seq;
6715 src_count++;
6716 }
6717 }
6718 }
6719
6720 if (gone) {
6721 // send one last descriptor message so client may see last state
6722 // If we can't send the notification now, it
6723 // will be sent in the idle cleanup.
6724 if (NSTAT_LOCK_SHARED_TO_EXCLUSIVE()) {
6725 NSTAT_NOTE_SRC(nstat_query_request_upgrade, client, src);
6726 // Successfully upgraded the lock, now we can remove the source from the client
6727 result = nstat_client_send_description(client, src, 0, 0);
6728 if (result != 0) {
6729 nstat_stats.nstat_control_send_description_failures++;
6730 if (nstat_debug != 0) {
6731 NSTAT_LOG_ERROR("nstat_client_send_description() %d", result);
6732 }
6733 client->ntc_flags &= ~NSTAT_FLAG_REQCOUNTS;
6734 NSTAT_NOTE_SRC(nstat_query_request_nodesc, client, src);
6735 NSTAT_LOCK_EXCLUSIVE_TO_SHARED();
6736 break;
6737 }
6738
6739 // pull src out of the list
6740 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
6741 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
6742 NSTAT_LOCK_EXCLUSIVE_TO_SHARED();
6743 } else {
6744 // The upgrade failed and the shared lock has been dropped
6745 // This should be rare. Simply drop out here and have user level retry
6746 // the poll, have the idle cleanup catch the "gone" source
6747 NSTAT_NOTE_SRC(nstat_query_request_noupgrade, client, src);
6748 NSTAT_LOCK_SHARED();
6749 break;
6750 }
6751 }
6752
6753 if (all_srcs) {
6754 if (src_count >= QUERY_CONTINUATION_SRC_COUNT) {
6755 NSTAT_NOTE_SRC(nstat_query_request_limit, client, src);
6756 break;
6757 }
6758 if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) &&
6759 (NSTAT_LOCK_WOULD_YIELD())) {
6760 // A possibly higher priority thread is waiting
6761 // Exit from here and have user level initiate the next fragment
6762 NSTAT_NOTE_SRC(nstat_query_request_yield, client, src);
6763 break;
6764 }
6765 } else if (req.srcref == src->nts_srcref) {
6766 break;
6767 }
6768 }
6769
6770 nstat_flush_accumulated_msgs(client);
6771
6772 u_int16_t flags = 0;
6773 if (req.srcref == NSTAT_SRC_REF_ALL) {
6774 flags = nstat_client_end_query(client, src, partial);
6775 }
6776
6777 NSTAT_UNLOCK_SHARED();
6778
6779 /*
6780 * If an error occurred enqueueing data, then allow the error to
6781 * propagate to nstat_client_send. This way, the error is sent to
6782 * user-level.
6783 */
6784 if (all_srcs && ENOMEM != result && ENOBUFS != result) {
6785 nstat_enqueue_success(req.hdr.context, client, flags);
6786 result = 0;
6787 }
6788
6789 while ((src = TAILQ_FIRST(&dead_list))) {
6790 TAILQ_REMOVE(&dead_list, src, nts_client_link);
6791 nstat_client_cleanup_source(client, src, FALSE);
6792 }
6793
6794 return result;
6795 }
6796
6797 static errno_t
nstat_client_handle_get_src_description(nstat_client * client,mbuf_t m)6798 nstat_client_handle_get_src_description(
6799 nstat_client *client,
6800 mbuf_t m)
6801 {
6802 nstat_msg_get_src_description req;
6803 errno_t result = ENOENT;
6804 nstat_src *src;
6805
6806 if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) {
6807 return EINVAL;
6808 }
6809
6810 NSTAT_LOCK_SHARED();
6811 u_int64_t src_count = 0;
6812 boolean_t partial = FALSE;
6813 const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL);
6814
6815 if (all_srcs) {
6816 NSTAT_NOTE_QUAL(nstat_query_description_all, client, 0);
6817 } else {
6818 NSTAT_NOTE_QUAL(nstat_query_description_one, client, req.srcref);
6819 }
6820
6821 /*
6822 * Error handling policy and sequence number generation is folded into
6823 * nstat_client_begin_query.
6824 */
6825 partial = nstat_client_begin_query(client, &req.hdr);
6826
6827 TAILQ_FOREACH(src, &client->ntc_src_queue, nts_client_link)
6828 {
6829 if (all_srcs || src->nts_srcref == req.srcref) {
6830 if (nstat_client_reporting_allowed(client, src, 0)
6831 && (!all_srcs || !partial || src->nts_seq != client->ntc_seq)) {
6832 if (all_srcs) {
6833 result = nstat_client_append_description(client, src);
6834 } else {
6835 result = nstat_client_send_description(client, src, req.hdr.context, 0);
6836 }
6837
6838 if (ENOMEM == result || ENOBUFS == result) {
6839 /*
6840 * If the description message failed to
6841 * enqueue then we give up for now.
6842 */
6843 break;
6844 }
6845 if (partial) {
6846 /*
6847 * Note, we skip over hard errors and
6848 * filtered sources.
6849 */
6850 src->nts_seq = client->ntc_seq;
6851 src_count++;
6852 if (src_count >= QUERY_CONTINUATION_SRC_COUNT) {
6853 NSTAT_NOTE_SRC(nstat_query_description_limit, client, src);
6854 break;
6855 }
6856 if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) &&
6857 (NSTAT_LOCK_WOULD_YIELD())) {
6858 // A possibly higher priority thread is waiting
6859 // Exit from here and have user level initiate the next fragment
6860 NSTAT_NOTE_SRC(nstat_query_description_yield, client, src);
6861 break;
6862 }
6863 }
6864 }
6865
6866 if (!all_srcs) {
6867 break;
6868 }
6869 }
6870 }
6871 nstat_flush_accumulated_msgs(client);
6872
6873 u_int16_t flags = 0;
6874 if (req.srcref == NSTAT_SRC_REF_ALL) {
6875 flags = nstat_client_end_query(client, src, partial);
6876 }
6877
6878 NSTAT_UNLOCK_SHARED();
6879 /*
6880 * If an error occurred enqueueing data, then allow the error to
6881 * propagate to nstat_client_send. This way, the error is sent to
6882 * user-level.
6883 */
6884 if (all_srcs && ENOMEM != result && ENOBUFS != result) {
6885 nstat_enqueue_success(req.hdr.context, client, flags);
6886 result = 0;
6887 }
6888
6889 return result;
6890 }
6891
6892 static void
nstat_send_error(nstat_client * client,u_int64_t context,u_int32_t error)6893 nstat_send_error(
6894 nstat_client *client,
6895 u_int64_t context,
6896 u_int32_t error)
6897 {
6898 errno_t result;
6899 struct nstat_msg_error err;
6900
6901 bzero(&err, sizeof(err));
6902 err.hdr.type = NSTAT_MSG_TYPE_ERROR;
6903 err.hdr.length = sizeof(err);
6904 err.hdr.context = context;
6905 err.error = error;
6906
6907 result = ctl_enqueuedata(client->ntc_kctl, client->ntc_unit, &err,
6908 sizeof(err), CTL_DATA_EOR | CTL_DATA_CRIT);
6909 if (result != 0) {
6910 nstat_stats.nstat_msgerrorfailures++;
6911 }
6912 }
6913
6914 static boolean_t
nstat_client_begin_query(nstat_client * client,const nstat_msg_hdr * hdrp)6915 nstat_client_begin_query(
6916 nstat_client *client,
6917 const nstat_msg_hdr *hdrp)
6918 {
6919 boolean_t partial = FALSE;
6920
6921 if (hdrp->flags & NSTAT_MSG_HDR_FLAG_CONTINUATION) {
6922 /* A partial query all has been requested. */
6923 partial = TRUE;
6924
6925 if (client->ntc_context != hdrp->context) {
6926 if (client->ntc_context != 0) {
6927 nstat_send_error(client, client->ntc_context, EAGAIN);
6928 }
6929
6930 /* Initialize client for a partial query all. */
6931 client->ntc_context = hdrp->context;
6932 client->ntc_seq++;
6933 }
6934 }
6935
6936 return partial;
6937 }
6938
6939 static u_int16_t
nstat_client_end_query(nstat_client * client,nstat_src * last_src,boolean_t partial)6940 nstat_client_end_query(
6941 nstat_client *client,
6942 nstat_src *last_src,
6943 boolean_t partial)
6944 {
6945 u_int16_t flags = 0;
6946
6947 if (last_src == NULL || !partial) {
6948 /*
6949 * We iterated through the entire srcs list or exited early
6950 * from the loop when a partial update was not requested (an
6951 * error occurred), so clear context to indicate internally
6952 * that the query is finished.
6953 */
6954 client->ntc_context = 0;
6955 } else {
6956 /*
6957 * Indicate to userlevel to make another partial request as
6958 * there are still sources left to be reported.
6959 */
6960 flags |= NSTAT_MSG_HDR_FLAG_CONTINUATION;
6961 }
6962
6963 return flags;
6964 }
6965
6966 static errno_t
nstat_client_handle_get_update(nstat_client * client,mbuf_t m)6967 nstat_client_handle_get_update(
6968 nstat_client *client,
6969 mbuf_t m)
6970 {
6971 nstat_msg_query_src_req req;
6972
6973 if (mbuf_copydata(m, 0, sizeof(req), &req) != 0) {
6974 return EINVAL;
6975 }
6976
6977 NSTAT_LOCK_SHARED();
6978
6979 client->ntc_flags |= NSTAT_FLAG_SUPPORTS_UPDATES;
6980
6981 errno_t result = ENOENT;
6982 nstat_src *src, *tmpsrc;
6983 tailq_head_nstat_src dead_list;
6984 u_int64_t src_count = 0;
6985 boolean_t partial = FALSE;
6986 const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL);
6987 TAILQ_INIT(&dead_list);
6988
6989 if (all_srcs) {
6990 NSTAT_NOTE_QUAL(nstat_query_update_all, client, 0);
6991 } else {
6992 NSTAT_NOTE_QUAL(nstat_query_update_one, client, req.srcref);
6993 }
6994
6995 /*
6996 * Error handling policy and sequence number generation is folded into
6997 * nstat_client_begin_query.
6998 */
6999 partial = nstat_client_begin_query(client, &req.hdr);
7000
7001 TAILQ_FOREACH_SAFE(src, &client->ntc_src_queue, nts_client_link, tmpsrc) {
7002 int gone = 0;
7003 if (all_srcs) {
7004 // Check to see if we should handle this source or if we're still skipping to find where to continue
7005 if ((FALSE == partial || src->nts_seq != client->ntc_seq)) {
7006 u_int64_t suppression_flags = (src->nts_reported)? NSTAT_FILTER_SUPPRESS_BORING_POLL: 0;
7007 if (nstat_client_reporting_allowed(client, src, suppression_flags)) {
7008 result = nstat_client_append_update(client, src, &gone);
7009 if (ENOMEM == result || ENOBUFS == result) {
7010 /*
7011 * If the update message failed to
7012 * enqueue then give up.
7013 */
7014 NSTAT_NOTE_SRC(nstat_query_update_nobuf, client, src);
7015 break;
7016 }
7017 if (partial) {
7018 /*
7019 * We skip over hard errors and
7020 * filtered sources.
7021 */
7022 src->nts_seq = client->ntc_seq;
7023 src_count++;
7024 }
7025 }
7026 }
7027 } else if (src->nts_srcref == req.srcref) {
7028 if (nstat_client_reporting_allowed(client, src, 0)) {
7029 result = nstat_client_send_update(client, src, req.hdr.context, 0, 0, &gone);
7030 }
7031 }
7032
7033 if (gone) {
7034 if (NSTAT_LOCK_SHARED_TO_EXCLUSIVE()) {
7035 // Successfully upgraded the lock, now we can remove the source from the client
7036 // pull src out of the list
7037 TAILQ_REMOVE(&client->ntc_src_queue, src, nts_client_link);
7038 TAILQ_INSERT_TAIL(&dead_list, src, nts_client_link);
7039 NSTAT_LOCK_EXCLUSIVE_TO_SHARED();
7040 NSTAT_NOTE_SRC(nstat_query_update_upgrade, client, src);
7041 } else {
7042 // The upgrade failed and the shared lock has been dropped
7043 // This should be rare. Simply drop out here and have user level retry
7044 // the poll, have the idle cleanup catch the "gone" source
7045 NSTAT_NOTE_SRC(nstat_query_update_noupgrade, client, src);
7046 NSTAT_LOCK_SHARED();
7047 break;
7048 }
7049 }
7050
7051 if (!all_srcs && req.srcref == src->nts_srcref) {
7052 break;
7053 }
7054 if (src_count >= QUERY_CONTINUATION_SRC_COUNT) {
7055 NSTAT_NOTE_SRC(nstat_query_update_limit, client, src);
7056 break;
7057 }
7058 if ((src_count >= QUERY_CONTINUATION_MIN_SRC_COUNT) &&
7059 (NSTAT_LOCK_WOULD_YIELD())) {
7060 // A possibly higher priority thread is waiting
7061 // Exit from here and have user level initiate the next fragment
7062 NSTAT_NOTE_SRC(nstat_query_update_yield, client, src);
7063 break;
7064 }
7065 }
7066
7067 nstat_flush_accumulated_msgs(client);
7068
7069
7070 u_int16_t flags = 0;
7071 if (req.srcref == NSTAT_SRC_REF_ALL) {
7072 flags = nstat_client_end_query(client, src, partial);
7073 }
7074 NSTAT_UNLOCK_SHARED();
7075
7076 /*
7077 * If an error occurred enqueueing data, then allow the error to
7078 * propagate to nstat_client_send. This way, the error is sent to
7079 * user-level.
7080 */
7081 if (all_srcs && ENOMEM != result && ENOBUFS != result) {
7082 nstat_enqueue_success(req.hdr.context, client, flags);
7083 result = 0;
7084 }
7085
7086 while ((src = TAILQ_FIRST(&dead_list))) {
7087 TAILQ_REMOVE(&dead_list, src, nts_client_link);
7088 // release src and send notification
7089 nstat_client_cleanup_source(client, src, FALSE);
7090 }
7091
7092 return result;
7093 }
7094
7095 static errno_t
nstat_client_handle_subscribe_sysinfo(nstat_client * client)7096 nstat_client_handle_subscribe_sysinfo(
7097 nstat_client *client)
7098 {
7099 errno_t result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0);
7100
7101 if (result != 0) {
7102 return result;
7103 }
7104
7105 NSTAT_LOCK_EXCLUSIVE();
7106 client->ntc_flags |= NSTAT_FLAG_SYSINFO_SUBSCRIBED;
7107 NSTAT_UNLOCK_EXCLUSIVE();
7108
7109 return 0;
7110 }
7111
7112 static errno_t
nstat_client_send(kern_ctl_ref kctl,u_int32_t unit,void * uinfo,mbuf_t m,__unused int flags)7113 nstat_client_send(
7114 kern_ctl_ref kctl,
7115 u_int32_t unit,
7116 void *uinfo,
7117 mbuf_t m,
7118 __unused int flags)
7119 {
7120 nstat_client *client = (nstat_client*)uinfo;
7121 struct nstat_msg_hdr *hdr;
7122 struct nstat_msg_hdr storage;
7123 errno_t result = 0;
7124
7125 if (mbuf_pkthdr_len(m) < sizeof(*hdr)) {
7126 // Is this the right thing to do?
7127 mbuf_freem(m);
7128 return EINVAL;
7129 }
7130
7131 if (mbuf_len(m) >= sizeof(*hdr)) {
7132 hdr = mtod(m, struct nstat_msg_hdr *);
7133 } else {
7134 mbuf_copydata(m, 0, sizeof(storage), &storage);
7135 hdr = &storage;
7136 }
7137
7138 // Legacy clients may not set the length
7139 // Those clients are likely not setting the flags either
7140 // Fix everything up so old clients continue to work
7141 if (hdr->length != mbuf_pkthdr_len(m)) {
7142 hdr->flags = 0;
7143 assert(mbuf_pkthdr_len(m) <= MAX_NSTAT_MSG_HDR_LENGTH);
7144 hdr->length = (u_int16_t)mbuf_pkthdr_len(m);
7145 if (hdr == &storage) {
7146 mbuf_copyback(m, 0, sizeof(*hdr), hdr, MBUF_DONTWAIT);
7147 }
7148 }
7149
7150 lck_mtx_lock(&client->ntc_user_mtx); /* Prevent misbehaving clients from overlapping calls */
7151 switch (hdr->type) {
7152 case NSTAT_MSG_TYPE_ADD_SRC:
7153 result = nstat_client_handle_add_request(client, m);
7154 break;
7155
7156 case NSTAT_MSG_TYPE_ADD_ALL_SRCS:
7157 result = nstat_client_handle_add_all(client, m);
7158 break;
7159
7160 case NSTAT_MSG_TYPE_REM_SRC:
7161 result = nstat_client_handle_remove_request(client, m);
7162 break;
7163
7164 case NSTAT_MSG_TYPE_QUERY_SRC:
7165 result = nstat_client_handle_query_request(client, m);
7166 break;
7167
7168 case NSTAT_MSG_TYPE_GET_SRC_DESC:
7169 result = nstat_client_handle_get_src_description(client, m);
7170 break;
7171
7172 case NSTAT_MSG_TYPE_GET_UPDATE:
7173 result = nstat_client_handle_get_update(client, m);
7174 break;
7175
7176 case NSTAT_MSG_TYPE_SUBSCRIBE_SYSINFO:
7177 result = nstat_client_handle_subscribe_sysinfo(client);
7178 break;
7179
7180 default:
7181 result = EINVAL;
7182 break;
7183 }
7184
7185 if (result != 0) {
7186 struct nstat_msg_error err;
7187
7188 bzero(&err, sizeof(err));
7189 err.hdr.type = NSTAT_MSG_TYPE_ERROR;
7190 err.hdr.length = (u_int16_t)(sizeof(err) + mbuf_pkthdr_len(m));
7191 err.hdr.context = hdr->context;
7192 err.error = result;
7193
7194 if (mbuf_prepend(&m, sizeof(err), MBUF_DONTWAIT) == 0 &&
7195 mbuf_copyback(m, 0, sizeof(err), &err, MBUF_DONTWAIT) == 0) {
7196 result = ctl_enqueuembuf(kctl, unit, m, CTL_DATA_EOR | CTL_DATA_CRIT);
7197 if (result != 0) {
7198 mbuf_freem(m);
7199 }
7200 m = NULL;
7201 }
7202
7203 if (result != 0) {
7204 // Unable to prepend the error to the request - just send the error
7205 err.hdr.length = sizeof(err);
7206 result = ctl_enqueuedata(kctl, unit, &err, sizeof(err),
7207 CTL_DATA_EOR | CTL_DATA_CRIT);
7208 if (result != 0) {
7209 nstat_stats.nstat_msgerrorfailures += 1;
7210 }
7211 }
7212 nstat_stats.nstat_handle_msg_failures += 1;
7213 }
7214 lck_mtx_unlock(&client->ntc_user_mtx);
7215
7216 if (m) {
7217 mbuf_freem(m);
7218 }
7219 return result;
7220 }
7221
7222
7223 /* Performs interface matching based on NSTAT_IFNET_IS… filter flags provided by an external caller */
7224 static bool
nstat_interface_matches_filter_flag(uint32_t filter_flags,struct ifnet * ifp)7225 nstat_interface_matches_filter_flag(uint32_t filter_flags, struct ifnet *ifp)
7226 {
7227 bool result = false;
7228
7229 if (ifp) {
7230 uint32_t flag_mask = (NSTAT_FILTER_IFNET_FLAGS & ~(NSTAT_IFNET_IS_NON_LOCAL | NSTAT_IFNET_IS_LOCAL));
7231 filter_flags &= flag_mask;
7232
7233 uint32_t flags = nstat_ifnet_to_flags(ifp);
7234 if (filter_flags & flags) {
7235 result = true;
7236 }
7237 }
7238 return result;
7239 }
7240
7241
7242 static int
progress_indicators_for_interface(unsigned int ifindex,uint64_t recentflow_maxduration,uint32_t filter_flags,uint64_t transport_protocol_mask,struct nstat_progress_indicators * indicators)7243 progress_indicators_for_interface(unsigned int ifindex, uint64_t recentflow_maxduration, uint32_t filter_flags, uint64_t transport_protocol_mask, struct nstat_progress_indicators *indicators)
7244 {
7245 int error = 0;
7246 struct inpcb *inp;
7247 uint64_t min_recent_start_time;
7248 bool update_tcp_indicators = ((transport_protocol_mask & PR_PROTO_TCP) == PR_PROTO_TCP) || (transport_protocol_mask == 0);
7249 bool update_quic_indicators = (transport_protocol_mask & PR_PROTO_QUIC) == PR_PROTO_QUIC;
7250 #if SKYWALK
7251 struct nstat_tu_shadow *shad;
7252 #endif /* SKYWALK */
7253
7254 min_recent_start_time = mach_continuous_time() - recentflow_maxduration;
7255 bzero(indicators, sizeof(*indicators));
7256
7257 if (nstat_debug != 0) {
7258 /* interface index -1 may be passed in to only match against the filters specified in the flags */
7259 if (ifindex < UINT_MAX) {
7260 NSTAT_LOG_DEBUG("for interface index %u with flags %x", ifindex, filter_flags);
7261 } else {
7262 NSTAT_LOG_DEBUG("for matching interface with flags %x", filter_flags);
7263 }
7264 }
7265
7266 if (update_tcp_indicators) {
7267 lck_rw_lock_shared(&tcbinfo.ipi_lock);
7268 /*
7269 * For progress indicators we don't need to special case TCP to collect time wait connections
7270 */
7271 LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list)
7272 {
7273 struct tcpcb *tp = intotcpcb(inp);
7274 /* radar://57100452
7275 * The conditional logic implemented below performs an *inclusive* match based on the desired interface index in addition to any filter values.
7276 * While the general expectation is that only one criteria normally is used for queries, the capability exists satisfy any eccentric future needs.
7277 */
7278 if (tp && inp->inp_state != INPCB_STATE_DEAD && inp->inp_last_outifp &&
7279 /* matches the given interface index, or against any provided filter flags */
7280 (((inp->inp_last_outifp->if_index == ifindex) ||
7281 nstat_interface_matches_filter_flag(filter_flags, inp->inp_last_outifp)) &&
7282 /* perform flow state matching based any provided filter flags */
7283 (((filter_flags & (NSTAT_IFNET_IS_NON_LOCAL | NSTAT_IFNET_IS_LOCAL)) == 0) ||
7284 ((filter_flags & NSTAT_IFNET_IS_NON_LOCAL) && !(tp->t_flags & TF_LOCAL)) ||
7285 ((filter_flags & NSTAT_IFNET_IS_LOCAL) && (tp->t_flags & TF_LOCAL))))) {
7286 struct tcp_conn_status connstatus;
7287
7288 if (nstat_debug != 0) {
7289 NSTAT_LOG_DEBUG("*matched non-Skywalk* [filter match: %d]", nstat_interface_matches_filter_flag(filter_flags, inp->inp_last_outifp));
7290 }
7291
7292 indicators->np_numflows++;
7293 tcp_get_connectivity_status(tp, &connstatus);
7294 if (connstatus.write_probe_failed) {
7295 indicators->np_write_probe_fails++;
7296 }
7297 if (connstatus.read_probe_failed) {
7298 indicators->np_read_probe_fails++;
7299 }
7300 if (connstatus.conn_probe_failed) {
7301 indicators->np_conn_probe_fails++;
7302 }
7303 if (inp->inp_start_timestamp > min_recent_start_time) {
7304 uint64_t flow_count;
7305
7306 indicators->np_recentflows++;
7307 flow_count = os_atomic_load(&inp->inp_stat->rxbytes, relaxed);
7308 indicators->np_recentflows_rxbytes += flow_count;
7309 flow_count = os_atomic_load(&inp->inp_stat->txbytes, relaxed);
7310 indicators->np_recentflows_txbytes += flow_count;
7311
7312 indicators->np_recentflows_rxooo += tp->t_stat.rxoutoforderbytes;
7313 indicators->np_recentflows_rxdup += tp->t_stat.rxduplicatebytes;
7314 indicators->np_recentflows_retx += tp->t_stat.txretransmitbytes;
7315 if (tp->snd_max - tp->snd_una) {
7316 indicators->np_recentflows_unacked++;
7317 }
7318 }
7319 }
7320 }
7321 lck_rw_done(&tcbinfo.ipi_lock);
7322 }
7323
7324 #if SKYWALK
7325 NSTAT_LOCK_SHARED();
7326
7327 TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) {
7328 assert(shad->shad_magic == TU_SHADOW_MAGIC);
7329
7330 bool consider_shad = false;
7331 if (shad->shad_live) {
7332 if (shad->shad_provider == NSTAT_PROVIDER_QUIC_USERLAND) {
7333 consider_shad = update_quic_indicators;
7334 } else if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) {
7335 consider_shad = update_tcp_indicators;
7336 }
7337 }
7338
7339 if (consider_shad) {
7340 u_int32_t ifflags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
7341 if (filter_flags != 0) {
7342 bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, &ifflags, NULL, NULL, NULL);
7343 error = (result)? 0 : EIO;
7344 if (error) {
7345 NSTAT_LOG_ERROR("nstat get ifflags %d", error);
7346 continue;
7347 }
7348
7349 if ((ifflags & filter_flags) == 0) {
7350 continue;
7351 }
7352 // Skywalk locality flags are not yet in place, see <rdar://problem/35607563>
7353 // Instead of checking flags with a simple logical and, check the inverse.
7354 // This allows for default action of fallthrough if the flags are not set.
7355 if ((filter_flags & NSTAT_IFNET_IS_NON_LOCAL) && (ifflags & NSTAT_IFNET_IS_LOCAL)) {
7356 continue;
7357 }
7358 if ((filter_flags & NSTAT_IFNET_IS_LOCAL) && (ifflags & NSTAT_IFNET_IS_NON_LOCAL)) {
7359 continue;
7360 }
7361 }
7362
7363 nstat_progress_digest digest;
7364 bzero(&digest, sizeof(digest));
7365 bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, &digest, NULL, NULL);
7366
7367 error = (result)? 0 : EIO;
7368 if (error) {
7369 NSTAT_LOG_ERROR("nstat get progressdigest returned %d", error);
7370 continue;
7371 }
7372 if ((digest.ifindex == (u_int32_t)ifindex) ||
7373 (filter_flags & ifflags)) {
7374 if (nstat_debug != 0) {
7375 NSTAT_LOG_DEBUG("*matched Skywalk* [filter match: %x %x]", filter_flags, ifflags);
7376 }
7377
7378 indicators->np_numflows++;
7379 if (digest.connstatus.write_probe_failed) {
7380 indicators->np_write_probe_fails++;
7381 }
7382 if (digest.connstatus.read_probe_failed) {
7383 indicators->np_read_probe_fails++;
7384 }
7385 if (digest.connstatus.conn_probe_failed) {
7386 indicators->np_conn_probe_fails++;
7387 }
7388 if (shad->shad_start_timestamp > min_recent_start_time) {
7389 indicators->np_recentflows++;
7390 indicators->np_recentflows_rxbytes += digest.rxbytes;
7391 indicators->np_recentflows_txbytes += digest.txbytes;
7392 indicators->np_recentflows_rxooo += digest.rxoutoforderbytes;
7393 indicators->np_recentflows_rxdup += digest.rxduplicatebytes;
7394 indicators->np_recentflows_retx += digest.txretransmit;
7395 if (digest.txunacked) {
7396 indicators->np_recentflows_unacked++;
7397 }
7398 }
7399 }
7400 }
7401 }
7402
7403 NSTAT_UNLOCK_SHARED();
7404
7405 #endif /* SKYWALK */
7406 return error;
7407 }
7408
7409
7410 static int
tcp_progress_probe_enable_for_interface(unsigned int ifindex,uint32_t filter_flags,uint32_t enable_flags)7411 tcp_progress_probe_enable_for_interface(unsigned int ifindex, uint32_t filter_flags, uint32_t enable_flags)
7412 {
7413 int error = 0;
7414 struct ifnet *ifp;
7415
7416 if (nstat_debug != 0) {
7417 NSTAT_LOG_DEBUG("for interface index %u with flags %d", ifindex, filter_flags);
7418 }
7419
7420 ifnet_head_lock_shared();
7421 TAILQ_FOREACH(ifp, &ifnet_head, if_link)
7422 {
7423 if ((ifp->if_index == ifindex) ||
7424 nstat_interface_matches_filter_flag(filter_flags, ifp)) {
7425 if (nstat_debug != 0) {
7426 NSTAT_LOG_DEBUG("*matched* interface index %d, enable: %d", ifp->if_index, enable_flags);
7427 }
7428 error = if_probe_connectivity(ifp, enable_flags);
7429 if (error) {
7430 NSTAT_LOG_ERROR("(%d) - nstat set tcp probe %d for interface index %d", error, enable_flags, ifp->if_index);
7431 }
7432 }
7433 }
7434 ifnet_head_done();
7435
7436 return error;
7437 }
7438
7439
7440 static int
ntstat_progress_indicators(struct sysctl_req * req)7441 ntstat_progress_indicators(struct sysctl_req *req)
7442 {
7443 struct nstat_progress_indicators indicators = {};
7444 int error = 0;
7445 struct nstat_progress_req requested;
7446
7447 if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) {
7448 return EACCES;
7449 }
7450 if (req->newptr == USER_ADDR_NULL) {
7451 return EINVAL;
7452 }
7453 if (req->newlen < sizeof(requested)) {
7454 return EINVAL;
7455 }
7456 error = SYSCTL_IN(req, &requested, sizeof(requested));
7457 if (error != 0) {
7458 return error;
7459 }
7460 error = progress_indicators_for_interface((unsigned int)requested.np_ifindex, requested.np_recentflow_maxduration, (uint32_t)requested.np_filter_flags, requested.np_transport_protocol_mask, &indicators);
7461 if (error != 0) {
7462 return error;
7463 }
7464 error = SYSCTL_OUT(req, &indicators, sizeof(indicators));
7465 return error;
7466 }
7467
7468
7469 __private_extern__ int
ntstat_tcp_progress_enable(struct sysctl_req * req)7470 ntstat_tcp_progress_enable(struct sysctl_req *req)
7471 {
7472 int error = 0;
7473 struct tcpprobereq requested;
7474
7475 if (priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) {
7476 return EACCES;
7477 }
7478 if (req->newptr == USER_ADDR_NULL) {
7479 return EINVAL;
7480 }
7481 if (req->newlen < sizeof(requested)) {
7482 return EINVAL;
7483 }
7484 error = SYSCTL_IN(req, &requested, sizeof(requested));
7485 if (error != 0) {
7486 return error;
7487 }
7488 error = tcp_progress_probe_enable_for_interface((unsigned int)requested.ifindex, (uint32_t)requested.filter_flags, (uint32_t)requested.enable);
7489
7490 return error;
7491 }
7492
7493
7494 #if SKYWALK
7495
7496 #pragma mark -- netstat support for user level providers --
7497
7498 static int
nstat_gather_flow_data(nstat_provider_id_t provider,nstat_flow_data * __counted_by (n)flow_data_start,int n)7499 nstat_gather_flow_data(nstat_provider_id_t provider, nstat_flow_data *__counted_by(n)flow_data_start, int n)
7500 {
7501 nstat_flow_data * flow_data = flow_data_start;
7502 struct nstat_tu_shadow *shad;
7503 int prepared = 0;
7504 errno_t err;
7505
7506 NSTAT_ASSERT_LOCKED();
7507
7508 TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) {
7509 assert(shad->shad_magic == TU_SHADOW_MAGIC);
7510
7511 if ((shad->shad_provider == provider) && (shad->shad_live)) {
7512 if (prepared >= n) {
7513 break;
7514 }
7515 err = nstat_userland_tu_copy_descriptor((nstat_provider_cookie_t) shad,
7516 &flow_data->flow_descriptor, sizeof(flow_data->flow_descriptor));
7517
7518 if (err != 0) {
7519 NSTAT_LOG_ERROR("nstat_userland_tu_copy_descriptor returned %d", err);
7520 }
7521 err = nstat_userland_tu_counts((nstat_provider_cookie_t) shad,
7522 &flow_data->counts, NULL);
7523 if (err != 0) {
7524 NSTAT_LOG_ERROR("nstat_userland_tu_counts returned %d", err);
7525 }
7526 flow_data++;
7527 prepared++;
7528 }
7529 }
7530 return prepared;
7531 }
7532
7533 static void
nstat_userland_to_xinpcb_n(nstat_provider_id_t provider,nstat_flow_data * flow_data,struct xinpcb_n * xinp)7534 nstat_userland_to_xinpcb_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xinpcb_n *xinp)
7535 {
7536 xinp->xi_len = sizeof(struct xinpcb_n);
7537 xinp->xi_kind = XSO_INPCB;
7538
7539 if (provider == NSTAT_PROVIDER_TCP_USERLAND) {
7540 nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor;
7541 struct sockaddr_in *sa = &desc->local.v4;
7542 if (sa->sin_family == AF_INET) {
7543 xinp->inp_vflag = INP_IPV4;
7544 xinp->inp_laddr = desc->local.v4.sin_addr;
7545 xinp->inp_lport = desc->local.v4.sin_port;
7546 xinp->inp_faddr = desc->remote.v4.sin_addr;
7547 xinp->inp_fport = desc->remote.v4.sin_port;
7548 } else if (sa->sin_family == AF_INET6) {
7549 xinp->inp_vflag = INP_IPV6;
7550 xinp->in6p_laddr = desc->local.v6.sin6_addr;
7551 xinp->in6p_lport = desc->local.v6.sin6_port;
7552 xinp->in6p_faddr = desc->remote.v6.sin6_addr;
7553 xinp->in6p_fport = desc->remote.v6.sin6_port;
7554 }
7555 } else if (provider == NSTAT_PROVIDER_UDP_USERLAND) {
7556 nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor;
7557 struct sockaddr_in *sa = &desc->local.v4;
7558 if (sa->sin_family == AF_INET) {
7559 xinp->inp_vflag = INP_IPV4;
7560 xinp->inp_laddr = desc->local.v4.sin_addr;
7561 xinp->inp_lport = desc->local.v4.sin_port;
7562 xinp->inp_faddr = desc->remote.v4.sin_addr;
7563 xinp->inp_fport = desc->remote.v4.sin_port;
7564 } else if (sa->sin_family == AF_INET6) {
7565 xinp->inp_vflag = INP_IPV6;
7566 xinp->in6p_laddr = desc->local.v6.sin6_addr;
7567 xinp->in6p_lport = desc->local.v6.sin6_port;
7568 xinp->in6p_faddr = desc->remote.v6.sin6_addr;
7569 xinp->in6p_fport = desc->remote.v6.sin6_port;
7570 }
7571 }
7572 }
7573
7574 static void
nstat_userland_to_xsocket_n(nstat_provider_id_t provider,nstat_flow_data * flow_data,struct xsocket_n * xso)7575 nstat_userland_to_xsocket_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsocket_n *xso)
7576 {
7577 xso->xso_len = sizeof(struct xsocket_n);
7578 xso->xso_kind = XSO_SOCKET;
7579
7580 if (provider == NSTAT_PROVIDER_TCP_USERLAND) {
7581 nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor;
7582 xso->xso_protocol = IPPROTO_TCP;
7583 xso->so_e_pid = desc->epid;
7584 xso->so_last_pid = desc->pid;
7585 } else {
7586 nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor;
7587 xso->xso_protocol = IPPROTO_UDP;
7588 xso->so_e_pid = desc->epid;
7589 xso->so_last_pid = desc->pid;
7590 }
7591 }
7592
7593 static void
nstat_userland_to_rcv_xsockbuf_n(nstat_provider_id_t provider,nstat_flow_data * flow_data,struct xsockbuf_n * xsbrcv)7594 nstat_userland_to_rcv_xsockbuf_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsockbuf_n *xsbrcv)
7595 {
7596 xsbrcv->xsb_len = sizeof(struct xsockbuf_n);
7597 xsbrcv->xsb_kind = XSO_RCVBUF;
7598
7599 if (provider == NSTAT_PROVIDER_TCP_USERLAND) {
7600 nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor;
7601 xsbrcv->sb_hiwat = desc->rcvbufsize;
7602 xsbrcv->sb_cc = desc->rcvbufused;
7603 } else {
7604 nstat_udp_descriptor *desc = &flow_data->flow_descriptor.udp_descriptor;
7605 xsbrcv->sb_hiwat = desc->rcvbufsize;
7606 xsbrcv->sb_cc = desc->rcvbufused;
7607 }
7608 }
7609
7610 static void
nstat_userland_to_snd_xsockbuf_n(nstat_provider_id_t provider,nstat_flow_data * flow_data,struct xsockbuf_n * xsbsnd)7611 nstat_userland_to_snd_xsockbuf_n(nstat_provider_id_t provider, nstat_flow_data *flow_data, struct xsockbuf_n *xsbsnd)
7612 {
7613 xsbsnd->xsb_len = sizeof(struct xsockbuf_n);
7614 xsbsnd->xsb_kind = XSO_SNDBUF;
7615
7616 if (provider == NSTAT_PROVIDER_TCP_USERLAND) {
7617 nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor;
7618 xsbsnd->sb_hiwat = desc->sndbufsize;
7619 xsbsnd->sb_cc = desc->sndbufused;
7620 } else {
7621 }
7622 }
7623
7624 static void
nstat_userland_to_xsockstat_n(nstat_flow_data * flow_data,struct xsockstat_n * xst)7625 nstat_userland_to_xsockstat_n(nstat_flow_data *flow_data, struct xsockstat_n *xst)
7626 {
7627 xst->xst_len = sizeof(struct xsockstat_n);
7628 xst->xst_kind = XSO_STATS;
7629
7630 // The kernel version supports an array of counts, here we only support one and map to first entry
7631 xst->xst_tc_stats[0].rxpackets = flow_data->counts.nstat_rxpackets;
7632 xst->xst_tc_stats[0].rxbytes = flow_data->counts.nstat_rxbytes;
7633 xst->xst_tc_stats[0].txpackets = flow_data->counts.nstat_txpackets;
7634 xst->xst_tc_stats[0].txbytes = flow_data->counts.nstat_txbytes;
7635 }
7636
7637 static void
nstat_userland_to_xtcpcb_n(nstat_flow_data * flow_data,struct xtcpcb_n * xt)7638 nstat_userland_to_xtcpcb_n(nstat_flow_data *flow_data, struct xtcpcb_n *xt)
7639 {
7640 nstat_tcp_descriptor *desc = &flow_data->flow_descriptor.tcp_descriptor;
7641 xt->xt_len = sizeof(struct xtcpcb_n);
7642 xt->xt_kind = XSO_TCPCB;
7643 xt->t_state = desc->state;
7644 xt->snd_wnd = desc->txwindow;
7645 xt->snd_cwnd = desc->txcwindow;
7646 }
7647
7648
7649 __private_extern__ int
ntstat_userland_count(short proto)7650 ntstat_userland_count(short proto)
7651 {
7652 int n = 0;
7653 if (proto == IPPROTO_TCP) {
7654 n = nstat_userland_tcp_shadows;
7655 } else if (proto == IPPROTO_UDP) {
7656 n = nstat_userland_udp_shadows;
7657 }
7658 return n;
7659 }
7660
7661 __private_extern__ int
nstat_userland_get_snapshot(short proto,void * __sized_by (* snapshot_size)* snapshotp,size_t * snapshot_size,int * countp)7662 nstat_userland_get_snapshot(short proto, void *__sized_by(*snapshot_size) * snapshotp, size_t *snapshot_size, int *countp)
7663 {
7664 int error = 0;
7665 int n = 0;
7666 size_t data_size = 0;
7667 nstat_provider_id_t provider;
7668 nstat_flow_data *flow_data = NULL;
7669
7670 NSTAT_LOCK_SHARED();
7671 if (proto == IPPROTO_TCP) {
7672 n = nstat_userland_tcp_shadows;
7673 provider = NSTAT_PROVIDER_TCP_USERLAND;
7674 } else if (proto == IPPROTO_UDP) {
7675 n = nstat_userland_udp_shadows;
7676 provider = NSTAT_PROVIDER_UDP_USERLAND;
7677 }
7678 if (n == 0) {
7679 goto done;
7680 }
7681
7682 data_size = n * sizeof(*flow_data);
7683 flow_data = (nstat_flow_data *) kalloc_data(data_size,
7684 Z_WAITOK | Z_ZERO);
7685 if (flow_data) {
7686 n = nstat_gather_flow_data(provider, flow_data, n);
7687 } else {
7688 error = ENOMEM;
7689 }
7690 done:
7691 NSTAT_UNLOCK_SHARED();
7692 *snapshotp = flow_data;
7693 *snapshot_size = data_size;
7694 *countp = n;
7695 return error;
7696 }
7697
7698 // nstat_userland_list_snapshot() does most of the work for a sysctl that uses a return format
7699 // as per get_pcblist_n() even though the vast majority of fields are unused.
7700 // Additional items are required in the sysctl output before and after the data added
7701 // by this function.
7702 __private_extern__ int
nstat_userland_list_snapshot(short proto,struct sysctl_req * req,void * __sized_by (n * sizeof (nstat_flow_data))userlandsnapshot,int n)7703 nstat_userland_list_snapshot(short proto, struct sysctl_req *req, void *__sized_by(n * sizeof(nstat_flow_data))userlandsnapshot, int n)
7704 {
7705 int error = 0;
7706 int i;
7707 nstat_provider_id_t provider;
7708 void *buf = NULL;
7709 nstat_flow_data *flow_data, *flow_data_array = NULL;
7710 size_t item_size = ROUNDUP64(sizeof(struct xinpcb_n)) +
7711 ROUNDUP64(sizeof(struct xsocket_n)) +
7712 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
7713 ROUNDUP64(sizeof(struct xsockstat_n));
7714
7715 if ((n == 0) || (userlandsnapshot == NULL)) {
7716 goto done;
7717 }
7718
7719 if (proto == IPPROTO_TCP) {
7720 item_size += ROUNDUP64(sizeof(struct xtcpcb_n));
7721 provider = NSTAT_PROVIDER_TCP_USERLAND;
7722 } else if (proto == IPPROTO_UDP) {
7723 provider = NSTAT_PROVIDER_UDP_USERLAND;
7724 } else {
7725 error = EINVAL;
7726 goto done;
7727 }
7728
7729 buf = (void *) kalloc_data(item_size, Z_WAITOK);
7730 if (buf) {
7731 struct xinpcb_n *xi = (struct xinpcb_n *)buf;
7732 struct xsocket_n *xso = (struct xsocket_n *) ADVANCE64(xi, sizeof(*xi));
7733 struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *) ADVANCE64(xso, sizeof(*xso));
7734 struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *) ADVANCE64(xsbrcv, sizeof(*xsbrcv));
7735 struct xsockstat_n *xsostats = (struct xsockstat_n *) ADVANCE64(xsbsnd, sizeof(*xsbsnd));
7736 struct xtcpcb_n *xt = (struct xtcpcb_n *) ADVANCE64(xsostats, sizeof(*xsostats));
7737
7738 flow_data_array = (nstat_flow_data *)userlandsnapshot;
7739
7740 for (i = 0; i < n; i++) {
7741 flow_data = &flow_data_array[i];
7742 bzero(buf, item_size);
7743
7744 nstat_userland_to_xinpcb_n(provider, flow_data, xi);
7745 nstat_userland_to_xsocket_n(provider, flow_data, xso);
7746 nstat_userland_to_rcv_xsockbuf_n(provider, flow_data, xsbrcv);
7747 nstat_userland_to_snd_xsockbuf_n(provider, flow_data, xsbsnd);
7748 nstat_userland_to_xsockstat_n(flow_data, xsostats);
7749 if (proto == IPPROTO_TCP) {
7750 nstat_userland_to_xtcpcb_n(flow_data, xt);
7751 }
7752 error = SYSCTL_OUT(req, buf, item_size);
7753 if (error) {
7754 break;
7755 }
7756 }
7757 kfree_data(buf, item_size);
7758 } else {
7759 error = ENOMEM;
7760 }
7761 done:
7762 return error;
7763 }
7764
7765 __private_extern__ void
nstat_userland_release_snapshot(void * snapshot,int nuserland)7766 nstat_userland_release_snapshot(void *snapshot, int nuserland)
7767 {
7768 if (snapshot != NULL) {
7769 kfree_data(snapshot, nuserland * sizeof(nstat_flow_data));
7770 }
7771 }
7772
7773 #if NTSTAT_SUPPORTS_STANDALONE_SYSCTL
7774
7775 __private_extern__ int
ntstat_userland_list_n(short proto,struct sysctl_req * req)7776 ntstat_userland_list_n(short proto, struct sysctl_req *req)
7777 {
7778 int error = 0;
7779 int n;
7780 struct xinpgen xig;
7781 void *snapshot = NULL;
7782 size_t item_size = ROUNDUP64(sizeof(struct xinpcb_n)) +
7783 ROUNDUP64(sizeof(struct xsocket_n)) +
7784 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
7785 ROUNDUP64(sizeof(struct xsockstat_n));
7786
7787 if (proto == IPPROTO_TCP) {
7788 item_size += ROUNDUP64(sizeof(struct xtcpcb_n));
7789 }
7790
7791 if (req->oldptr == USER_ADDR_NULL) {
7792 n = ntstat_userland_count(proto);
7793 req->oldidx = 2 * (sizeof(xig)) + (n + 1 + n / 8) * item_size;
7794 goto done;
7795 }
7796
7797 if (req->newptr != USER_ADDR_NULL) {
7798 error = EPERM;
7799 goto done;
7800 }
7801
7802 error = nstat_userland_get_snapshot(proto, &snapshot, &n);
7803
7804 if (error) {
7805 goto done;
7806 }
7807
7808 bzero(&xig, sizeof(xig));
7809 xig.xig_len = sizeof(xig);
7810 xig.xig_gen = 0;
7811 xig.xig_sogen = 0;
7812 xig.xig_count = n;
7813 error = SYSCTL_OUT(req, &xig, sizeof(xig));
7814 if (error) {
7815 goto done;
7816 }
7817 /*
7818 * We are done if there are no flows
7819 */
7820 if (n == 0) {
7821 goto done;
7822 }
7823
7824 error = nstat_userland_list_snapshot(proto, req, snapshot, n);
7825
7826 if (!error) {
7827 /*
7828 * Give the user an updated idea of our state,
7829 * which is unchanged
7830 */
7831 error = SYSCTL_OUT(req, &xig, sizeof(xig));
7832 }
7833 done:
7834 nstat_userland_release_snapshot(snapshot, n);
7835 return error;
7836 }
7837
7838 #endif /* NTSTAT_SUPPORTS_STANDALONE_SYSCTL */
7839 #endif /* SKYWALK */
7840