1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 /*
25 * LOCKING STRATEGY
26 *
27 * The struct socket's so_flow_db field (struct soflow_db and its hash entries
28 * struct soflow_hash_entry) is protected by the socket lock. This covers all the
29 * socket paths that calls soflow_get_flow() as well as the garbage collection.
30 * For the socket detach path, soflow_detach() cannot assume the socket lock is
31 * held. Thus, reference counts are added to both struct soflow_db and struct
32 * soflow_hash_entry to avoid access after freed issues.
33 *
34 * The global list, soflow_entry_head, keeps track of all struct soflow_hash_entry
35 * entries which is used by garbage collection when detecting idle entries. This list
36 * is protected by the global lock soflow_lck_rw.
37 *
38 */
39
40 #include <sys/types.h>
41 #include <sys/kern_control.h>
42 #include <sys/queue.h>
43 #include <sys/domain.h>
44 #include <sys/protosw.h>
45 #include <sys/syslog.h>
46 #include <sys/systm.h>
47 #include <sys/sysproto.h>
48 #include <sys/socket.h>
49 #include <sys/socketvar.h>
50
51 #include <kern/sched_prim.h>
52 #include <kern/locks.h>
53 #include <kern/zalloc.h>
54 #include <kern/debug.h>
55 #include <net/ntstat.h>
56 #include <netinet6/in6_var.h>
57
58 #define _IP_VHL
59 #include <netinet/ip.h>
60 #include <netinet/in_pcb.h>
61 #include <netinet/udp.h>
62 #include <netinet/udp_var.h>
63
64 #include <string.h>
65 #include <libkern/libkern.h>
66 #include <kern/socket_flows.h>
67
68 extern struct inpcbinfo ripcbinfo;
69
70 /*
71 * Per-Socket Flow Management
72 */
73
74 static int soflow_log_level = LOG_ERR;
75 static int soflow_log_port = 0;
76 static int soflow_log_pid = 0;
77 static int soflow_log_proto = 0;
78 static int soflow_nstat_disable = 0;
79 static int soflow_disable = 0;
80 static long soflow_attached_count = 0;
81 static long soflow_attached_high_water_mark = 0;
82 static os_log_t soflow_log_handle = NULL;
83
84 /*
85 * Sysctls for debug logs control
86 */
87 SYSCTL_NODE(_net, OID_AUTO, soflow, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "soflow");
88
89 SYSCTL_INT(_net_soflow, OID_AUTO, log_level, CTLFLAG_RW | CTLFLAG_LOCKED,
90 &soflow_log_level, 0, "");
91
92 SYSCTL_INT(_net_soflow, OID_AUTO, log_port, CTLFLAG_RW | CTLFLAG_LOCKED,
93 &soflow_log_port, 0, "");
94
95 SYSCTL_INT(_net_soflow, OID_AUTO, log_pid, CTLFLAG_RW | CTLFLAG_LOCKED,
96 &soflow_log_pid, 0, "");
97
98 SYSCTL_INT(_net_soflow, OID_AUTO, log_proto, CTLFLAG_RW | CTLFLAG_LOCKED,
99 &soflow_log_proto, 0, "");
100
101 SYSCTL_INT(_net_soflow, OID_AUTO, nstat_disable, CTLFLAG_RW | CTLFLAG_LOCKED,
102 &soflow_nstat_disable, 0, "");
103
104 SYSCTL_INT(_net_soflow, OID_AUTO, disable, CTLFLAG_RW | CTLFLAG_LOCKED,
105 &soflow_disable, 0, "");
106
107 SYSCTL_LONG(_net_soflow, OID_AUTO, count, CTLFLAG_LOCKED | CTLFLAG_RD, &soflow_attached_count, "");
108 SYSCTL_LONG(_net_soflow, OID_AUTO, high_water_mark, CTLFLAG_LOCKED | CTLFLAG_RD, &soflow_attached_high_water_mark, "");
109
110 #define SOFLOW_LOG(level, so, debug, fmt, ...) \
111 do { \
112 if (soflow_log_level >= level && debug && soflow_log_handle) { \
113 if (level == LOG_ERR) { \
114 os_log_error(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
115 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
116 } else { \
117 os_log(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
118 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
119 } \
120 } \
121 } while (0)
122
123 #define SOFLOW_ENTRY_LOG(level, so, entry, debug, msg) \
124 do { \
125 if (soflow_log_level >= level && entry && debug) { \
126 soflow_entry_log(level, so, entry, msg); \
127 } \
128 } while (0)
129
130 #define SOFLOW_HASH_SIZE 16
131 #define SOFLOW_HASH(laddr, faddr, lport, fport) ((faddr) ^ ((laddr) >> 16) ^ (fport) ^ (lport))
132
133 #define SOFLOW_IS_UDP(so) (so && so->so_proto && so->so_proto->pr_type == SOCK_DGRAM && so->so_proto->pr_protocol == IPPROTO_UDP)
134 #define SOFLOW_GET_SO_PROTO(so) ((so && so->so_proto) ? so->so_proto->pr_protocol : IPPROTO_MAX)
135
136 #define SOFLOW_SOCKET_PID(so) ((so->so_flags & SOF_DELEGATED) ? so->e_pid : so->last_pid)
137
138 #define SOFLOW_ENABLE_DEBUG(so, entry) \
139 ((soflow_log_port == 0 || !entry || soflow_log_port == ntohs(entry->soflow_lport) || soflow_log_port == ntohs(entry->soflow_fport)) && \
140 (soflow_log_pid == 0 || !so || soflow_log_pid == SOFLOW_SOCKET_PID(so)) && \
141 (soflow_log_proto == 0 || !so || soflow_log_proto == SOFLOW_GET_SO_PROTO(so)))
142
143 os_refgrp_decl(static, soflow_refgrp, "soflow_ref_group", NULL);
144
145 #define SOFLOW_ENTRY_FREE(entry) \
146 if (entry && (os_ref_release(&entry->soflow_ref_count) == 0)) { \
147 soflow_entry_free(entry); \
148 }
149
150 #define SOFLOW_DB_FREE(db) \
151 if (db && (os_ref_release(&db->soflow_db_ref_count) == 0)) { \
152 soflow_db_free(db); \
153 }
154
155 LIST_HEAD(soflow_hash_head, soflow_hash_entry);
156
157 static int soflow_initialized = 0;
158
159 TAILQ_HEAD(soflow_entry_head, soflow_hash_entry) soflow_entry_head;
160 static LCK_GRP_DECLARE(soflow_lck_grp, "Socket Flow");
161 static LCK_RW_DECLARE(soflow_lck_rw, &soflow_lck_grp);
162
163 #define SOFLOW_LOCK_EXCLUSIVE lck_rw_lock_exclusive(&soflow_lck_rw)
164 #define SOFLOW_UNLOCK_EXCLUSIVE lck_rw_unlock_exclusive(&soflow_lck_rw)
165 #define SOFLOW_LOCK_SHARED lck_rw_lock_shared(&soflow_lck_rw)
166 #define SOFLOW_UNLOCK_SHARED lck_rw_unlock_shared(&soflow_lck_rw)
167
168 /*
169 * Flow Garbage Collection:
170 */
171 static struct thread *soflow_gc_thread;
172 static soflow_feat_gc_needed_func soflow_feat_gc_needed_func_ptr = NULL;
173 static soflow_feat_gc_perform_func soflow_feat_gc_perform_func_ptr = NULL;
174
175 #define SOFLOW_GC_IDLE_TO 30 // Flow Idle Timeout in seconds
176 #define SOFLOW_GC_MAX_COUNT 100 // Max sockets to be handled per run
177 #define SOFLOW_GC_RUN_INTERVAL_NSEC (10 * NSEC_PER_SEC) // GC wakes up every 10 seconds
178
179 /*
180 * Feature Context Handling:
181 */
182 static soflow_feat_detach_entry_func soflow_feat_detach_entry_func_ptr = NULL;
183 static soflow_feat_detach_db_func soflow_feat_detach_db_func_ptr = NULL;
184
185 static void soflow_gc_thread_func(void *v, wait_result_t w);
186 static void soflow_gc_expire(void *v, wait_result_t w);
187 static boolean_t soflow_entry_local_address_needs_update(struct soflow_hash_entry *);
188 static boolean_t soflow_entry_local_port_needs_update(struct socket *, struct soflow_hash_entry *);
189
190 static void
soflow_init(void)191 soflow_init(void)
192 {
193 if (soflow_initialized) {
194 return;
195 }
196 soflow_initialized = 1;
197
198 if (soflow_log_handle == NULL) {
199 soflow_log_handle = os_log_create("com.apple.xnu.net.soflow", "soflow");
200 }
201
202 TAILQ_INIT(&soflow_entry_head);
203
204 // Spawn thread for gargage collection
205 if (kernel_thread_start(soflow_gc_thread_func, NULL,
206 &soflow_gc_thread) != KERN_SUCCESS) {
207 panic_plain("%s: Can't create SOFLOW GC thread", __func__);
208 /* NOTREACHED */
209 }
210 /* this must not fail */
211 VERIFY(soflow_gc_thread != NULL);
212 }
213
214 static void
soflow_entry_log(int level,struct socket * so,struct soflow_hash_entry * entry,const char * msg)215 soflow_entry_log(int level, struct socket *so, struct soflow_hash_entry *entry, const char* msg)
216 {
217 #pragma unused(level, msg)
218 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
219 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
220 const void *addr;
221
222 // No sock or not UDP, no-op
223 if (entry == NULL) {
224 return;
225 }
226
227 switch (entry->soflow_family) {
228 case AF_INET6:
229 addr = &entry->soflow_laddr.addr6;
230 inet_ntop(AF_INET6, addr, local, sizeof(local));
231 addr = &entry->soflow_faddr.addr6;
232 inet_ntop(AF_INET6, addr, remote, sizeof(local));
233 break;
234 case AF_INET:
235 addr = &entry->soflow_laddr.addr46.ia46_addr4.s_addr;
236 inet_ntop(AF_INET, addr, local, sizeof(local));
237 addr = &entry->soflow_faddr.addr46.ia46_addr4.s_addr;
238 inet_ntop(AF_INET, addr, remote, sizeof(local));
239 break;
240 default:
241 return;
242 }
243
244 SOFLOW_LOG(level, so, entry->soflow_debug, "<%s>: %s <%s(%d) entry %p, featureID %llu> outifp %d lport %d fport %d laddr %s faddr %s hash %X "
245 "<rx p %llu b %llu, tx p %llu b %llu>",
246 msg, entry->soflow_outgoing ? "OUT" : "IN ",
247 SOFLOW_IS_UDP(so) ? "UDP" : "proto", SOFLOW_GET_SO_PROTO(so),
248 entry, entry->soflow_feat_ctxt_id,
249 entry->soflow_outifindex,
250 ntohs(entry->soflow_lport), ntohs(entry->soflow_fport), local, remote,
251 entry->soflow_flowhash,
252 entry->soflow_rxpackets, entry->soflow_rxbytes, entry->soflow_txpackets, entry->soflow_txbytes);
253 }
254
255 bool
soflow_fill_hash_entry_from_address(struct soflow_hash_entry * entry,bool isLocal,struct sockaddr * addr,bool islocalUpdate)256 soflow_fill_hash_entry_from_address(struct soflow_hash_entry *entry, bool isLocal, struct sockaddr *addr, bool islocalUpdate)
257 {
258 struct sockaddr_in *sin = NULL;
259 struct sockaddr_in6 *sin6 = NULL;
260
261 if (entry == NULL || addr == NULL) {
262 return FALSE;
263 }
264
265 switch (addr->sa_family) {
266 case AF_INET:
267 sin = satosin(addr);
268 if (sin->sin_len < sizeof(*sin)) {
269 return FALSE;
270 }
271 if (isLocal == TRUE) {
272 if (sin->sin_port != 0) {
273 entry->soflow_lport = sin->sin_port;
274 if (islocalUpdate) {
275 entry->soflow_lport_updated = TRUE;
276 }
277 }
278 if (sin->sin_addr.s_addr != INADDR_ANY) {
279 entry->soflow_laddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
280 if (islocalUpdate) {
281 entry->soflow_laddr_updated = TRUE;
282 }
283 }
284 } else {
285 if (sin->sin_port != 0) {
286 entry->soflow_fport = sin->sin_port;
287 }
288 if (sin->sin_addr.s_addr != INADDR_ANY) {
289 entry->soflow_faddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
290 }
291 }
292 entry->soflow_family = AF_INET;
293 return TRUE;
294 case AF_INET6:
295 sin6 = satosin6(addr);
296 if (sin6->sin6_len < sizeof(*sin6)) {
297 return FALSE;
298 }
299 if (isLocal == TRUE) {
300 if (sin6->sin6_port != 0) {
301 entry->soflow_lport = sin6->sin6_port;
302 if (islocalUpdate) {
303 entry->soflow_lport_updated = TRUE;
304 }
305 }
306 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
307 entry->soflow_laddr.addr6 = sin6->sin6_addr;
308 entry->soflow_laddr6_ifscope = sin6->sin6_scope_id;
309 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
310 if (islocalUpdate) {
311 entry->soflow_laddr_updated = TRUE;
312 }
313 }
314 } else {
315 if (sin6->sin6_port != 0) {
316 entry->soflow_fport = sin6->sin6_port;
317 }
318 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
319 entry->soflow_faddr.addr6 = sin6->sin6_addr;
320 entry->soflow_faddr6_ifscope = sin6->sin6_scope_id;
321 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
322 }
323 }
324 entry->soflow_family = AF_INET6;
325 return TRUE;
326 default:
327 return FALSE;
328 }
329 }
330
331 bool
soflow_fill_hash_entry_from_inp(struct soflow_hash_entry * entry,bool isLocal,struct inpcb * inp,bool islocalUpdate)332 soflow_fill_hash_entry_from_inp(struct soflow_hash_entry *entry, bool isLocal, struct inpcb *inp, bool islocalUpdate)
333 {
334 if (entry == NULL || inp == NULL) {
335 return FALSE;
336 }
337
338 if (inp->inp_vflag & INP_IPV6) {
339 if (isLocal == TRUE) {
340 if (inp->inp_lport) {
341 entry->soflow_lport = inp->inp_lport;
342 if (islocalUpdate) {
343 entry->soflow_lport_updated = TRUE;
344 }
345 }
346 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
347 entry->soflow_laddr.addr6 = inp->in6p_laddr;
348 entry->soflow_laddr6_ifscope = inp->inp_lifscope;
349 in6_verify_ifscope(&entry->soflow_laddr.addr6, inp->inp_lifscope);
350 if (islocalUpdate) {
351 entry->soflow_laddr_updated = TRUE;
352 }
353 }
354 } else {
355 if (inp->inp_fport) {
356 entry->soflow_fport = inp->inp_fport;
357 }
358 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
359 entry->soflow_faddr.addr6 = inp->in6p_faddr;
360 entry->soflow_faddr6_ifscope = inp->inp_fifscope;
361 in6_verify_ifscope(&entry->soflow_faddr.addr6, inp->inp_fifscope);
362 }
363 }
364 entry->soflow_family = AF_INET6;
365 return TRUE;
366 } else if (inp->inp_vflag & INP_IPV4) {
367 if (isLocal == TRUE) {
368 if (inp->inp_lport) {
369 entry->soflow_lport = inp->inp_lport;
370 if (islocalUpdate) {
371 entry->soflow_lport_updated = TRUE;
372 }
373 }
374 if (inp->inp_laddr.s_addr) {
375 entry->soflow_laddr.addr46.ia46_addr4.s_addr = inp->inp_laddr.s_addr;
376 if (islocalUpdate) {
377 entry->soflow_laddr_updated = TRUE;
378 }
379 }
380 } else {
381 if (inp->inp_fport) {
382 entry->soflow_fport = inp->inp_fport;
383 }
384 if (inp->inp_faddr.s_addr) {
385 entry->soflow_faddr.addr46.ia46_addr4.s_addr = inp->inp_faddr.s_addr;
386 }
387 }
388 entry->soflow_family = AF_INET;
389 return TRUE;
390 }
391 return FALSE;
392 }
393
394 static errno_t
soflow_db_init(struct socket * so)395 soflow_db_init(struct socket *so)
396 {
397 errno_t error = 0;
398 struct soflow_db *db = NULL;
399 struct soflow_hash_entry *hash_entry = NULL;
400
401 db = kalloc_type(struct soflow_db, Z_WAITOK | Z_ZERO | Z_NOFAIL);
402 db->soflow_db_so = so;
403 db->soflow_db_hashbase = hashinit(SOFLOW_HASH_SIZE, M_CFIL, &db->soflow_db_hashmask);
404 if (db->soflow_db_hashbase == NULL) {
405 kfree_type(struct soflow_db, db);
406 error = ENOMEM;
407 goto done;
408 }
409 db->soflow_db_debug = SOFLOW_ENABLE_DEBUG(so, hash_entry);
410 os_ref_init(&db->soflow_db_ref_count, &soflow_refgrp);
411 so->so_flow_db = db;
412 done:
413 return error;
414 }
415
416 static void
soflow_entry_free(struct soflow_hash_entry * hash_entry)417 soflow_entry_free(struct soflow_hash_entry *hash_entry)
418 {
419 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
420
421 if (hash_entry == NULL) {
422 return;
423 }
424
425 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Free entry");
426 kfree_type(struct soflow_hash_entry, hash_entry);
427 }
428
429 static void
soflow_db_remove_entry(struct soflow_db * db,struct soflow_hash_entry * hash_entry)430 soflow_db_remove_entry(struct soflow_db *db, struct soflow_hash_entry *hash_entry)
431 {
432 if (hash_entry == NULL) {
433 return;
434 }
435 if (db == NULL || db->soflow_db_count == 0) {
436 return;
437 }
438
439 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
440 if (hash_entry->soflow_nstat_context != NULL) {
441 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, hash_entry->soflow_debug, "<Close nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
442 nstat_provider_stats_close(hash_entry->soflow_nstat_context);
443 hash_entry->soflow_nstat_context = NULL;
444 SOFLOW_ENTRY_FREE(hash_entry);
445 }
446 #endif
447
448 db->soflow_db_count--;
449 if (db->soflow_db_only_entry == hash_entry) {
450 db->soflow_db_only_entry = NULL;
451 }
452 LIST_REMOVE(hash_entry, soflow_entry_link);
453
454 // Feature context present, give feature a chance to detach and clean up
455 if (hash_entry->soflow_feat_ctxt != NULL && soflow_feat_detach_entry_func_ptr != NULL) {
456 soflow_feat_detach_entry_func_ptr(db->soflow_db_so, hash_entry);
457 hash_entry->soflow_feat_ctxt = NULL;
458 hash_entry->soflow_feat_ctxt_id = 0;
459 }
460
461 hash_entry->soflow_db = NULL;
462
463 SOFLOW_LOCK_EXCLUSIVE;
464 if (soflow_initialized) {
465 TAILQ_REMOVE(&soflow_entry_head, hash_entry, soflow_entry_list_link);
466 soflow_attached_count--;
467 }
468 SOFLOW_UNLOCK_EXCLUSIVE;
469
470 SOFLOW_ENTRY_FREE(hash_entry);
471 }
472
473 static void
soflow_db_free(struct soflow_db * db)474 soflow_db_free(struct soflow_db *db)
475 {
476 struct soflow_hash_entry *entry = NULL;
477 struct soflow_hash_entry *temp_entry = NULL;
478 struct soflow_hash_head *flowhash = NULL;
479
480 if (db == NULL) {
481 return;
482 }
483
484 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "<db %p> freeing db (count == %d)", db, db->soflow_db_count);
485
486 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
487 flowhash = &db->soflow_db_hashbase[i];
488 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
489 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Remove entry");
490 soflow_db_remove_entry(db, entry);
491 }
492 }
493
494 if (soflow_feat_detach_db_func_ptr != NULL) {
495 soflow_feat_detach_db_func_ptr(db->soflow_db_so, db);
496 }
497
498 // Make sure all entries are cleaned up!
499 VERIFY(db->soflow_db_count == 0);
500 hashdestroy(db->soflow_db_hashbase, M_CFIL, db->soflow_db_hashmask);
501 kfree_type(struct soflow_db, db);
502 }
503
504 void
soflow_detach(struct socket * so)505 soflow_detach(struct socket *so)
506 {
507 if (so == NULL || so->so_flow_db == NULL) {
508 return;
509 }
510 SOFLOW_DB_FREE(so->so_flow_db);
511 so->so_flow_db = NULL;
512 }
513
514 static boolean_t
soflow_match_entries_v4(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)515 soflow_match_entries_v4(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
516 {
517 if (entry1 == NULL || entry2 == NULL) {
518 return false;
519 }
520
521 // Ignore local match if remoteOnly or if local has been updated since entry added
522 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
523 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
524 entry1->soflow_laddr.addr46.ia46_addr4.s_addr == entry2->soflow_laddr.addr46.ia46_addr4.s_addr);
525
526 // Entries match if local and remote ports and addresses all matched
527 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
528 laddr_matched && entry1->soflow_faddr.addr46.ia46_addr4.s_addr == entry2->soflow_faddr.addr46.ia46_addr4.s_addr;
529 }
530
531 static boolean_t
soflow_match_entries_v6(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)532 soflow_match_entries_v6(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
533 {
534 if (entry1 == NULL || entry2 == NULL) {
535 return false;
536 }
537
538 // Ignore local match if remoteOnly or if local has been updated since entry added
539 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
540 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
541 in6_are_addr_equal_scoped(&entry1->soflow_laddr.addr6, &entry2->soflow_laddr.addr6, entry1->soflow_laddr6_ifscope, entry2->soflow_laddr6_ifscope));
542
543 // Entries match if local and remote ports and addresses all matched
544 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
545 laddr_matched && in6_are_addr_equal_scoped(&entry1->soflow_faddr.addr6, &entry2->soflow_faddr.addr6, entry1->soflow_faddr6_ifscope, entry2->soflow_faddr6_ifscope);
546 }
547
548 static struct soflow_hash_entry *
soflow_db_lookup_entry_internal(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly,boolean_t withLocalPort)549 soflow_db_lookup_entry_internal(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly, boolean_t withLocalPort)
550 {
551 struct soflow_hash_entry matchentry = { };
552 struct soflow_hash_entry *nextentry = NULL;
553 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
554 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
555 u_int16_t hashkey_fport = 0, hashkey_lport = 0;
556 int inp_hash_element = 0;
557 struct soflow_hash_head *flowhash = NULL;
558
559 if (inp == NULL || db == NULL) {
560 return NULL;
561 }
562
563 if (local != NULL) {
564 soflow_fill_hash_entry_from_address(&matchentry, TRUE, local, FALSE);
565 } else {
566 soflow_fill_hash_entry_from_inp(&matchentry, TRUE, inp, FALSE);
567 }
568 if (remote != NULL) {
569 soflow_fill_hash_entry_from_address(&matchentry, FALSE, remote, FALSE);
570 } else {
571 soflow_fill_hash_entry_from_inp(&matchentry, FALSE, inp, FALSE);
572 }
573 matchentry.soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, (&matchentry));
574 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, true, "Looking for entry");
575
576 if (inp->inp_vflag & INP_IPV6) {
577 hashkey_faddr = matchentry.soflow_faddr.addr6.s6_addr32[3];
578 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr6.s6_addr32[3] : 0;
579 } else {
580 hashkey_faddr = matchentry.soflow_faddr.addr46.ia46_addr4.s_addr;
581 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr46.ia46_addr4.s_addr : 0;
582 }
583
584 hashkey_fport = matchentry.soflow_fport;
585 hashkey_lport = (remoteOnly == false || withLocalPort == true) ? matchentry.soflow_lport : 0;
586
587 inp_hash_element = SOFLOW_HASH(hashkey_laddr, hashkey_faddr, hashkey_lport, hashkey_fport);
588 inp_hash_element &= db->soflow_db_hashmask;
589 flowhash = &db->soflow_db_hashbase[inp_hash_element];
590
591 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
592 if (inp->inp_vflag & INP_IPV6) {
593 if (soflow_match_entries_v6(nextentry, &matchentry, remoteOnly)) {
594 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v6");
595 break;
596 }
597 } else if (inp->inp_vflag & INP_IPV4) {
598 if (soflow_match_entries_v4(nextentry, &matchentry, remoteOnly)) {
599 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v4");
600 break;
601 }
602 }
603 }
604
605 if (nextentry == NULL) {
606 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, matchentry.soflow_debug, "Entry not found");
607 }
608 return nextentry;
609 }
610
611 static struct soflow_hash_entry *
soflow_db_lookup_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly)612 soflow_db_lookup_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly)
613 {
614 struct soflow_hash_entry *entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, false);
615 if (entry == NULL && remoteOnly == true) {
616 entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, true);
617 }
618 return entry;
619 }
620
621 static struct soflow_hash_entry *
soflow_db_lookup_by_feature_context_id(struct soflow_db * db,u_int64_t feature_context_id)622 soflow_db_lookup_by_feature_context_id(struct soflow_db *db, u_int64_t feature_context_id)
623 {
624 struct soflow_hash_head *flowhash = NULL;
625 u_int32_t inp_hash_element = (u_int32_t)(feature_context_id & 0x0ffffffff);
626 struct soflow_hash_entry *nextentry;
627
628 inp_hash_element &= db->soflow_db_hashmask;
629 flowhash = &db->soflow_db_hashbase[inp_hash_element];
630
631 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
632 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Looking at entry");
633 if (nextentry->soflow_feat_ctxt != NULL &&
634 nextentry->soflow_feat_ctxt_id == feature_context_id) {
635 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry by feature context id");
636 break;
637 }
638 }
639
640 if (nextentry == NULL) {
641 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "No entry found for featureID %llu <count %d hash %X %X>",
642 feature_context_id, db->soflow_db_count, inp_hash_element, (u_int32_t)(feature_context_id & 0x0ffffffff));
643 }
644 return nextentry;
645 }
646
647 void *
soflow_db_get_feature_context(struct soflow_db * db,u_int64_t feature_context_id)648 soflow_db_get_feature_context(struct soflow_db *db, u_int64_t feature_context_id)
649 {
650 struct soflow_hash_entry *hash_entry = NULL;
651 void *context = NULL;
652
653 if (db == NULL || db->soflow_db_so == NULL || feature_context_id == 0) {
654 return NULL;
655 }
656
657 socket_lock_assert_owned(db->soflow_db_so);
658
659 // Take refcount of db before use.
660 // Abort if db is already being freed.
661 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
662 return NULL;
663 }
664
665 // This is an optimization for datagram sockets with only one single flow.
666 if (db->soflow_db_count == 1) {
667 if (db->soflow_db_only_entry != NULL &&
668 db->soflow_db_only_entry->soflow_feat_ctxt != NULL && db->soflow_db_only_entry->soflow_feat_ctxt_id == feature_context_id) {
669 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_only_entry, db->soflow_db_only_entry->soflow_debug, "MATCHED only entry for featureID");
670 context = db->soflow_db_only_entry->soflow_feat_ctxt;
671 } else {
672 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "MISMATCHED only entry for featureID %llu (entry %p - cfil %p id %llu)",
673 feature_context_id,
674 db->soflow_db_only_entry,
675 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt : NULL,
676 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt_id : 0);
677 }
678 } else {
679 hash_entry = soflow_db_lookup_by_feature_context_id(db, feature_context_id);
680 context = hash_entry != NULL ? hash_entry->soflow_feat_ctxt : NULL;
681 }
682
683 SOFLOW_DB_FREE(db);
684 return context;
685 }
686
687 u_int64_t
soflow_db_get_feature_context_id(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)688 soflow_db_get_feature_context_id(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
689 {
690 struct soflow_hash_entry *hash_entry = NULL;
691 uint64_t context_id = 0;
692
693 if (db == NULL || db->soflow_db_so == NULL) {
694 return 0;
695 }
696
697 socket_lock_assert_owned(db->soflow_db_so);
698
699 // Take refcount of db before use.
700 // Abort if db is already being freed.
701 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
702 return 0;
703 }
704
705 hash_entry = soflow_db_lookup_entry(db, local, remote, false);
706 if (hash_entry == NULL) {
707 // No match with both local and remote, try match with remote only
708 hash_entry = soflow_db_lookup_entry(db, local, remote, true);
709 }
710 if (hash_entry != NULL && hash_entry->soflow_feat_ctxt != NULL) {
711 context_id = hash_entry->soflow_feat_ctxt_id;
712 }
713
714 SOFLOW_DB_FREE(db);
715
716 return context_id;
717 }
718
719 static struct soflow_hash_entry *
soflow_db_add_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)720 soflow_db_add_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
721 {
722 struct soflow_hash_entry *entry = NULL;
723 struct inpcb *inp = db ? sotoinpcb(db->soflow_db_so) : NULL;
724 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
725 int inp_hash_element = 0;
726 struct soflow_hash_head *flowhash = NULL;
727
728 if (db == NULL || inp == NULL) {
729 goto done;
730 }
731
732 entry = kalloc_type(struct soflow_hash_entry, Z_WAITOK | Z_ZERO | Z_NOFAIL);
733 os_ref_init(&entry->soflow_ref_count, &soflow_refgrp);
734
735 if (local != NULL) {
736 soflow_fill_hash_entry_from_address(entry, TRUE, local, FALSE);
737 } else {
738 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, FALSE);
739 }
740 if (remote != NULL) {
741 soflow_fill_hash_entry_from_address(entry, FALSE, remote, FALSE);
742 } else {
743 soflow_fill_hash_entry_from_inp(entry, FALSE, inp, FALSE);
744 }
745 entry->soflow_lastused = net_uptime();
746 entry->soflow_db = db;
747 entry->soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, entry);
748
749 if (inp->inp_vflag & INP_IPV6) {
750 hashkey_faddr = entry->soflow_faddr.addr6.s6_addr32[3];
751 hashkey_laddr = entry->soflow_laddr.addr6.s6_addr32[3];
752 } else {
753 hashkey_faddr = entry->soflow_faddr.addr46.ia46_addr4.s_addr;
754 hashkey_laddr = entry->soflow_laddr.addr46.ia46_addr4.s_addr;
755 }
756 entry->soflow_flowhash = SOFLOW_HASH(hashkey_laddr, hashkey_faddr,
757 entry->soflow_lport, entry->soflow_fport);
758 inp_hash_element = entry->soflow_flowhash & db->soflow_db_hashmask;
759
760 socket_lock_assert_owned(db->soflow_db_so);
761
762 // Take refcount of db before use.
763 // Abort if db is already being freed.
764 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
765 return NULL;
766 }
767
768 flowhash = &db->soflow_db_hashbase[inp_hash_element];
769
770 LIST_INSERT_HEAD(flowhash, entry, soflow_entry_link);
771 db->soflow_db_count++;
772 db->soflow_db_only_entry = entry;
773 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "total count %d", db->soflow_db_count);
774
775 SOFLOW_DB_FREE(db);
776
777 done:
778 return entry;
779 }
780
781 static int
soflow_udp_get_address_from_control(sa_family_t family,struct mbuf * control,uint8_t ** address_ptr)782 soflow_udp_get_address_from_control(sa_family_t family, struct mbuf *control, uint8_t **address_ptr)
783 {
784 struct cmsghdr *cm;
785 struct in6_pktinfo *pi6;
786 struct socket *so = NULL;
787
788 if (control == NULL || address_ptr == NULL) {
789 return 0;
790 }
791
792 for (; control != NULL; control = control->m_next) {
793 if (control->m_type != MT_CONTROL) {
794 continue;
795 }
796
797 for (cm = M_FIRST_CMSGHDR(control);
798 is_cmsg_valid(control, cm);
799 cm = M_NXT_CMSGHDR(control, cm)) {
800 SOFLOW_LOG(LOG_DEBUG, so, true, "Check control type %d", cm->cmsg_type);
801
802 switch (cm->cmsg_type) {
803 case IP_RECVDSTADDR:
804 if (family == AF_INET &&
805 cm->cmsg_level == IPPROTO_IP &&
806 cm->cmsg_len == CMSG_LEN(sizeof(struct in_addr))) {
807 *address_ptr = CMSG_DATA(cm);
808 return sizeof(struct in_addr);
809 }
810 break;
811 case IPV6_PKTINFO:
812 case IPV6_2292PKTINFO:
813 if (family == AF_INET6 &&
814 cm->cmsg_level == IPPROTO_IPV6 &&
815 cm->cmsg_len == CMSG_LEN(sizeof(struct in6_pktinfo))) {
816 pi6 = (struct in6_pktinfo *)(void *)CMSG_DATA(cm);
817 *address_ptr = (uint8_t *)&pi6->ipi6_addr;
818 return sizeof(struct in6_addr);
819 }
820 break;
821 default:
822 break;
823 }
824 }
825 }
826 return 0;
827 }
828
829 static boolean_t
soflow_entry_local_address_needs_update(struct soflow_hash_entry * entry)830 soflow_entry_local_address_needs_update(struct soflow_hash_entry *entry)
831 {
832 if (entry->soflow_family == AF_INET6) {
833 return IN6_IS_ADDR_UNSPECIFIED(&entry->soflow_laddr.addr6);
834 } else if (entry->soflow_family == AF_INET) {
835 return entry->soflow_laddr.addr46.ia46_addr4.s_addr == INADDR_ANY;
836 }
837 return false;
838 }
839
840 static boolean_t
soflow_entry_local_port_needs_update(struct socket * so,struct soflow_hash_entry * entry)841 soflow_entry_local_port_needs_update(struct socket *so, struct soflow_hash_entry *entry)
842 {
843 if (SOFLOW_IS_UDP(so)) {
844 return entry->soflow_lport == 0;
845 }
846 return false;
847 }
848
849 static void
soflow_entry_update_local(struct soflow_db * db,struct soflow_hash_entry * entry,struct sockaddr * local,struct mbuf * control,u_short rcv_ifindex)850 soflow_entry_update_local(struct soflow_db *db, struct soflow_hash_entry *entry, struct sockaddr *local, struct mbuf *control, u_short rcv_ifindex)
851 {
852 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
853 union sockaddr_in_4_6 address_buf = { };
854
855 if (inp == NULL || entry == NULL) {
856 return;
857 }
858
859 if (entry->soflow_outifindex == 0 && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
860 entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
861 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated outifp");
862 }
863
864 if (soflow_entry_local_address_needs_update(entry)) {
865 // Flow does not have a local address yet. Retrieve local address
866 // from control mbufs if present.
867 if (local == NULL && control != NULL) {
868 uint8_t *addr_ptr = NULL;
869 int size = soflow_udp_get_address_from_control(entry->soflow_family, control, &addr_ptr);
870
871 if (size && addr_ptr) {
872 switch (entry->soflow_family) {
873 case AF_INET:
874 if (size == sizeof(struct in_addr)) {
875 address_buf.sin.sin_port = 0;
876 address_buf.sin.sin_family = AF_INET;
877 address_buf.sin.sin_len = sizeof(struct sockaddr_in);
878 (void) memcpy(&address_buf.sin.sin_addr, addr_ptr, sizeof(struct in_addr));
879 local = sintosa(&address_buf.sin);
880 }
881 break;
882 case AF_INET6:
883 if (size == sizeof(struct in6_addr)) {
884 address_buf.sin6.sin6_port = 0;
885 address_buf.sin6.sin6_family = AF_INET6;
886 address_buf.sin6.sin6_len = sizeof(struct sockaddr_in6);
887 (void) memcpy(&address_buf.sin6.sin6_addr, addr_ptr, sizeof(struct in6_addr));
888 local = sin6tosa(&address_buf.sin6);
889 }
890 break;
891 default:
892 break;
893 }
894 }
895 }
896 if (local != NULL) {
897 soflow_fill_hash_entry_from_address(entry, TRUE, local, TRUE);
898 } else {
899 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
900 }
901 if (entry->soflow_laddr_updated) {
902 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated address");
903 }
904 }
905
906 if (soflow_entry_local_port_needs_update(db->soflow_db_so, entry)) {
907 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
908 if (entry->soflow_lport_updated) {
909 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated port");
910 }
911 }
912
913 return;
914 }
915
916 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
917 static u_int32_t
ifnet_to_flags(struct ifnet * ifp,struct socket * so)918 ifnet_to_flags(struct ifnet *ifp, struct socket *so)
919 {
920 u_int32_t flags = 0;
921
922 if (ifp != NULL) {
923 flags = nstat_ifnet_to_flags(ifp);
924 if ((flags & NSTAT_IFNET_IS_WIFI) && ((flags & (NSTAT_IFNET_IS_AWDL | NSTAT_IFNET_IS_LLW)) == 0)) {
925 flags |= NSTAT_IFNET_IS_WIFI_INFRA;
926 }
927 } else {
928 flags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
929 }
930
931 if (so != NULL && (so->so_flags1 & SOF1_CELLFALLBACK)) {
932 flags |= NSTAT_IFNET_VIA_CELLFALLBACK;
933 }
934 return flags;
935 }
936
937 static bool
soflow_nstat_provider_request_vals(nstat_provider_context ctx,u_int32_t * ifflagsp,nstat_counts * countsp,void * metadatap)938 soflow_nstat_provider_request_vals(nstat_provider_context ctx,
939 u_int32_t *ifflagsp,
940 nstat_counts *countsp,
941 void *metadatap)
942 {
943 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
944 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
945 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
946 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
947 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
948 const void *addr = NULL;
949
950 if (hash_entry == NULL || so == NULL || inp == NULL) {
951 return false;
952 }
953
954 if (ifflagsp) {
955 if (hash_entry->soflow_outifindex) {
956 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
957 *ifflagsp = ifnet_to_flags(ifp, so);
958 }
959 if ((countsp == NULL) && (metadatap == NULL)) {
960 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "ifflagsp set to 0x%X", *ifflagsp);
961 goto done;
962 }
963 }
964
965 if (countsp) {
966 bzero(countsp, sizeof(*countsp));
967 countsp->nstat_rxpackets = hash_entry->soflow_rxpackets;
968 countsp->nstat_rxbytes = hash_entry->soflow_rxbytes;
969 countsp->nstat_txpackets = hash_entry->soflow_txpackets;
970 countsp->nstat_txbytes = hash_entry->soflow_txbytes;
971
972 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
973 "Collected NSTAT counts: rxpackets %llu rxbytes %llu txpackets %llu txbytes %llu",
974 countsp->nstat_rxpackets, countsp->nstat_rxbytes, countsp->nstat_txpackets, countsp->nstat_txbytes);
975 }
976
977 if (metadatap) {
978 nstat_udp_descriptor *desc = (nstat_udp_descriptor *)metadatap;
979 bzero(desc, sizeof(*desc));
980
981 if (so->so_flags & SOF_DELEGATED) {
982 desc->eupid = so->e_upid;
983 desc->epid = so->e_pid;
984 uuid_copy(desc->euuid, so->e_uuid);
985 } else {
986 desc->eupid = so->last_upid;
987 desc->epid = so->last_pid;
988 uuid_copy(desc->euuid, so->last_uuid);
989 }
990
991 uuid_copy(desc->vuuid, so->so_vuuid);
992 uuid_copy(desc->fuuid, hash_entry->soflow_uuid);
993
994 if (hash_entry->soflow_family == AF_INET6) {
995 in6_ip6_to_sockaddr(&hash_entry->soflow_laddr.addr6, hash_entry->soflow_lport, hash_entry->soflow_laddr6_ifscope,
996 &desc->local.v6, sizeof(desc->local.v6));
997 in6_ip6_to_sockaddr(&hash_entry->soflow_faddr.addr6, hash_entry->soflow_fport, hash_entry->soflow_faddr6_ifscope,
998 &desc->remote.v6, sizeof(desc->remote.v6));
999 } else if (hash_entry->soflow_family == AF_INET) {
1000 desc->local.v4.sin_family = AF_INET;
1001 desc->local.v4.sin_len = sizeof(struct sockaddr_in);
1002 desc->local.v4.sin_port = hash_entry->soflow_lport;
1003 desc->local.v4.sin_addr = hash_entry->soflow_laddr.addr46.ia46_addr4;
1004
1005 desc->remote.v4.sin_family = AF_INET;
1006 desc->remote.v4.sin_len = sizeof(struct sockaddr_in);
1007 desc->remote.v4.sin_port = hash_entry->soflow_fport;
1008 desc->remote.v4.sin_addr = hash_entry->soflow_faddr.addr46.ia46_addr4;
1009 }
1010
1011 desc->ifindex = hash_entry->soflow_outifindex;
1012 if (hash_entry->soflow_outifindex) {
1013 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
1014 desc->ifnet_properties = (uint16_t)ifnet_to_flags(ifp, so);
1015 }
1016
1017 desc->rcvbufsize = so->so_rcv.sb_hiwat;
1018 desc->rcvbufused = so->so_rcv.sb_cc;
1019 desc->traffic_class = so->so_traffic_class;
1020 inp_get_activity_bitmap(inp, &desc->activity_bitmap);
1021
1022 if (hash_entry->soflow_debug) {
1023 switch (hash_entry->soflow_family) {
1024 case AF_INET6:
1025 addr = &desc->local.v6;
1026 inet_ntop(AF_INET6, addr, local, sizeof(local));
1027 addr = &desc->remote.v6;
1028 inet_ntop(AF_INET6, addr, remote, sizeof(local));
1029 break;
1030 case AF_INET:
1031 addr = &desc->local.v4.sin_addr;
1032 inet_ntop(AF_INET, addr, local, sizeof(local));
1033 addr = &desc->remote.v4.sin_addr;
1034 inet_ntop(AF_INET, addr, remote, sizeof(local));
1035 break;
1036 default:
1037 break;
1038 }
1039
1040 uint8_t *ptr = (uint8_t *)&desc->euuid;
1041
1042 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
1043 "Collected NSTAT metadata: eupid %llu epid %d euuid %x%x%x%x-%x%x%x%x-%x%x%x%x-%x%x%x%x "
1044 "outifp %d properties 0x%X lport %d fport %d laddr %s faddr %s "
1045 "rcvbufsize %u rcvbufused %u traffic_class %u",
1046 desc->eupid, desc->epid,
1047 ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7],
1048 ptr[8], ptr[9], ptr[10], ptr[11], ptr[12], ptr[13], ptr[14], ptr[15],
1049 desc->ifindex, desc->ifnet_properties,
1050 ntohs(desc->local.v4.sin_port), ntohs(desc->remote.v4.sin_port), local, remote,
1051 desc->rcvbufsize, desc->rcvbufused, desc->traffic_class);
1052 }
1053 }
1054 done:
1055 return true;
1056 }
1057
1058 static size_t
soflow_nstat_provider_request_extensions(nstat_provider_context ctx,int requested_extension,void * buf,size_t buf_size)1059 soflow_nstat_provider_request_extensions(nstat_provider_context ctx,
1060 int requested_extension,
1061 void *buf,
1062 size_t buf_size)
1063 {
1064 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
1065 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1066 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
1067 struct nstat_domain_info *domain_info = NULL;
1068 size_t size = 0;
1069
1070 if (hash_entry == NULL || so == NULL || inp == NULL) {
1071 return 0;
1072 }
1073
1074 if (buf == NULL) {
1075 switch (requested_extension) {
1076 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1077 return sizeof(nstat_domain_info);
1078 default:
1079 return 0;
1080 }
1081 }
1082
1083 if (buf_size < sizeof(nstat_domain_info)) {
1084 return 0;
1085 }
1086
1087 switch (requested_extension) {
1088 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1089
1090 domain_info = (struct nstat_domain_info *)buf;
1091 necp_copy_inp_domain_info(inp, so, domain_info);
1092
1093 if (hash_entry->soflow_debug) {
1094 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "Collected NSTAT domain_info:pid %d domain <%s> owner <%s> "
1095 "ctxt <%s> bundle id <%s> is_tracker %d is_non_app_initiated %d is_silent %d",
1096 so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid,
1097 domain_info->domain_name,
1098 domain_info->domain_owner,
1099 domain_info->domain_tracker_ctxt,
1100 domain_info->domain_attributed_bundle_id,
1101 domain_info->is_tracker,
1102 domain_info->is_non_app_initiated,
1103 domain_info->is_silent);
1104 }
1105 size = sizeof(nstat_domain_info);
1106
1107 default:
1108 break;
1109 }
1110
1111 return size;
1112 }
1113 #endif
1114
1115 static void
soflow_update_flow_stats(struct soflow_hash_entry * hash_entry,size_t data_size,bool outgoing)1116 soflow_update_flow_stats(struct soflow_hash_entry *hash_entry, size_t data_size, bool outgoing)
1117 {
1118 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1119
1120 if (hash_entry != NULL) {
1121 if (outgoing) {
1122 hash_entry->soflow_txbytes += data_size;
1123 hash_entry->soflow_txpackets++;
1124 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Outgoing");
1125 } else {
1126 hash_entry->soflow_rxbytes += data_size;
1127 hash_entry->soflow_rxpackets++;
1128 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Incoming");
1129 }
1130 }
1131 }
1132
1133 struct soflow_hash_entry *
soflow_get_flow(struct socket * so,struct sockaddr * local,struct sockaddr * remote,struct mbuf * control,size_t data_size,bool outgoing,uint16_t rcv_ifindex)1134 soflow_get_flow(struct socket *so, struct sockaddr *local, struct sockaddr *remote, struct mbuf *control,
1135 size_t data_size, bool outgoing, uint16_t rcv_ifindex)
1136 {
1137 struct soflow_hash_entry *hash_entry = NULL;
1138 struct inpcb *inp = sotoinpcb(so);
1139
1140 // Check if feature is disabled
1141 if (soflow_disable) {
1142 return NULL;
1143 }
1144
1145 socket_lock_assert_owned(so);
1146
1147 if (so->so_flow_db != NULL) {
1148 // Take refcount of db before use.
1149 // Abort if db is already being freed.
1150 if (os_ref_retain_try(&so->so_flow_db->soflow_db_ref_count) == false) {
1151 return NULL;
1152 }
1153
1154 // DB already exists, check if this is existing flow
1155 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, false);
1156 if (hash_entry == NULL) {
1157 // No match with both local and remote, try match with remote only
1158 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, true);
1159 }
1160 if (hash_entry != NULL) {
1161 // Take refcount of entry before use.
1162 // Abort if entry is already being freed.
1163 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1164 SOFLOW_DB_FREE(so->so_flow_db);
1165 return NULL;
1166 }
1167
1168 // Try to update flow info from socket and/or control mbufs if necessary
1169 if (hash_entry->soflow_outifindex == 0 ||
1170 soflow_entry_local_address_needs_update(hash_entry) || soflow_entry_local_port_needs_update(so, hash_entry)) {
1171 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1172 }
1173 hash_entry->soflow_lastused = net_uptime();
1174 soflow_update_flow_stats(hash_entry, data_size, outgoing);
1175
1176 SOFLOW_DB_FREE(so->so_flow_db);
1177 return hash_entry;
1178 }
1179
1180 SOFLOW_DB_FREE(so->so_flow_db);
1181 } else {
1182 // If new socket, allocate cfil db
1183 if (soflow_db_init(so) != 0) {
1184 return NULL;
1185 }
1186 }
1187
1188 hash_entry = soflow_db_add_entry(so->so_flow_db, local, remote);
1189 if (hash_entry == NULL) {
1190 SOFLOW_LOG(LOG_ERR, so, true, "Failed to add entry");
1191 return NULL;
1192 }
1193
1194 // Take refcount of entry before use.
1195 // Abort if entry is already being freed.
1196 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1197 return NULL;
1198 }
1199
1200 if (inp && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
1201 hash_entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
1202 }
1203
1204 // Check if we can update the new flow's local address from control mbufs
1205 if (control != NULL) {
1206 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1207 }
1208 hash_entry->soflow_outgoing = outgoing;
1209 soflow_update_flow_stats(hash_entry, data_size, outgoing);
1210
1211 // Only report flow to NSTAT if unconnected UDP
1212 if (!soflow_nstat_disable && SOFLOW_IS_UDP(so) && !(so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING))) {
1213 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
1214 // Take refcount of entry before handing it to nstat. Abort if fail.
1215 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1216 return NULL;
1217 }
1218 uuid_generate_random(hash_entry->soflow_uuid);
1219 hash_entry->soflow_nstat_context = nstat_provider_stats_open((nstat_provider_context) hash_entry,
1220 NSTAT_PROVIDER_UDP_SUBFLOW, 0,
1221 soflow_nstat_provider_request_vals,
1222 soflow_nstat_provider_request_extensions);
1223 SOFLOW_LOG(LOG_INFO, so, hash_entry->soflow_debug, "<Open nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
1224 #endif
1225 }
1226
1227 SOFLOW_LOCK_EXCLUSIVE;
1228 if (soflow_initialized == 0) {
1229 soflow_init();
1230 }
1231 TAILQ_INSERT_TAIL(&soflow_entry_head, hash_entry, soflow_entry_list_link);
1232 if (soflow_attached_count == 0) {
1233 thread_wakeup((caddr_t)&soflow_attached_count);
1234 }
1235 soflow_attached_count++;
1236 if (soflow_attached_high_water_mark < soflow_attached_count) {
1237 soflow_attached_high_water_mark = soflow_attached_count;
1238 }
1239 SOFLOW_UNLOCK_EXCLUSIVE;
1240
1241 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Added entry");
1242 return hash_entry;
1243 }
1244
1245 void
soflow_free_flow(struct soflow_hash_entry * entry)1246 soflow_free_flow(struct soflow_hash_entry *entry)
1247 {
1248 SOFLOW_ENTRY_FREE(entry);
1249 }
1250
1251 static bool
soflow_socket_safe_lock(struct inpcb * inp,struct inpcbinfo * pcbinfo)1252 soflow_socket_safe_lock(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1253 {
1254 struct socket *so = NULL;
1255
1256 VERIFY(pcbinfo != NULL);
1257
1258 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
1259 // Safeguarded the inp state, unlock pcbinfo before locking socket.
1260 lck_rw_done(&pcbinfo->ipi_lock);
1261
1262 so = inp->inp_socket;
1263 socket_lock(so, 1);
1264 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) != WNT_STOPUSING) {
1265 return true;
1266 }
1267 } else {
1268 // Failed to safeguarded the inp state, unlock pcbinfo and abort.
1269 lck_rw_done(&pcbinfo->ipi_lock);
1270 }
1271
1272 if (so) {
1273 socket_unlock(so, 1);
1274 }
1275 return false;
1276 }
1277
1278 static struct socket *
soflow_validate_dgram_socket(struct socket * so)1279 soflow_validate_dgram_socket(struct socket *so)
1280 {
1281 struct inpcb *inp = NULL;
1282 struct inpcbinfo *pcbinfo = NULL;
1283 struct socket *locked = NULL;
1284
1285 pcbinfo = &udbinfo;
1286 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1287 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1288 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1289 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1290 locked = inp->inp_socket;
1291 }
1292 /* pcbinfo is already unlocked, we are done. */
1293 goto done;
1294 }
1295 }
1296 lck_rw_done(&pcbinfo->ipi_lock);
1297 if (locked != NULL) {
1298 goto done;
1299 }
1300
1301 pcbinfo = &ripcbinfo;
1302 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1303 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1304 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1305 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1306 locked = inp->inp_socket;
1307 }
1308 /* pcbinfo is already unlocked, we are done. */
1309 goto done;
1310 }
1311 }
1312 lck_rw_done(&pcbinfo->ipi_lock);
1313
1314 done:
1315 return locked;
1316 }
1317
1318 static void
soflow_gc_thread_sleep(bool forever)1319 soflow_gc_thread_sleep(bool forever)
1320 {
1321 if (forever) {
1322 (void) assert_wait((event_t) &soflow_attached_count,
1323 THREAD_INTERRUPTIBLE);
1324 } else {
1325 uint64_t deadline = 0;
1326 nanoseconds_to_absolutetime(SOFLOW_GC_RUN_INTERVAL_NSEC, &deadline);
1327 clock_absolutetime_interval_to_deadline(deadline, &deadline);
1328
1329 (void) assert_wait_deadline(&soflow_attached_count,
1330 THREAD_INTERRUPTIBLE, deadline);
1331 }
1332 }
1333
1334 static void
soflow_gc_thread_func(void * v,wait_result_t w)1335 soflow_gc_thread_func(void *v, wait_result_t w)
1336 {
1337 #pragma unused(v, w)
1338
1339 ASSERT(soflow_gc_thread == current_thread());
1340 thread_set_thread_name(current_thread(), "SOFLOW_GC");
1341
1342 // Kick off gc shortly
1343 soflow_gc_thread_sleep(false);
1344 thread_block_parameter((thread_continue_t) soflow_gc_expire, NULL);
1345 /* NOTREACHED */
1346 }
1347
1348 static bool
soflow_gc_idle_timed_out(struct soflow_hash_entry * hash_entry,int timeout,u_int64_t current_time)1349 soflow_gc_idle_timed_out(struct soflow_hash_entry *hash_entry, int timeout, u_int64_t current_time)
1350 {
1351 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1352
1353 if (hash_entry && (current_time - hash_entry->soflow_lastused >= (u_int64_t)timeout)) {
1354 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "GC Idle Timeout detected");
1355 return true;
1356 }
1357 return false;
1358 }
1359
1360 static int
soflow_gc_cleanup(struct socket * so)1361 soflow_gc_cleanup(struct socket *so)
1362 {
1363 struct soflow_hash_entry *entry = NULL;
1364 struct soflow_hash_entry *temp_entry = NULL;
1365 struct soflow_hash_head *flowhash = NULL;
1366 struct soflow_db *db = NULL;
1367 int cleaned = 0;
1368
1369 if (so == NULL || so->so_flow_db == NULL) {
1370 return 0;
1371 }
1372 db = so->so_flow_db;
1373
1374 socket_lock_assert_owned(so);
1375
1376 // Take refcount of db before use.
1377 // Abort if db is already being freed.
1378 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1379 return 0;
1380 }
1381
1382 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1383 flowhash = &db->soflow_db_hashbase[i];
1384 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1385 if (entry->soflow_gc || entry->soflow_feat_gc) {
1386 if (entry->soflow_feat_ctxt != NULL && soflow_feat_gc_perform_func_ptr != NULL) {
1387 soflow_feat_gc_perform_func_ptr(so, entry);
1388 entry->soflow_feat_ctxt = NULL;
1389 entry->soflow_feat_ctxt_id = 0;
1390 }
1391 entry->soflow_feat_gc = 0;
1392
1393 if (entry->soflow_gc) {
1394 SOFLOW_ENTRY_LOG(LOG_INFO, so, entry, entry->soflow_debug, "GC cleanup entry");
1395 entry->soflow_gc = 0;
1396 soflow_db_remove_entry(db, entry);
1397 cleaned++;
1398 }
1399 }
1400 }
1401 }
1402
1403 SOFLOW_DB_FREE(db);
1404 return cleaned;
1405 }
1406
1407 static void
soflow_gc_expire(void * v,wait_result_t w)1408 soflow_gc_expire(void *v, wait_result_t w)
1409 {
1410 #pragma unused(v, w)
1411
1412 static struct socket *socket_array[SOFLOW_GC_MAX_COUNT];
1413 struct soflow_hash_entry *hash_entry = NULL;
1414 struct socket *so = NULL;
1415 u_int64_t current_time = net_uptime();
1416 uint32_t socket_count = 0;
1417 uint32_t cleaned_count = 0;
1418 bool recorded = false;
1419
1420 // Collect a list of socket with expired flows
1421
1422 SOFLOW_LOCK_SHARED;
1423
1424 if (soflow_attached_count == 0) {
1425 SOFLOW_UNLOCK_SHARED;
1426 goto go_sleep;
1427 }
1428
1429 // Go thorough all flows in the flow list and record any socket with expired flows.
1430 TAILQ_FOREACH(hash_entry, &soflow_entry_head, soflow_entry_list_link) {
1431 if (socket_count >= SOFLOW_GC_MAX_COUNT) {
1432 break;
1433 }
1434 so = hash_entry->soflow_db ? hash_entry->soflow_db->soflow_db_so : NULL;
1435
1436 // Check if we need to perform cleanup due to idle time or feature specified rules
1437 hash_entry->soflow_gc = soflow_gc_idle_timed_out(hash_entry, SOFLOW_GC_IDLE_TO, current_time);
1438 hash_entry->soflow_feat_gc = (soflow_feat_gc_needed_func_ptr != NULL && soflow_feat_gc_needed_func_ptr(so, hash_entry, current_time));
1439
1440 if (hash_entry->soflow_gc || hash_entry->soflow_feat_gc) {
1441 if (so != NULL) {
1442 recorded = false;
1443 for (int i = 0; i < socket_count; i++) {
1444 if (socket_array[socket_count] == so) {
1445 recorded = true;
1446 break;
1447 }
1448 }
1449 if (recorded == false) {
1450 socket_array[socket_count] = so;
1451 socket_count++;
1452 }
1453 }
1454 }
1455 }
1456 SOFLOW_UNLOCK_SHARED;
1457
1458 if (socket_count == 0) {
1459 goto go_sleep;
1460 }
1461
1462 for (uint32_t i = 0; i < socket_count; i++) {
1463 // Validate socket and lock it
1464 so = soflow_validate_dgram_socket(socket_array[i]);
1465 if (so == NULL) {
1466 continue;
1467 }
1468 cleaned_count += soflow_gc_cleanup(so);
1469 socket_unlock(so, 1);
1470 }
1471
1472 so = NULL;
1473 SOFLOW_LOG(LOG_INFO, so, true, "<GC cleaned %d flows>", cleaned_count);
1474
1475 go_sleep:
1476
1477 // Sleep forever (until waken up) if no more UDP flow to clean
1478 SOFLOW_LOCK_SHARED;
1479 soflow_gc_thread_sleep(soflow_attached_count == 0 ? true : false);
1480 SOFLOW_UNLOCK_SHARED;
1481 thread_block_parameter((thread_continue_t)soflow_gc_expire, NULL);
1482 /* NOTREACHED */
1483 }
1484
1485 void
soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,soflow_feat_gc_perform_func gc_perform_fn,soflow_feat_detach_entry_func feat_detach_entry_fn,soflow_feat_detach_db_func feat_detach_db_fn)1486 soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,
1487 soflow_feat_gc_perform_func gc_perform_fn,
1488 soflow_feat_detach_entry_func feat_detach_entry_fn,
1489 soflow_feat_detach_db_func feat_detach_db_fn)
1490 {
1491 soflow_feat_gc_needed_func_ptr = gc_needed_fn;
1492 soflow_feat_gc_perform_func_ptr = gc_perform_fn;
1493 soflow_feat_detach_entry_func_ptr = feat_detach_entry_fn;
1494 soflow_feat_detach_db_func_ptr = feat_detach_db_fn;
1495 }
1496
1497 bool
soflow_db_apply(struct soflow_db * db,soflow_entry_apply_func entry_apply_fn,void * context)1498 soflow_db_apply(struct soflow_db *db, soflow_entry_apply_func entry_apply_fn, void *context)
1499 {
1500 struct soflow_hash_entry *entry = NULL;
1501 struct soflow_hash_entry *temp_entry = NULL;
1502 struct soflow_hash_head *flowhash = NULL;
1503
1504 if (db == NULL || db->soflow_db_so == NULL || entry_apply_fn == NULL) {
1505 return false;
1506 }
1507
1508 socket_lock_assert_owned(db->soflow_db_so);
1509
1510 // Take refcount of db before use.
1511 // Abort if db is already being freed.
1512 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1513 return false;
1514 }
1515
1516 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1517 flowhash = &db->soflow_db_hashbase[i];
1518 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1519 if (entry_apply_fn(db->soflow_db_so, entry, context) == false) {
1520 goto done;
1521 }
1522 }
1523 }
1524
1525 done:
1526 SOFLOW_DB_FREE(db);
1527 return true;
1528 }
1529