1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 /*
25 * LOCKING STRATEGY
26 *
27 * The struct socket's so_flow_db field (struct soflow_db and its hash entries
28 * struct soflow_hash_entry) is protected by the socket lock. This covers all the
29 * socket paths that calls soflow_get_flow() as well as the garbage collection.
30 * For the socket detach path, soflow_detach() cannot assume the socket lock is
31 * held. Thus, reference counts are added to both struct soflow_db and struct
32 * soflow_hash_entry to avoid access after freed issues.
33 *
34 * The global list, soflow_entry_head, keeps track of all struct soflow_hash_entry
35 * entries which is used by garbage collection when detecting idle entries. This list
36 * is protected by the global lock soflow_lck_rw.
37 *
38 */
39
40 #include <sys/types.h>
41 #include <sys/kern_control.h>
42 #include <sys/queue.h>
43 #include <sys/domain.h>
44 #include <sys/protosw.h>
45 #include <sys/syslog.h>
46 #include <sys/systm.h>
47 #include <sys/sysproto.h>
48 #include <sys/socket.h>
49 #include <sys/socketvar.h>
50
51 #include <kern/sched_prim.h>
52 #include <kern/locks.h>
53 #include <kern/zalloc.h>
54 #include <kern/debug.h>
55 #include <net/ntstat.h>
56 #include <netinet6/in6_var.h>
57
58 #define _IP_VHL
59 #include <netinet/ip.h>
60 #include <netinet/in_pcb.h>
61 #include <netinet/udp.h>
62 #include <netinet/udp_var.h>
63
64 #include <string.h>
65 #include <libkern/libkern.h>
66 #include <kern/socket_flows.h>
67
68 extern struct inpcbinfo ripcbinfo;
69
70 /*
71 * Per-Socket Flow Management
72 */
73
74 static int soflow_log_level = LOG_ERR;
75 static int soflow_log_port = 0;
76 static int soflow_log_pid = 0;
77 static int soflow_log_proto = 0;
78 static int soflow_nstat_disable = 0;
79 static int soflow_disable = 0;
80 static os_log_t soflow_log_handle = NULL;
81
82 /*
83 * Sysctls for debug logs control
84 */
85 SYSCTL_NODE(_net, OID_AUTO, soflow, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "soflow");
86
87 SYSCTL_INT(_net_soflow, OID_AUTO, log_level, CTLFLAG_RW | CTLFLAG_LOCKED,
88 &soflow_log_level, 0, "");
89
90 SYSCTL_INT(_net_soflow, OID_AUTO, log_port, CTLFLAG_RW | CTLFLAG_LOCKED,
91 &soflow_log_port, 0, "");
92
93 SYSCTL_INT(_net_soflow, OID_AUTO, log_pid, CTLFLAG_RW | CTLFLAG_LOCKED,
94 &soflow_log_pid, 0, "");
95
96 SYSCTL_INT(_net_soflow, OID_AUTO, log_proto, CTLFLAG_RW | CTLFLAG_LOCKED,
97 &soflow_log_proto, 0, "");
98
99 SYSCTL_INT(_net_soflow, OID_AUTO, nstat_disable, CTLFLAG_RW | CTLFLAG_LOCKED,
100 &soflow_nstat_disable, 0, "");
101
102 SYSCTL_INT(_net_soflow, OID_AUTO, disable, CTLFLAG_RW | CTLFLAG_LOCKED,
103 &soflow_disable, 0, "");
104
105 #define SOFLOW_LOG(level, so, debug, fmt, ...) \
106 do { \
107 if (soflow_log_level >= level && debug && soflow_log_handle) { \
108 if (level == LOG_ERR) { \
109 os_log_error(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
110 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
111 } else { \
112 os_log(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
113 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
114 } \
115 } \
116 } while (0)
117
118 #define SOFLOW_ENTRY_LOG(level, so, entry, debug, msg) \
119 do { \
120 if (soflow_log_level >= level && entry && debug) { \
121 soflow_entry_log(level, so, entry, msg); \
122 } \
123 } while (0)
124
125 #define SOFLOW_HASH_SIZE 16
126 #define SOFLOW_HASH(laddr, faddr, lport, fport) ((faddr) ^ ((laddr) >> 16) ^ (fport) ^ (lport))
127
128 #define SOFLOW_IS_UDP(so) (so && so->so_proto && so->so_proto->pr_type == SOCK_DGRAM && so->so_proto->pr_protocol == IPPROTO_UDP)
129 #define SOFLOW_GET_SO_PROTO(so) ((so && so->so_proto) ? so->so_proto->pr_protocol : IPPROTO_MAX)
130
131 #define SOFLOW_SOCKET_PID(so) ((so->so_flags & SOF_DELEGATED) ? so->e_pid : so->last_pid)
132
133 #define SOFLOW_ENABLE_DEBUG(so, entry) \
134 ((soflow_log_port == 0 || !entry || soflow_log_port == ntohs(entry->soflow_lport) || soflow_log_port == ntohs(entry->soflow_fport)) && \
135 (soflow_log_pid == 0 || !so || soflow_log_pid == SOFLOW_SOCKET_PID(so)) && \
136 (soflow_log_proto == 0 || !so || soflow_log_proto == SOFLOW_GET_SO_PROTO(so)))
137
138 os_refgrp_decl(static, soflow_refgrp, "soflow_ref_group", NULL);
139
140 #define SOFLOW_ENTRY_FREE(entry) \
141 if (entry && (os_ref_release(&entry->soflow_ref_count) == 0)) { \
142 soflow_entry_free(entry); \
143 }
144
145 #define SOFLOW_DB_FREE(db) \
146 if (db && (os_ref_release(&db->soflow_db_ref_count) == 0)) { \
147 soflow_db_free(db); \
148 }
149
150 LIST_HEAD(soflow_hash_head, soflow_hash_entry);
151
152 static int soflow_initialized = 0;
153 static int soflow_attached_count = 0;
154
155 TAILQ_HEAD(soflow_entry_head, soflow_hash_entry) soflow_entry_head;
156 static LCK_GRP_DECLARE(soflow_lck_grp, "Socket Flow");
157 static LCK_RW_DECLARE(soflow_lck_rw, &soflow_lck_grp);
158
159 #define SOFLOW_LOCK_EXCLUSIVE lck_rw_lock_exclusive(&soflow_lck_rw)
160 #define SOFLOW_UNLOCK_EXCLUSIVE lck_rw_unlock_exclusive(&soflow_lck_rw)
161 #define SOFLOW_LOCK_SHARED lck_rw_lock_shared(&soflow_lck_rw)
162 #define SOFLOW_UNLOCK_SHARED lck_rw_unlock_shared(&soflow_lck_rw)
163
164 /*
165 * Flow Garbage Collection:
166 */
167 static struct thread *soflow_gc_thread;
168 static soflow_feat_gc_needed_func soflow_feat_gc_needed_func_ptr = NULL;
169 static soflow_feat_gc_perform_func soflow_feat_gc_perform_func_ptr = NULL;
170
171 #define SOFLOW_GC_IDLE_TO 30 // Flow Idle Timeout in seconds
172 #define SOFLOW_GC_MAX_COUNT 100 // Max sockets to be handled per run
173 #define SOFLOW_GC_RUN_INTERVAL_NSEC (10 * NSEC_PER_SEC) // GC wakes up every 10 seconds
174
175 /*
176 * Feature Context Handling:
177 */
178 static soflow_feat_detach_entry_func soflow_feat_detach_entry_func_ptr = NULL;
179 static soflow_feat_detach_db_func soflow_feat_detach_db_func_ptr = NULL;
180
181 static void soflow_gc_thread_func(void *v, wait_result_t w);
182 static void soflow_gc_expire(void *v, wait_result_t w);
183 static boolean_t soflow_entry_local_address_needs_update(struct soflow_hash_entry *);
184 static boolean_t soflow_entry_local_port_needs_update(struct socket *, struct soflow_hash_entry *);
185
186 static void
soflow_init(void)187 soflow_init(void)
188 {
189 if (soflow_initialized) {
190 return;
191 }
192 soflow_initialized = 1;
193
194 if (soflow_log_handle == NULL) {
195 soflow_log_handle = os_log_create("com.apple.xnu.net.soflow", "soflow");
196 }
197
198 TAILQ_INIT(&soflow_entry_head);
199
200 // Spawn thread for gargage collection
201 if (kernel_thread_start(soflow_gc_thread_func, NULL,
202 &soflow_gc_thread) != KERN_SUCCESS) {
203 panic_plain("%s: Can't create SOFLOW GC thread", __func__);
204 /* NOTREACHED */
205 }
206 /* this must not fail */
207 VERIFY(soflow_gc_thread != NULL);
208 }
209
210 static void
soflow_entry_log(int level,struct socket * so,struct soflow_hash_entry * entry,const char * msg)211 soflow_entry_log(int level, struct socket *so, struct soflow_hash_entry *entry, const char* msg)
212 {
213 #pragma unused(level, msg)
214 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
215 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
216 const void *addr;
217
218 // No sock or not UDP, no-op
219 if (entry == NULL) {
220 return;
221 }
222
223 switch (entry->soflow_family) {
224 case AF_INET6:
225 addr = &entry->soflow_laddr.addr6;
226 inet_ntop(AF_INET6, addr, local, sizeof(local));
227 addr = &entry->soflow_faddr.addr6;
228 inet_ntop(AF_INET6, addr, remote, sizeof(local));
229 break;
230 case AF_INET:
231 addr = &entry->soflow_laddr.addr46.ia46_addr4.s_addr;
232 inet_ntop(AF_INET, addr, local, sizeof(local));
233 addr = &entry->soflow_faddr.addr46.ia46_addr4.s_addr;
234 inet_ntop(AF_INET, addr, remote, sizeof(local));
235 break;
236 default:
237 return;
238 }
239
240 SOFLOW_LOG(level, so, entry->soflow_debug, "<%s>: %s <%s(%d) entry %p, featureID %llu> outifp %d lport %d fport %d laddr %s faddr %s hash %X "
241 "<rx p %llu b %llu, tx p %llu b %llu>",
242 msg, entry->soflow_outgoing ? "OUT" : "IN ",
243 SOFLOW_IS_UDP(so) ? "UDP" : "proto", SOFLOW_GET_SO_PROTO(so),
244 entry, entry->soflow_feat_ctxt_id,
245 entry->soflow_outifindex,
246 ntohs(entry->soflow_lport), ntohs(entry->soflow_fport), local, remote,
247 entry->soflow_flowhash,
248 entry->soflow_rxpackets, entry->soflow_rxbytes, entry->soflow_txpackets, entry->soflow_txbytes);
249 }
250
251 bool
soflow_fill_hash_entry_from_address(struct soflow_hash_entry * entry,bool isLocal,struct sockaddr * addr,bool islocalUpdate)252 soflow_fill_hash_entry_from_address(struct soflow_hash_entry *entry, bool isLocal, struct sockaddr *addr, bool islocalUpdate)
253 {
254 struct sockaddr_in *sin = NULL;
255 struct sockaddr_in6 *sin6 = NULL;
256
257 if (entry == NULL || addr == NULL) {
258 return FALSE;
259 }
260
261 switch (addr->sa_family) {
262 case AF_INET:
263 sin = satosin(addr);
264 if (sin->sin_len < sizeof(*sin)) {
265 return FALSE;
266 }
267 if (isLocal == TRUE) {
268 if (sin->sin_port != 0) {
269 entry->soflow_lport = sin->sin_port;
270 if (islocalUpdate) {
271 entry->soflow_lport_updated = TRUE;
272 }
273 }
274 if (sin->sin_addr.s_addr != INADDR_ANY) {
275 entry->soflow_laddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
276 if (islocalUpdate) {
277 entry->soflow_laddr_updated = TRUE;
278 }
279 }
280 } else {
281 if (sin->sin_port != 0) {
282 entry->soflow_fport = sin->sin_port;
283 }
284 if (sin->sin_addr.s_addr != INADDR_ANY) {
285 entry->soflow_faddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
286 }
287 }
288 entry->soflow_family = AF_INET;
289 return TRUE;
290 case AF_INET6:
291 sin6 = satosin6(addr);
292 if (sin6->sin6_len < sizeof(*sin6)) {
293 return FALSE;
294 }
295 if (isLocal == TRUE) {
296 if (sin6->sin6_port != 0) {
297 entry->soflow_lport = sin6->sin6_port;
298 if (islocalUpdate) {
299 entry->soflow_lport_updated = TRUE;
300 }
301 }
302 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
303 entry->soflow_laddr.addr6 = sin6->sin6_addr;
304 entry->soflow_laddr6_ifscope = sin6->sin6_scope_id;
305 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
306 if (islocalUpdate) {
307 entry->soflow_laddr_updated = TRUE;
308 }
309 }
310 } else {
311 if (sin6->sin6_port != 0) {
312 entry->soflow_fport = sin6->sin6_port;
313 }
314 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
315 entry->soflow_faddr.addr6 = sin6->sin6_addr;
316 entry->soflow_faddr6_ifscope = sin6->sin6_scope_id;
317 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
318 }
319 }
320 entry->soflow_family = AF_INET6;
321 return TRUE;
322 default:
323 return FALSE;
324 }
325 }
326
327 bool
soflow_fill_hash_entry_from_inp(struct soflow_hash_entry * entry,bool isLocal,struct inpcb * inp,bool islocalUpdate)328 soflow_fill_hash_entry_from_inp(struct soflow_hash_entry *entry, bool isLocal, struct inpcb *inp, bool islocalUpdate)
329 {
330 if (entry == NULL || inp == NULL) {
331 return FALSE;
332 }
333
334 if (inp->inp_vflag & INP_IPV6) {
335 if (isLocal == TRUE) {
336 if (inp->inp_lport) {
337 entry->soflow_lport = inp->inp_lport;
338 if (islocalUpdate) {
339 entry->soflow_lport_updated = TRUE;
340 }
341 }
342 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
343 entry->soflow_laddr.addr6 = inp->in6p_laddr;
344 entry->soflow_laddr6_ifscope = inp->inp_lifscope;
345 in6_verify_ifscope(&entry->soflow_laddr.addr6, inp->inp_lifscope);
346 if (islocalUpdate) {
347 entry->soflow_laddr_updated = TRUE;
348 }
349 }
350 } else {
351 if (inp->inp_fport) {
352 entry->soflow_fport = inp->inp_fport;
353 }
354 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
355 entry->soflow_faddr.addr6 = inp->in6p_faddr;
356 entry->soflow_faddr6_ifscope = inp->inp_fifscope;
357 in6_verify_ifscope(&entry->soflow_faddr.addr6, inp->inp_fifscope);
358 }
359 }
360 entry->soflow_family = AF_INET6;
361 return TRUE;
362 } else if (inp->inp_vflag & INP_IPV4) {
363 if (isLocal == TRUE) {
364 if (inp->inp_lport) {
365 entry->soflow_lport = inp->inp_lport;
366 if (islocalUpdate) {
367 entry->soflow_lport_updated = TRUE;
368 }
369 }
370 if (inp->inp_laddr.s_addr) {
371 entry->soflow_laddr.addr46.ia46_addr4.s_addr = inp->inp_laddr.s_addr;
372 if (islocalUpdate) {
373 entry->soflow_laddr_updated = TRUE;
374 }
375 }
376 } else {
377 if (inp->inp_fport) {
378 entry->soflow_fport = inp->inp_fport;
379 }
380 if (inp->inp_faddr.s_addr) {
381 entry->soflow_faddr.addr46.ia46_addr4.s_addr = inp->inp_faddr.s_addr;
382 }
383 }
384 entry->soflow_family = AF_INET;
385 return TRUE;
386 }
387 return FALSE;
388 }
389
390 static errno_t
soflow_db_init(struct socket * so)391 soflow_db_init(struct socket *so)
392 {
393 errno_t error = 0;
394 struct soflow_db *db = NULL;
395 struct soflow_hash_entry *hash_entry = NULL;
396
397 db = kalloc_type(struct soflow_db, Z_WAITOK | Z_ZERO | Z_NOFAIL);
398 db->soflow_db_so = so;
399 db->soflow_db_hashbase = hashinit(SOFLOW_HASH_SIZE, M_CFIL, &db->soflow_db_hashmask);
400 if (db->soflow_db_hashbase == NULL) {
401 kfree_type(struct soflow_db, db);
402 error = ENOMEM;
403 goto done;
404 }
405 db->soflow_db_debug = SOFLOW_ENABLE_DEBUG(so, hash_entry);
406 os_ref_init(&db->soflow_db_ref_count, &soflow_refgrp);
407 so->so_flow_db = db;
408 done:
409 return error;
410 }
411
412 static void
soflow_entry_free(struct soflow_hash_entry * hash_entry)413 soflow_entry_free(struct soflow_hash_entry *hash_entry)
414 {
415 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
416
417 if (hash_entry == NULL) {
418 return;
419 }
420
421 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Free entry");
422 kfree_type(struct soflow_hash_entry, hash_entry);
423 }
424
425 static void
soflow_db_remove_entry(struct soflow_db * db,struct soflow_hash_entry * hash_entry)426 soflow_db_remove_entry(struct soflow_db *db, struct soflow_hash_entry *hash_entry)
427 {
428 if (hash_entry == NULL) {
429 return;
430 }
431 if (db == NULL || db->soflow_db_count == 0) {
432 return;
433 }
434
435 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
436 if (hash_entry->soflow_nstat_context != NULL) {
437 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, hash_entry->soflow_debug, "<Close nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
438 nstat_provider_stats_close(hash_entry->soflow_nstat_context);
439 hash_entry->soflow_nstat_context = NULL;
440 SOFLOW_ENTRY_FREE(hash_entry);
441 }
442 #endif
443
444 db->soflow_db_count--;
445 if (db->soflow_db_only_entry == hash_entry) {
446 db->soflow_db_only_entry = NULL;
447 }
448 LIST_REMOVE(hash_entry, soflow_entry_link);
449
450 // Feature context present, give feature a chance to detach and clean up
451 if (hash_entry->soflow_feat_ctxt != NULL && soflow_feat_detach_entry_func_ptr != NULL) {
452 soflow_feat_detach_entry_func_ptr(db->soflow_db_so, hash_entry);
453 hash_entry->soflow_feat_ctxt = NULL;
454 hash_entry->soflow_feat_ctxt_id = 0;
455 }
456
457 hash_entry->soflow_db = NULL;
458
459 SOFLOW_LOCK_EXCLUSIVE;
460 if (soflow_initialized) {
461 TAILQ_REMOVE(&soflow_entry_head, hash_entry, soflow_entry_list_link);
462 soflow_attached_count--;
463 }
464 SOFLOW_UNLOCK_EXCLUSIVE;
465
466 SOFLOW_ENTRY_FREE(hash_entry);
467 }
468
469 static void
soflow_db_free(struct soflow_db * db)470 soflow_db_free(struct soflow_db *db)
471 {
472 struct soflow_hash_entry *entry = NULL;
473 struct soflow_hash_entry *temp_entry = NULL;
474 struct soflow_hash_head *flowhash = NULL;
475
476 if (db == NULL) {
477 return;
478 }
479
480 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "<db %p> freeing db (count == %d)", db, db->soflow_db_count);
481
482 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
483 flowhash = &db->soflow_db_hashbase[i];
484 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
485 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Remove entry");
486 soflow_db_remove_entry(db, entry);
487 }
488 }
489
490 if (soflow_feat_detach_db_func_ptr != NULL) {
491 soflow_feat_detach_db_func_ptr(db->soflow_db_so, db);
492 }
493
494 // Make sure all entries are cleaned up!
495 VERIFY(db->soflow_db_count == 0);
496 hashdestroy(db->soflow_db_hashbase, M_CFIL, db->soflow_db_hashmask);
497 kfree_type(struct soflow_db, db);
498 }
499
500 void
soflow_detach(struct socket * so)501 soflow_detach(struct socket *so)
502 {
503 if (so == NULL || so->so_flow_db == NULL) {
504 return;
505 }
506 SOFLOW_DB_FREE(so->so_flow_db);
507 so->so_flow_db = NULL;
508 }
509
510 static boolean_t
soflow_match_entries_v4(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)511 soflow_match_entries_v4(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
512 {
513 if (entry1 == NULL || entry2 == NULL) {
514 return false;
515 }
516
517 // Ignore local match if remoteOnly or if local has been updated since entry added
518 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
519 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
520 entry1->soflow_laddr.addr46.ia46_addr4.s_addr == entry2->soflow_laddr.addr46.ia46_addr4.s_addr);
521
522 // Entries match if local and remote ports and addresses all matched
523 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
524 laddr_matched && entry1->soflow_faddr.addr46.ia46_addr4.s_addr == entry2->soflow_faddr.addr46.ia46_addr4.s_addr;
525 }
526
527 static boolean_t
soflow_match_entries_v6(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)528 soflow_match_entries_v6(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
529 {
530 if (entry1 == NULL || entry2 == NULL) {
531 return false;
532 }
533
534 // Ignore local match if remoteOnly or if local has been updated since entry added
535 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
536 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
537 in6_are_addr_equal_scoped(&entry1->soflow_laddr.addr6, &entry2->soflow_laddr.addr6, entry1->soflow_laddr6_ifscope, entry2->soflow_laddr6_ifscope));
538
539 // Entries match if local and remote ports and addresses all matched
540 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
541 laddr_matched && in6_are_addr_equal_scoped(&entry1->soflow_faddr.addr6, &entry2->soflow_faddr.addr6, entry1->soflow_faddr6_ifscope, entry2->soflow_faddr6_ifscope);
542 }
543
544 static struct soflow_hash_entry *
soflow_db_lookup_entry_internal(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly,boolean_t withLocalPort)545 soflow_db_lookup_entry_internal(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly, boolean_t withLocalPort)
546 {
547 struct soflow_hash_entry matchentry = { };
548 struct soflow_hash_entry *nextentry = NULL;
549 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
550 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
551 u_int16_t hashkey_fport = 0, hashkey_lport = 0;
552 int inp_hash_element = 0;
553 struct soflow_hash_head *flowhash = NULL;
554
555 if (inp == NULL || db == NULL) {
556 return NULL;
557 }
558
559 if (local != NULL) {
560 soflow_fill_hash_entry_from_address(&matchentry, TRUE, local, FALSE);
561 } else {
562 soflow_fill_hash_entry_from_inp(&matchentry, TRUE, inp, FALSE);
563 }
564 if (remote != NULL) {
565 soflow_fill_hash_entry_from_address(&matchentry, FALSE, remote, FALSE);
566 } else {
567 soflow_fill_hash_entry_from_inp(&matchentry, FALSE, inp, FALSE);
568 }
569 matchentry.soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, (&matchentry));
570 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, true, "Looking for entry");
571
572 if (inp->inp_vflag & INP_IPV6) {
573 hashkey_faddr = matchentry.soflow_faddr.addr6.s6_addr32[3];
574 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr6.s6_addr32[3] : 0;
575 } else {
576 hashkey_faddr = matchentry.soflow_faddr.addr46.ia46_addr4.s_addr;
577 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr46.ia46_addr4.s_addr : 0;
578 }
579
580 hashkey_fport = matchentry.soflow_fport;
581 hashkey_lport = (remoteOnly == false || withLocalPort == true) ? matchentry.soflow_lport : 0;
582
583 inp_hash_element = SOFLOW_HASH(hashkey_laddr, hashkey_faddr, hashkey_lport, hashkey_fport);
584 inp_hash_element &= db->soflow_db_hashmask;
585 flowhash = &db->soflow_db_hashbase[inp_hash_element];
586
587 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
588 if (inp->inp_vflag & INP_IPV6) {
589 if (soflow_match_entries_v6(nextentry, &matchentry, remoteOnly)) {
590 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v6");
591 break;
592 }
593 } else if (inp->inp_vflag & INP_IPV4) {
594 if (soflow_match_entries_v4(nextentry, &matchentry, remoteOnly)) {
595 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v4");
596 break;
597 }
598 }
599 }
600
601 if (nextentry == NULL) {
602 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, matchentry.soflow_debug, "Entry not found");
603 }
604 return nextentry;
605 }
606
607 static struct soflow_hash_entry *
soflow_db_lookup_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly)608 soflow_db_lookup_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly)
609 {
610 struct soflow_hash_entry *entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, false);
611 if (entry == NULL && remoteOnly == true) {
612 entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, true);
613 }
614 return entry;
615 }
616
617 static struct soflow_hash_entry *
soflow_db_lookup_by_feature_context_id(struct soflow_db * db,u_int64_t feature_context_id)618 soflow_db_lookup_by_feature_context_id(struct soflow_db *db, u_int64_t feature_context_id)
619 {
620 struct soflow_hash_head *flowhash = NULL;
621 u_int32_t inp_hash_element = (u_int32_t)(feature_context_id & 0x0ffffffff);
622 struct soflow_hash_entry *nextentry;
623
624 inp_hash_element &= db->soflow_db_hashmask;
625 flowhash = &db->soflow_db_hashbase[inp_hash_element];
626
627 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
628 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Looking at entry");
629 if (nextentry->soflow_feat_ctxt != NULL &&
630 nextentry->soflow_feat_ctxt_id == feature_context_id) {
631 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry by feature context id");
632 break;
633 }
634 }
635
636 if (nextentry == NULL) {
637 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "No entry found for featureID %llu <count %d hash %X %X>",
638 feature_context_id, db->soflow_db_count, inp_hash_element, (u_int32_t)(feature_context_id & 0x0ffffffff));
639 }
640 return nextentry;
641 }
642
643 void *
soflow_db_get_feature_context(struct soflow_db * db,u_int64_t feature_context_id)644 soflow_db_get_feature_context(struct soflow_db *db, u_int64_t feature_context_id)
645 {
646 struct soflow_hash_entry *hash_entry = NULL;
647 void *context = NULL;
648
649 if (db == NULL || db->soflow_db_so == NULL || feature_context_id == 0) {
650 return NULL;
651 }
652
653 socket_lock_assert_owned(db->soflow_db_so);
654
655 // Take refcount of db before use.
656 // Abort if db is already being freed.
657 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
658 return NULL;
659 }
660
661 // This is an optimization for datagram sockets with only one single flow.
662 if (db->soflow_db_count == 1) {
663 if (db->soflow_db_only_entry != NULL &&
664 db->soflow_db_only_entry->soflow_feat_ctxt != NULL && db->soflow_db_only_entry->soflow_feat_ctxt_id == feature_context_id) {
665 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_only_entry, db->soflow_db_only_entry->soflow_debug, "MATCHED only entry for featureID");
666 context = db->soflow_db_only_entry->soflow_feat_ctxt;
667 } else {
668 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "MISMATCHED only entry for featureID %llu (entry %p - cfil %p id %llu)",
669 feature_context_id,
670 db->soflow_db_only_entry,
671 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt : NULL,
672 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt_id : 0);
673 }
674 } else {
675 hash_entry = soflow_db_lookup_by_feature_context_id(db, feature_context_id);
676 context = hash_entry != NULL ? hash_entry->soflow_feat_ctxt : NULL;
677 }
678
679 SOFLOW_DB_FREE(db);
680 return context;
681 }
682
683 u_int64_t
soflow_db_get_feature_context_id(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)684 soflow_db_get_feature_context_id(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
685 {
686 struct soflow_hash_entry *hash_entry = NULL;
687 uint64_t context_id = 0;
688
689 if (db == NULL || db->soflow_db_so == NULL) {
690 return 0;
691 }
692
693 socket_lock_assert_owned(db->soflow_db_so);
694
695 // Take refcount of db before use.
696 // Abort if db is already being freed.
697 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
698 return 0;
699 }
700
701 hash_entry = soflow_db_lookup_entry(db, local, remote, false);
702 if (hash_entry == NULL) {
703 // No match with both local and remote, try match with remote only
704 hash_entry = soflow_db_lookup_entry(db, local, remote, true);
705 }
706 if (hash_entry != NULL && hash_entry->soflow_feat_ctxt != NULL) {
707 context_id = hash_entry->soflow_feat_ctxt_id;
708 }
709
710 SOFLOW_DB_FREE(db);
711
712 return context_id;
713 }
714
715 static struct soflow_hash_entry *
soflow_db_add_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)716 soflow_db_add_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
717 {
718 struct soflow_hash_entry *entry = NULL;
719 struct inpcb *inp = db ? sotoinpcb(db->soflow_db_so) : NULL;
720 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
721 int inp_hash_element = 0;
722 struct soflow_hash_head *flowhash = NULL;
723
724 if (db == NULL || inp == NULL) {
725 goto done;
726 }
727
728 entry = kalloc_type(struct soflow_hash_entry, Z_WAITOK | Z_ZERO | Z_NOFAIL);
729 os_ref_init(&entry->soflow_ref_count, &soflow_refgrp);
730
731 if (local != NULL) {
732 soflow_fill_hash_entry_from_address(entry, TRUE, local, FALSE);
733 } else {
734 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, FALSE);
735 }
736 if (remote != NULL) {
737 soflow_fill_hash_entry_from_address(entry, FALSE, remote, FALSE);
738 } else {
739 soflow_fill_hash_entry_from_inp(entry, FALSE, inp, FALSE);
740 }
741 entry->soflow_lastused = net_uptime();
742 entry->soflow_db = db;
743 entry->soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, entry);
744
745 if (inp->inp_vflag & INP_IPV6) {
746 hashkey_faddr = entry->soflow_faddr.addr6.s6_addr32[3];
747 hashkey_laddr = entry->soflow_laddr.addr6.s6_addr32[3];
748 } else {
749 hashkey_faddr = entry->soflow_faddr.addr46.ia46_addr4.s_addr;
750 hashkey_laddr = entry->soflow_laddr.addr46.ia46_addr4.s_addr;
751 }
752 entry->soflow_flowhash = SOFLOW_HASH(hashkey_laddr, hashkey_faddr,
753 entry->soflow_lport, entry->soflow_fport);
754 inp_hash_element = entry->soflow_flowhash & db->soflow_db_hashmask;
755
756 socket_lock_assert_owned(db->soflow_db_so);
757
758 // Take refcount of db before use.
759 // Abort if db is already being freed.
760 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
761 return NULL;
762 }
763
764 flowhash = &db->soflow_db_hashbase[inp_hash_element];
765
766 LIST_INSERT_HEAD(flowhash, entry, soflow_entry_link);
767 db->soflow_db_count++;
768 db->soflow_db_only_entry = entry;
769 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "total count %d", db->soflow_db_count);
770
771 SOFLOW_DB_FREE(db);
772
773 done:
774 return entry;
775 }
776
777 static int
soflow_udp_get_address_from_control(sa_family_t family,struct mbuf * control,uint8_t ** address_ptr)778 soflow_udp_get_address_from_control(sa_family_t family, struct mbuf *control, uint8_t **address_ptr)
779 {
780 struct cmsghdr *cm;
781 struct in6_pktinfo *pi6;
782 struct socket *so = NULL;
783
784 if (control == NULL || address_ptr == NULL) {
785 return 0;
786 }
787
788 for (; control != NULL; control = control->m_next) {
789 if (control->m_type != MT_CONTROL) {
790 continue;
791 }
792
793 for (cm = M_FIRST_CMSGHDR(control);
794 is_cmsg_valid(control, cm);
795 cm = M_NXT_CMSGHDR(control, cm)) {
796 SOFLOW_LOG(LOG_DEBUG, so, true, "Check control type %d", cm->cmsg_type);
797
798 switch (cm->cmsg_type) {
799 case IP_RECVDSTADDR:
800 if (family == AF_INET &&
801 cm->cmsg_level == IPPROTO_IP &&
802 cm->cmsg_len == CMSG_LEN(sizeof(struct in_addr))) {
803 *address_ptr = CMSG_DATA(cm);
804 return sizeof(struct in_addr);
805 }
806 break;
807 case IPV6_PKTINFO:
808 case IPV6_2292PKTINFO:
809 if (family == AF_INET6 &&
810 cm->cmsg_level == IPPROTO_IPV6 &&
811 cm->cmsg_len == CMSG_LEN(sizeof(struct in6_pktinfo))) {
812 pi6 = (struct in6_pktinfo *)(void *)CMSG_DATA(cm);
813 *address_ptr = (uint8_t *)&pi6->ipi6_addr;
814 return sizeof(struct in6_addr);
815 }
816 break;
817 default:
818 break;
819 }
820 }
821 }
822 return 0;
823 }
824
825 static boolean_t
soflow_entry_local_address_needs_update(struct soflow_hash_entry * entry)826 soflow_entry_local_address_needs_update(struct soflow_hash_entry *entry)
827 {
828 if (entry->soflow_family == AF_INET6) {
829 return IN6_IS_ADDR_UNSPECIFIED(&entry->soflow_laddr.addr6);
830 } else if (entry->soflow_family == AF_INET) {
831 return entry->soflow_laddr.addr46.ia46_addr4.s_addr == INADDR_ANY;
832 }
833 return false;
834 }
835
836 static boolean_t
soflow_entry_local_port_needs_update(struct socket * so,struct soflow_hash_entry * entry)837 soflow_entry_local_port_needs_update(struct socket *so, struct soflow_hash_entry *entry)
838 {
839 if (SOFLOW_IS_UDP(so)) {
840 return entry->soflow_lport == 0;
841 }
842 return false;
843 }
844
845 static void
soflow_entry_update_local(struct soflow_db * db,struct soflow_hash_entry * entry,struct sockaddr * local,struct mbuf * control,u_short rcv_ifindex)846 soflow_entry_update_local(struct soflow_db *db, struct soflow_hash_entry *entry, struct sockaddr *local, struct mbuf *control, u_short rcv_ifindex)
847 {
848 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
849 union sockaddr_in_4_6 address_buf = { };
850
851 if (inp == NULL || entry == NULL) {
852 return;
853 }
854
855 if (entry->soflow_outifindex == 0 && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
856 entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
857 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated outifp");
858 }
859
860 if (soflow_entry_local_address_needs_update(entry)) {
861 // Flow does not have a local address yet. Retrieve local address
862 // from control mbufs if present.
863 if (local == NULL && control != NULL) {
864 uint8_t *addr_ptr = NULL;
865 int size = soflow_udp_get_address_from_control(entry->soflow_family, control, &addr_ptr);
866
867 if (size && addr_ptr) {
868 switch (entry->soflow_family) {
869 case AF_INET:
870 if (size == sizeof(struct in_addr)) {
871 address_buf.sin.sin_port = 0;
872 address_buf.sin.sin_family = AF_INET;
873 address_buf.sin.sin_len = sizeof(struct sockaddr_in);
874 (void) memcpy(&address_buf.sin.sin_addr, addr_ptr, sizeof(struct in_addr));
875 local = sintosa(&address_buf.sin);
876 }
877 break;
878 case AF_INET6:
879 if (size == sizeof(struct in6_addr)) {
880 address_buf.sin6.sin6_port = 0;
881 address_buf.sin6.sin6_family = AF_INET6;
882 address_buf.sin6.sin6_len = sizeof(struct sockaddr_in6);
883 (void) memcpy(&address_buf.sin6.sin6_addr, addr_ptr, sizeof(struct in6_addr));
884 local = sin6tosa(&address_buf.sin6);
885 }
886 break;
887 default:
888 break;
889 }
890 }
891 }
892 if (local != NULL) {
893 soflow_fill_hash_entry_from_address(entry, TRUE, local, TRUE);
894 } else {
895 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
896 }
897 if (entry->soflow_laddr_updated) {
898 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated address");
899 }
900 }
901
902 if (soflow_entry_local_port_needs_update(db->soflow_db_so, entry)) {
903 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
904 if (entry->soflow_lport_updated) {
905 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated port");
906 }
907 }
908
909 return;
910 }
911
912 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
913 static u_int32_t
ifnet_to_flags(struct ifnet * ifp,struct socket * so)914 ifnet_to_flags(struct ifnet *ifp, struct socket *so)
915 {
916 u_int32_t flags = 0;
917
918 if (ifp != NULL) {
919 flags = nstat_ifnet_to_flags(ifp);
920 if ((flags & NSTAT_IFNET_IS_WIFI) && ((flags & (NSTAT_IFNET_IS_AWDL | NSTAT_IFNET_IS_LLW)) == 0)) {
921 flags |= NSTAT_IFNET_IS_WIFI_INFRA;
922 }
923 } else {
924 flags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
925 }
926
927 if (so != NULL && (so->so_flags1 & SOF1_CELLFALLBACK)) {
928 flags |= NSTAT_IFNET_VIA_CELLFALLBACK;
929 }
930 return flags;
931 }
932
933 static bool
soflow_nstat_provider_request_vals(nstat_provider_context ctx,u_int32_t * ifflagsp,nstat_counts * countsp,void * metadatap)934 soflow_nstat_provider_request_vals(nstat_provider_context ctx,
935 u_int32_t *ifflagsp,
936 nstat_counts *countsp,
937 void *metadatap)
938 {
939 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
940 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
941 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
942 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
943 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
944 const void *addr = NULL;
945
946 if (hash_entry == NULL || so == NULL || inp == NULL) {
947 return false;
948 }
949
950 if (ifflagsp) {
951 if (hash_entry->soflow_outifindex) {
952 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
953 *ifflagsp = ifnet_to_flags(ifp, so);
954 }
955 if ((countsp == NULL) && (metadatap == NULL)) {
956 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "ifflagsp set to 0x%X", *ifflagsp);
957 goto done;
958 }
959 }
960
961 if (countsp) {
962 bzero(countsp, sizeof(*countsp));
963 countsp->nstat_rxpackets = hash_entry->soflow_rxpackets;
964 countsp->nstat_rxbytes = hash_entry->soflow_rxbytes;
965 countsp->nstat_txpackets = hash_entry->soflow_txpackets;
966 countsp->nstat_txbytes = hash_entry->soflow_txbytes;
967
968 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
969 "Collected NSTAT counts: rxpackets %llu rxbytes %llu txpackets %llu txbytes %llu",
970 countsp->nstat_rxpackets, countsp->nstat_rxbytes, countsp->nstat_txpackets, countsp->nstat_txbytes);
971 }
972
973 if (metadatap) {
974 nstat_udp_descriptor *desc = (nstat_udp_descriptor *)metadatap;
975 bzero(desc, sizeof(*desc));
976
977 if (so->so_flags & SOF_DELEGATED) {
978 desc->eupid = so->e_upid;
979 desc->epid = so->e_pid;
980 uuid_copy(desc->euuid, so->e_uuid);
981 } else {
982 desc->eupid = so->last_upid;
983 desc->epid = so->last_pid;
984 uuid_copy(desc->euuid, so->last_uuid);
985 }
986
987 uuid_copy(desc->vuuid, so->so_vuuid);
988 uuid_copy(desc->fuuid, hash_entry->soflow_uuid);
989
990 if (hash_entry->soflow_family == AF_INET6) {
991 in6_ip6_to_sockaddr(&hash_entry->soflow_laddr.addr6, hash_entry->soflow_lport, hash_entry->soflow_laddr6_ifscope,
992 &desc->local.v6, sizeof(desc->local.v6));
993 in6_ip6_to_sockaddr(&hash_entry->soflow_faddr.addr6, hash_entry->soflow_fport, hash_entry->soflow_faddr6_ifscope,
994 &desc->remote.v6, sizeof(desc->remote.v6));
995 } else if (hash_entry->soflow_family == AF_INET) {
996 desc->local.v4.sin_family = AF_INET;
997 desc->local.v4.sin_len = sizeof(struct sockaddr_in);
998 desc->local.v4.sin_port = hash_entry->soflow_lport;
999 desc->local.v4.sin_addr = hash_entry->soflow_laddr.addr46.ia46_addr4;
1000
1001 desc->remote.v4.sin_family = AF_INET;
1002 desc->remote.v4.sin_len = sizeof(struct sockaddr_in);
1003 desc->remote.v4.sin_port = hash_entry->soflow_fport;
1004 desc->remote.v4.sin_addr = hash_entry->soflow_faddr.addr46.ia46_addr4;
1005 }
1006
1007 desc->ifindex = hash_entry->soflow_outifindex;
1008 if (hash_entry->soflow_outifindex) {
1009 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
1010 desc->ifnet_properties = (uint16_t)ifnet_to_flags(ifp, so);
1011 }
1012
1013 desc->rcvbufsize = so->so_rcv.sb_hiwat;
1014 desc->rcvbufused = so->so_rcv.sb_cc;
1015 desc->traffic_class = so->so_traffic_class;
1016 inp_get_activity_bitmap(inp, &desc->activity_bitmap);
1017
1018 if (hash_entry->soflow_debug) {
1019 switch (hash_entry->soflow_family) {
1020 case AF_INET6:
1021 addr = &desc->local.v6;
1022 inet_ntop(AF_INET6, addr, local, sizeof(local));
1023 addr = &desc->remote.v6;
1024 inet_ntop(AF_INET6, addr, remote, sizeof(local));
1025 break;
1026 case AF_INET:
1027 addr = &desc->local.v4.sin_addr;
1028 inet_ntop(AF_INET, addr, local, sizeof(local));
1029 addr = &desc->remote.v4.sin_addr;
1030 inet_ntop(AF_INET, addr, remote, sizeof(local));
1031 break;
1032 default:
1033 break;
1034 }
1035
1036 uint8_t *ptr = (uint8_t *)&desc->euuid;
1037
1038 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
1039 "Collected NSTAT metadata: eupid %llu epid %d euuid %x%x%x%x-%x%x%x%x-%x%x%x%x-%x%x%x%x "
1040 "outifp %d properties 0x%X lport %d fport %d laddr %s faddr %s "
1041 "rcvbufsize %u rcvbufused %u traffic_class %u",
1042 desc->eupid, desc->epid,
1043 ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7],
1044 ptr[8], ptr[9], ptr[10], ptr[11], ptr[12], ptr[13], ptr[14], ptr[15],
1045 desc->ifindex, desc->ifnet_properties,
1046 ntohs(desc->local.v4.sin_port), ntohs(desc->remote.v4.sin_port), local, remote,
1047 desc->rcvbufsize, desc->rcvbufused, desc->traffic_class);
1048 }
1049 }
1050 done:
1051 return true;
1052 }
1053
1054 static size_t
soflow_nstat_provider_request_extensions(nstat_provider_context ctx,int requested_extension,void * buf,size_t buf_size)1055 soflow_nstat_provider_request_extensions(nstat_provider_context ctx,
1056 int requested_extension,
1057 void *buf,
1058 size_t buf_size)
1059 {
1060 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
1061 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1062 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
1063 struct nstat_domain_info *domain_info = NULL;
1064 size_t size = 0;
1065
1066 if (hash_entry == NULL || so == NULL || inp == NULL) {
1067 return 0;
1068 }
1069
1070 if (buf == NULL) {
1071 switch (requested_extension) {
1072 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1073 return sizeof(nstat_domain_info);
1074 default:
1075 return 0;
1076 }
1077 }
1078
1079 if (buf_size < sizeof(nstat_domain_info)) {
1080 return 0;
1081 }
1082
1083 switch (requested_extension) {
1084 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1085
1086 domain_info = (struct nstat_domain_info *)buf;
1087 necp_copy_inp_domain_info(inp, so, domain_info);
1088
1089 if (hash_entry->soflow_debug) {
1090 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "Collected NSTAT domain_info:pid %d domain <%s> owner <%s> "
1091 "ctxt <%s> bundle id <%s> is_tracker %d is_non_app_initiated %d is_silent %d",
1092 so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid,
1093 domain_info->domain_name,
1094 domain_info->domain_owner,
1095 domain_info->domain_tracker_ctxt,
1096 domain_info->domain_attributed_bundle_id,
1097 domain_info->is_tracker,
1098 domain_info->is_non_app_initiated,
1099 domain_info->is_silent);
1100 }
1101 size = sizeof(nstat_domain_info);
1102
1103 default:
1104 break;
1105 }
1106
1107 return size;
1108 }
1109 #endif
1110
1111 static void
soflow_update_flow_stats(struct soflow_hash_entry * hash_entry,size_t data_size,bool outgoing)1112 soflow_update_flow_stats(struct soflow_hash_entry *hash_entry, size_t data_size, bool outgoing)
1113 {
1114 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1115
1116 if (hash_entry != NULL) {
1117 if (outgoing) {
1118 hash_entry->soflow_txbytes += data_size;
1119 hash_entry->soflow_txpackets++;
1120 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Outgoing");
1121 } else {
1122 hash_entry->soflow_rxbytes += data_size;
1123 hash_entry->soflow_rxpackets++;
1124 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Incoming");
1125 }
1126 }
1127 }
1128
1129 struct soflow_hash_entry *
soflow_get_flow(struct socket * so,struct sockaddr * local,struct sockaddr * remote,struct mbuf * control,size_t data_size,bool outgoing,uint16_t rcv_ifindex)1130 soflow_get_flow(struct socket *so, struct sockaddr *local, struct sockaddr *remote, struct mbuf *control,
1131 size_t data_size, bool outgoing, uint16_t rcv_ifindex)
1132 {
1133 struct soflow_hash_entry *hash_entry = NULL;
1134 struct inpcb *inp = sotoinpcb(so);
1135
1136 // Check if feature is disabled
1137 if (soflow_disable) {
1138 return NULL;
1139 }
1140
1141 socket_lock_assert_owned(so);
1142
1143 if (so->so_flow_db != NULL) {
1144 // Take refcount of db before use.
1145 // Abort if db is already being freed.
1146 if (os_ref_retain_try(&so->so_flow_db->soflow_db_ref_count) == false) {
1147 return NULL;
1148 }
1149
1150 // DB already exists, check if this is existing flow
1151 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, false);
1152 if (hash_entry == NULL) {
1153 // No match with both local and remote, try match with remote only
1154 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, true);
1155 }
1156 if (hash_entry != NULL) {
1157 // Take refcount of entry before use.
1158 // Abort if entry is already being freed.
1159 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1160 SOFLOW_DB_FREE(so->so_flow_db);
1161 return NULL;
1162 }
1163
1164 // Try to update flow info from socket and/or control mbufs if necessary
1165 if (hash_entry->soflow_outifindex == 0 ||
1166 soflow_entry_local_address_needs_update(hash_entry) || soflow_entry_local_port_needs_update(so, hash_entry)) {
1167 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1168 }
1169 hash_entry->soflow_lastused = net_uptime();
1170 soflow_update_flow_stats(hash_entry, data_size, outgoing);
1171
1172 SOFLOW_DB_FREE(so->so_flow_db);
1173 return hash_entry;
1174 }
1175
1176 SOFLOW_DB_FREE(so->so_flow_db);
1177 } else {
1178 // If new socket, allocate cfil db
1179 if (soflow_db_init(so) != 0) {
1180 return NULL;
1181 }
1182 }
1183
1184 hash_entry = soflow_db_add_entry(so->so_flow_db, local, remote);
1185 if (hash_entry == NULL) {
1186 SOFLOW_LOG(LOG_ERR, so, true, "Failed to add entry");
1187 return NULL;
1188 }
1189
1190 // Take refcount of entry before use.
1191 // Abort if entry is already being freed.
1192 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1193 return NULL;
1194 }
1195
1196 if (inp && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
1197 hash_entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
1198 }
1199
1200 // Check if we can update the new flow's local address from control mbufs
1201 if (control != NULL) {
1202 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1203 }
1204 hash_entry->soflow_outgoing = outgoing;
1205 soflow_update_flow_stats(hash_entry, data_size, outgoing);
1206
1207 // Only report flow to NSTAT if unconnected UDP
1208 if (!soflow_nstat_disable && SOFLOW_IS_UDP(so) && !(so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING))) {
1209 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
1210 // Take refcount of entry before handing it to nstat. Abort if fail.
1211 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1212 return NULL;
1213 }
1214 uuid_generate_random(hash_entry->soflow_uuid);
1215 hash_entry->soflow_nstat_context = nstat_provider_stats_open((nstat_provider_context) hash_entry,
1216 NSTAT_PROVIDER_UDP_SUBFLOW, 0,
1217 soflow_nstat_provider_request_vals,
1218 soflow_nstat_provider_request_extensions);
1219 SOFLOW_LOG(LOG_INFO, so, hash_entry->soflow_debug, "<Open nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
1220 #endif
1221 }
1222
1223 SOFLOW_LOCK_EXCLUSIVE;
1224 if (soflow_initialized == 0) {
1225 soflow_init();
1226 }
1227 TAILQ_INSERT_TAIL(&soflow_entry_head, hash_entry, soflow_entry_list_link);
1228 if (soflow_attached_count == 0) {
1229 thread_wakeup((caddr_t)&soflow_attached_count);
1230 }
1231 soflow_attached_count++;
1232 SOFLOW_UNLOCK_EXCLUSIVE;
1233
1234 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Added entry");
1235 return hash_entry;
1236 }
1237
1238 void
soflow_free_flow(struct soflow_hash_entry * entry)1239 soflow_free_flow(struct soflow_hash_entry *entry)
1240 {
1241 SOFLOW_ENTRY_FREE(entry);
1242 }
1243
1244 static bool
soflow_socket_safe_lock(struct inpcb * inp,struct inpcbinfo * pcbinfo)1245 soflow_socket_safe_lock(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1246 {
1247 struct socket *so = NULL;
1248
1249 VERIFY(pcbinfo != NULL);
1250
1251 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
1252 // Safeguarded the inp state, unlock pcbinfo before locking socket.
1253 lck_rw_done(&pcbinfo->ipi_lock);
1254
1255 so = inp->inp_socket;
1256 socket_lock(so, 1);
1257 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) != WNT_STOPUSING) {
1258 return true;
1259 }
1260 } else {
1261 // Failed to safeguarded the inp state, unlock pcbinfo and abort.
1262 lck_rw_done(&pcbinfo->ipi_lock);
1263 }
1264
1265 if (so) {
1266 socket_unlock(so, 1);
1267 }
1268 return false;
1269 }
1270
1271 static struct socket *
soflow_validate_dgram_socket(struct socket * so)1272 soflow_validate_dgram_socket(struct socket *so)
1273 {
1274 struct inpcb *inp = NULL;
1275 struct inpcbinfo *pcbinfo = NULL;
1276 struct socket *locked = NULL;
1277
1278 pcbinfo = &udbinfo;
1279 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1280 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1281 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1282 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1283 locked = inp->inp_socket;
1284 }
1285 /* pcbinfo is already unlocked, we are done. */
1286 goto done;
1287 }
1288 }
1289 lck_rw_done(&pcbinfo->ipi_lock);
1290 if (locked != NULL) {
1291 goto done;
1292 }
1293
1294 pcbinfo = &ripcbinfo;
1295 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1296 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1297 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1298 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1299 locked = inp->inp_socket;
1300 }
1301 /* pcbinfo is already unlocked, we are done. */
1302 goto done;
1303 }
1304 }
1305 lck_rw_done(&pcbinfo->ipi_lock);
1306
1307 done:
1308 return locked;
1309 }
1310
1311 static void
soflow_gc_thread_sleep(bool forever)1312 soflow_gc_thread_sleep(bool forever)
1313 {
1314 if (forever) {
1315 (void) assert_wait((event_t) &soflow_attached_count,
1316 THREAD_INTERRUPTIBLE);
1317 } else {
1318 uint64_t deadline = 0;
1319 nanoseconds_to_absolutetime(SOFLOW_GC_RUN_INTERVAL_NSEC, &deadline);
1320 clock_absolutetime_interval_to_deadline(deadline, &deadline);
1321
1322 (void) assert_wait_deadline(&soflow_attached_count,
1323 THREAD_INTERRUPTIBLE, deadline);
1324 }
1325 }
1326
1327 static void
soflow_gc_thread_func(void * v,wait_result_t w)1328 soflow_gc_thread_func(void *v, wait_result_t w)
1329 {
1330 #pragma unused(v, w)
1331
1332 ASSERT(soflow_gc_thread == current_thread());
1333 thread_set_thread_name(current_thread(), "SOFLOW_GC");
1334
1335 // Kick off gc shortly
1336 soflow_gc_thread_sleep(false);
1337 thread_block_parameter((thread_continue_t) soflow_gc_expire, NULL);
1338 /* NOTREACHED */
1339 }
1340
1341 static bool
soflow_gc_idle_timed_out(struct soflow_hash_entry * hash_entry,int timeout,u_int64_t current_time)1342 soflow_gc_idle_timed_out(struct soflow_hash_entry *hash_entry, int timeout, u_int64_t current_time)
1343 {
1344 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1345
1346 if (hash_entry && (current_time - hash_entry->soflow_lastused >= (u_int64_t)timeout)) {
1347 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "GC Idle Timeout detected");
1348 return true;
1349 }
1350 return false;
1351 }
1352
1353 static int
soflow_gc_cleanup(struct socket * so)1354 soflow_gc_cleanup(struct socket *so)
1355 {
1356 struct soflow_hash_entry *entry = NULL;
1357 struct soflow_hash_entry *temp_entry = NULL;
1358 struct soflow_hash_head *flowhash = NULL;
1359 struct soflow_db *db = NULL;
1360 int cleaned = 0;
1361
1362 if (so == NULL || so->so_flow_db == NULL) {
1363 return 0;
1364 }
1365 db = so->so_flow_db;
1366
1367 socket_lock_assert_owned(so);
1368
1369 // Take refcount of db before use.
1370 // Abort if db is already being freed.
1371 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1372 return 0;
1373 }
1374
1375 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1376 flowhash = &db->soflow_db_hashbase[i];
1377 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1378 if (entry->soflow_gc || entry->soflow_feat_gc) {
1379 if (entry->soflow_feat_ctxt != NULL && soflow_feat_gc_perform_func_ptr != NULL) {
1380 soflow_feat_gc_perform_func_ptr(so, entry);
1381 entry->soflow_feat_ctxt = NULL;
1382 entry->soflow_feat_ctxt_id = 0;
1383 }
1384 entry->soflow_feat_gc = 0;
1385
1386 if (entry->soflow_gc) {
1387 SOFLOW_ENTRY_LOG(LOG_INFO, so, entry, entry->soflow_debug, "GC cleanup entry");
1388 entry->soflow_gc = 0;
1389 soflow_db_remove_entry(db, entry);
1390 cleaned++;
1391 }
1392 }
1393 }
1394 }
1395
1396 SOFLOW_DB_FREE(db);
1397 return cleaned;
1398 }
1399
1400 static void
soflow_gc_expire(void * v,wait_result_t w)1401 soflow_gc_expire(void *v, wait_result_t w)
1402 {
1403 #pragma unused(v, w)
1404
1405 static struct socket *socket_array[SOFLOW_GC_MAX_COUNT];
1406 struct soflow_hash_entry *hash_entry = NULL;
1407 struct socket *so = NULL;
1408 u_int64_t current_time = net_uptime();
1409 uint32_t socket_count = 0;
1410 uint32_t cleaned_count = 0;
1411 bool recorded = false;
1412
1413 // Collect a list of socket with expired flows
1414
1415 SOFLOW_LOCK_SHARED;
1416
1417 if (soflow_attached_count == 0) {
1418 SOFLOW_UNLOCK_SHARED;
1419 goto go_sleep;
1420 }
1421
1422 // Go thorough all flows in the flow list and record any socket with expired flows.
1423 TAILQ_FOREACH(hash_entry, &soflow_entry_head, soflow_entry_list_link) {
1424 if (socket_count >= SOFLOW_GC_MAX_COUNT) {
1425 break;
1426 }
1427 so = hash_entry->soflow_db ? hash_entry->soflow_db->soflow_db_so : NULL;
1428
1429 // Check if we need to perform cleanup due to idle time or feature specified rules
1430 hash_entry->soflow_gc = soflow_gc_idle_timed_out(hash_entry, SOFLOW_GC_IDLE_TO, current_time);
1431 hash_entry->soflow_feat_gc = (soflow_feat_gc_needed_func_ptr != NULL && soflow_feat_gc_needed_func_ptr(so, hash_entry, current_time));
1432
1433 if (hash_entry->soflow_gc || hash_entry->soflow_feat_gc) {
1434 if (so != NULL) {
1435 recorded = false;
1436 for (int i = 0; i < socket_count; i++) {
1437 if (socket_array[socket_count] == so) {
1438 recorded = true;
1439 break;
1440 }
1441 }
1442 if (recorded == false) {
1443 socket_array[socket_count] = so;
1444 socket_count++;
1445 }
1446 }
1447 }
1448 }
1449 SOFLOW_UNLOCK_SHARED;
1450
1451 if (socket_count == 0) {
1452 goto go_sleep;
1453 }
1454
1455 for (uint32_t i = 0; i < socket_count; i++) {
1456 // Validate socket and lock it
1457 so = soflow_validate_dgram_socket(socket_array[i]);
1458 if (so == NULL) {
1459 continue;
1460 }
1461 cleaned_count += soflow_gc_cleanup(so);
1462 socket_unlock(so, 1);
1463 }
1464
1465 so = NULL;
1466 SOFLOW_LOG(LOG_INFO, so, true, "<GC cleaned %d flows>", cleaned_count);
1467
1468 go_sleep:
1469
1470 // Sleep forever (until waken up) if no more UDP flow to clean
1471 SOFLOW_LOCK_SHARED;
1472 soflow_gc_thread_sleep(soflow_attached_count == 0 ? true : false);
1473 SOFLOW_UNLOCK_SHARED;
1474 thread_block_parameter((thread_continue_t)soflow_gc_expire, NULL);
1475 /* NOTREACHED */
1476 }
1477
1478 void
soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,soflow_feat_gc_perform_func gc_perform_fn,soflow_feat_detach_entry_func feat_detach_entry_fn,soflow_feat_detach_db_func feat_detach_db_fn)1479 soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,
1480 soflow_feat_gc_perform_func gc_perform_fn,
1481 soflow_feat_detach_entry_func feat_detach_entry_fn,
1482 soflow_feat_detach_db_func feat_detach_db_fn)
1483 {
1484 soflow_feat_gc_needed_func_ptr = gc_needed_fn;
1485 soflow_feat_gc_perform_func_ptr = gc_perform_fn;
1486 soflow_feat_detach_entry_func_ptr = feat_detach_entry_fn;
1487 soflow_feat_detach_db_func_ptr = feat_detach_db_fn;
1488 }
1489
1490 bool
soflow_db_apply(struct soflow_db * db,soflow_entry_apply_func entry_apply_fn,void * context)1491 soflow_db_apply(struct soflow_db *db, soflow_entry_apply_func entry_apply_fn, void *context)
1492 {
1493 struct soflow_hash_entry *entry = NULL;
1494 struct soflow_hash_entry *temp_entry = NULL;
1495 struct soflow_hash_head *flowhash = NULL;
1496
1497 if (db == NULL || db->soflow_db_so == NULL || entry_apply_fn == NULL) {
1498 return false;
1499 }
1500
1501 socket_lock_assert_owned(db->soflow_db_so);
1502
1503 // Take refcount of db before use.
1504 // Abort if db is already being freed.
1505 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1506 return false;
1507 }
1508
1509 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1510 flowhash = &db->soflow_db_hashbase[i];
1511 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1512 if (entry_apply_fn(db->soflow_db_so, entry, context) == false) {
1513 goto done;
1514 }
1515 }
1516 }
1517
1518 done:
1519 SOFLOW_DB_FREE(db);
1520 return true;
1521 }
1522