1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30
31 #include <dev/random/randomdev.h>
32 #include <net/flowhash.h>
33 #include <netkey/key.h>
34
35 #include <skywalk/nexus/flowswitch/fsw_var.h>
36 #include <skywalk/nexus/flowswitch/flow/flow_var.h>
37 #include <skywalk/nexus/netif/nx_netif.h>
38 #include <skywalk/namespace/flowidns.h>
39
40 struct flow_entry *fe_alloc(boolean_t);
41 static void fe_free(struct flow_entry *);
42 static int fe_id_cmp(const struct flow_entry *, const struct flow_entry *);
43 static void fe_stats_init(struct flow_entry *);
44 static void fe_stats_update(struct flow_entry *);
45
46 RB_GENERATE_PREV(flow_entry_id_tree, flow_entry, fe_id_link, fe_id_cmp);
47
48 os_refgrp_decl(static, flow_entry_refgrp, "flow_entry", NULL);
49
50 static SKMEM_TYPE_DEFINE(sk_fed_zone, struct flow_entry_dead);
51
52 const struct flow_key fk_mask_2tuple
53 __sk_aligned(16) =
54 {
55 .fk_mask = FKMASK_2TUPLE,
56 .fk_ipver = 0,
57 .fk_proto = 0xff,
58 .fk_sport = 0xffff,
59 .fk_dport = 0,
60 .fk_src._addr64[0] = 0,
61 .fk_src._addr64[1] = 0,
62 .fk_dst._addr64[0] = 0,
63 .fk_dst._addr64[1] = 0,
64 .fk_pad[0] = 0,
65 };
66
67 const struct flow_key fk_mask_3tuple
68 __sk_aligned(16) =
69 {
70 .fk_mask = FKMASK_3TUPLE,
71 .fk_ipver = 0xff,
72 .fk_proto = 0xff,
73 .fk_sport = 0xffff,
74 .fk_dport = 0,
75 .fk_src._addr64[0] = 0xffffffffffffffffULL,
76 .fk_src._addr64[1] = 0xffffffffffffffffULL,
77 .fk_dst._addr64[0] = 0,
78 .fk_dst._addr64[1] = 0,
79 .fk_pad[0] = 0,
80 };
81
82 const struct flow_key fk_mask_4tuple
83 __sk_aligned(16) =
84 {
85 .fk_mask = FKMASK_4TUPLE,
86 .fk_ipver = 0xff,
87 .fk_proto = 0xff,
88 .fk_sport = 0xffff,
89 .fk_dport = 0xffff,
90 .fk_src._addr64[0] = 0xffffffffffffffffULL,
91 .fk_src._addr64[1] = 0xffffffffffffffffULL,
92 .fk_dst._addr64[0] = 0,
93 .fk_dst._addr64[1] = 0,
94 .fk_pad[0] = 0,
95 };
96
97 const struct flow_key fk_mask_5tuple
98 __sk_aligned(16) =
99 {
100 .fk_mask = FKMASK_5TUPLE,
101 .fk_ipver = 0xff,
102 .fk_proto = 0xff,
103 .fk_sport = 0xffff,
104 .fk_dport = 0xffff,
105 .fk_src._addr64[0] = 0xffffffffffffffffULL,
106 .fk_src._addr64[1] = 0xffffffffffffffffULL,
107 .fk_dst._addr64[0] = 0xffffffffffffffffULL,
108 .fk_dst._addr64[1] = 0xffffffffffffffffULL,
109 .fk_pad[0] = 0,
110 };
111
112 const struct flow_key fk_mask_ipflow1
113 __sk_aligned(16) =
114 {
115 .fk_mask = FKMASK_IPFLOW1,
116 .fk_ipver = 0,
117 .fk_proto = 0xff,
118 .fk_sport = 0,
119 .fk_dport = 0,
120 .fk_src._addr64[0] = 0,
121 .fk_src._addr64[1] = 0,
122 .fk_dst._addr64[0] = 0,
123 .fk_dst._addr64[1] = 0,
124 .fk_pad[0] = 0,
125 };
126
127 const struct flow_key fk_mask_ipflow2
128 __sk_aligned(16) =
129 {
130 .fk_mask = FKMASK_IPFLOW2,
131 .fk_ipver = 0xff,
132 .fk_proto = 0xff,
133 .fk_sport = 0,
134 .fk_dport = 0,
135 .fk_src._addr64[0] = 0xffffffffffffffffULL,
136 .fk_src._addr64[1] = 0xffffffffffffffffULL,
137 .fk_dst._addr64[0] = 0,
138 .fk_dst._addr64[1] = 0,
139 .fk_pad[0] = 0,
140 };
141
142 const struct flow_key fk_mask_ipflow3
143 __sk_aligned(16) =
144 {
145 .fk_mask = FKMASK_IPFLOW3,
146 .fk_ipver = 0xff,
147 .fk_proto = 0xff,
148 .fk_sport = 0,
149 .fk_dport = 0,
150 .fk_src._addr64[0] = 0xffffffffffffffffULL,
151 .fk_src._addr64[1] = 0xffffffffffffffffULL,
152 .fk_dst._addr64[0] = 0xffffffffffffffffULL,
153 .fk_dst._addr64[1] = 0xffffffffffffffffULL,
154 .fk_pad[0] = 0,
155 };
156
157 struct flow_owner *
flow_owner_find_by_pid(struct flow_owner_bucket * fob,pid_t pid,void * context,bool low_latency)158 flow_owner_find_by_pid(struct flow_owner_bucket *fob, pid_t pid, void *context,
159 bool low_latency)
160 {
161 struct flow_owner find = { .fo_context = context, .fo_pid = pid,
162 .fo_low_latency = low_latency};
163
164 ASSERT(low_latency == true || low_latency == false);
165 FOB_LOCK_ASSERT_HELD(fob);
166 return RB_FIND(flow_owner_tree, &fob->fob_owner_head, &find);
167 }
168
169 struct flow_entry *
flow_entry_find_by_uuid(struct flow_owner * fo,uuid_t uuid)170 flow_entry_find_by_uuid(struct flow_owner *fo, uuid_t uuid)
171 {
172 struct flow_entry find, *fe = NULL;
173 FOB_LOCK_ASSERT_HELD(FO_BUCKET(fo));
174
175 uuid_copy(find.fe_uuid, uuid);
176 fe = RB_FIND(flow_entry_id_tree, &fo->fo_flow_entry_id_head, &find);
177 if (fe != NULL) {
178 flow_entry_retain(fe);
179 }
180
181 return fe;
182 }
183
184 static uint32_t
flow_entry_calc_flowid(struct flow_entry * fe)185 flow_entry_calc_flowid(struct flow_entry *fe)
186 {
187 uint32_t flowid;
188 struct flowidns_flow_key fk;
189
190 bzero(&fk, sizeof(fk));
191 _CASSERT(sizeof(fe->fe_key.fk_src) == sizeof(fk.ffk_laddr));
192 _CASSERT(sizeof(fe->fe_key.fk_dst) == sizeof(fk.ffk_raddr));
193 bcopy(&fe->fe_key.fk_src, &fk.ffk_laddr, sizeof(fk.ffk_laddr));
194 bcopy(&fe->fe_key.fk_dst, &fk.ffk_raddr, sizeof(fk.ffk_raddr));
195
196 fk.ffk_lport = fe->fe_key.fk_sport;
197 fk.ffk_rport = fe->fe_key.fk_dport;
198 fk.ffk_af = (fe->fe_key.fk_ipver == 4) ? AF_INET : AF_INET6;
199 fk.ffk_proto = fe->fe_key.fk_proto;
200
201 flowidns_allocate_flowid(FLOWIDNS_DOMAIN_FLOWSWITCH, &fk, &flowid);
202 return flowid;
203 }
204
205 static bool
flow_entry_add_child(struct flow_entry * parent_fe,struct flow_entry * child_fe)206 flow_entry_add_child(struct flow_entry *parent_fe, struct flow_entry *child_fe)
207 {
208 SK_LOG_VAR(char dbgbuf[FLOWENTRY_DBGBUF_SIZE]);
209 ASSERT(parent_fe->fe_flags & FLOWENTF_PARENT);
210
211 lck_rw_lock_exclusive(&parent_fe->fe_child_list_lock);
212
213 if (parent_fe->fe_flags & FLOWENTF_NONVIABLE) {
214 SK_ERR("child entry add failed, parent fe \"%s\" non viable 0x%llx "
215 "flags 0x%b %s(%d)", fe_as_string(parent_fe,
216 dbgbuf, sizeof(dbgbuf)), SK_KVA(parent_fe), parent_fe->fe_flags,
217 FLOWENTF_BITS, parent_fe->fe_proc_name,
218 parent_fe->fe_pid);
219 lck_rw_unlock_exclusive(&parent_fe->fe_child_list_lock);
220 return false;
221 }
222
223 struct flow_entry *__single fe, *__single tfe;
224 TAILQ_FOREACH_SAFE(fe, &parent_fe->fe_child_list, fe_child_link, tfe) {
225 if (!fe_id_cmp(fe, child_fe)) {
226 lck_rw_unlock_exclusive(&parent_fe->fe_child_list_lock);
227 SK_ERR("child entry \"%s\" already exists at fe 0x%llx "
228 "flags 0x%b %s(%d)", fe_as_string(fe,
229 dbgbuf, sizeof(dbgbuf)), SK_KVA(fe), fe->fe_flags,
230 FLOWENTF_BITS, fe->fe_proc_name,
231 fe->fe_pid);
232 return false;
233 }
234
235 if (fe->fe_flags & FLOWENTF_NONVIABLE) {
236 TAILQ_REMOVE(&parent_fe->fe_child_list, fe, fe_child_link);
237 ASSERT(--parent_fe->fe_child_count >= 0);
238 flow_entry_release(&fe);
239 }
240 }
241
242 flow_entry_retain(child_fe);
243 TAILQ_INSERT_TAIL(&parent_fe->fe_child_list, child_fe, fe_child_link);
244 ASSERT(++parent_fe->fe_child_count > 0);
245
246 lck_rw_unlock_exclusive(&parent_fe->fe_child_list_lock);
247
248 return true;
249 }
250
251 static void
flow_entry_remove_all_children(struct flow_entry * parent_fe,struct nx_flowswitch * fsw)252 flow_entry_remove_all_children(struct flow_entry *parent_fe, struct nx_flowswitch *fsw)
253 {
254 bool sched_reaper_thread = false;
255
256 ASSERT(parent_fe->fe_flags & FLOWENTF_PARENT);
257
258 lck_rw_lock_exclusive(&parent_fe->fe_child_list_lock);
259
260 struct flow_entry *__single fe, *__single tfe;
261 TAILQ_FOREACH_SAFE(fe, &parent_fe->fe_child_list, fe_child_link, tfe) {
262 if (!(fe->fe_flags & FLOWENTF_NONVIABLE)) {
263 /*
264 * fsw_pending_nonviable is a hint for reaper thread;
265 * due to the fact that setting fe_want_nonviable and
266 * incrementing fsw_pending_nonviable counter is not
267 * atomic, let the increment happen first, and the
268 * thread losing the CAS does decrement.
269 */
270 os_atomic_inc(&fsw->fsw_pending_nonviable, relaxed);
271 if (os_atomic_cmpxchg(&fe->fe_want_nonviable, 0, 1, acq_rel)) {
272 sched_reaper_thread = true;
273 } else {
274 os_atomic_dec(&fsw->fsw_pending_nonviable, relaxed);
275 }
276 }
277
278 TAILQ_REMOVE(&parent_fe->fe_child_list, fe, fe_child_link);
279 ASSERT(--parent_fe->fe_child_count >= 0);
280 flow_entry_release(&fe);
281 }
282
283 lck_rw_unlock_exclusive(&parent_fe->fe_child_list_lock);
284
285 if (sched_reaper_thread) {
286 fsw_reap_sched(fsw);
287 }
288 }
289
290 static void
flow_entry_set_demux_patterns(struct flow_entry * fe,struct nx_flow_req * req)291 flow_entry_set_demux_patterns(struct flow_entry *fe, struct nx_flow_req *req)
292 {
293 ASSERT(fe->fe_flags & FLOWENTF_CHILD);
294 ASSERT(req->nfr_flow_demux_count > 0);
295
296 fe->fe_demux_patterns = sk_alloc_type_array(struct kern_flow_demux_pattern, req->nfr_flow_demux_count,
297 Z_WAITOK | Z_NOFAIL, skmem_tag_flow_demux);
298 fe->fe_demux_pattern_count = req->nfr_flow_demux_count;
299
300 for (int i = 0; i < req->nfr_flow_demux_count; i++) {
301 bcopy(&req->nfr_flow_demux_patterns[i], &fe->fe_demux_patterns[i].fdp_demux_pattern,
302 sizeof(struct flow_demux_pattern));
303
304 fe->fe_demux_patterns[i].fdp_memcmp_mask = NULL;
305 if (req->nfr_flow_demux_patterns[i].fdp_len == 16) {
306 fe->fe_demux_patterns[i].fdp_memcmp_mask = sk_memcmp_mask_16B;
307 } else if (req->nfr_flow_demux_patterns[i].fdp_len == 32) {
308 fe->fe_demux_patterns[i].fdp_memcmp_mask = sk_memcmp_mask_32B;
309 } else if (req->nfr_flow_demux_patterns[i].fdp_len > 32) {
310 VERIFY(0);
311 }
312 }
313 }
314
315 static int
convert_flowkey_to_inet_td(struct flow_key * key,struct ifnet_traffic_descriptor_inet * td)316 convert_flowkey_to_inet_td(struct flow_key *key,
317 struct ifnet_traffic_descriptor_inet *td)
318 {
319 if ((key->fk_mask & FKMASK_IPVER) != 0) {
320 td->inet_ipver = key->fk_ipver;
321 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_IPVER;
322 }
323 if ((key->fk_mask & FKMASK_PROTO) != 0) {
324 td->inet_proto = key->fk_proto;
325 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_PROTO;
326 }
327 if ((key->fk_mask & FKMASK_SRC) != 0) {
328 if (td->inet_ipver == IPVERSION) {
329 bcopy(&key->fk_src4, &td->inet_laddr.iia_v4addr,
330 sizeof(key->fk_src4));
331 } else {
332 bcopy(&key->fk_src6, &td->inet_laddr,
333 sizeof(key->fk_src6));
334 }
335 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_LADDR;
336 }
337 if ((key->fk_mask & FKMASK_DST) != 0) {
338 if (td->inet_ipver == IPVERSION) {
339 bcopy(&key->fk_dst4, &td->inet_raddr.iia_v4addr,
340 sizeof(key->fk_dst4));
341 } else {
342 bcopy(&key->fk_dst6, &td->inet_raddr,
343 sizeof(key->fk_dst6));
344 }
345 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_RADDR;
346 }
347 if ((key->fk_mask & FKMASK_SPORT) != 0) {
348 td->inet_lport = key->fk_sport;
349 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_LPORT;
350 }
351 if ((key->fk_mask & FKMASK_DPORT) != 0) {
352 td->inet_rport = key->fk_dport;
353 td->inet_mask |= IFNET_TRAFFIC_DESCRIPTOR_INET_RPORT;
354 }
355 td->inet_common.itd_type = IFNET_TRAFFIC_DESCRIPTOR_TYPE_INET;
356 td->inet_common.itd_len = sizeof(*td);
357 td->inet_common.itd_flags = IFNET_TRAFFIC_DESCRIPTOR_FLAG_INBOUND |
358 IFNET_TRAFFIC_DESCRIPTOR_FLAG_OUTBOUND;
359 return 0;
360 }
361
362 void
flow_qset_select_dynamic(struct nx_flowswitch * fsw,struct flow_entry * fe,boolean_t skip_if_no_change)363 flow_qset_select_dynamic(struct nx_flowswitch *fsw, struct flow_entry *fe,
364 boolean_t skip_if_no_change)
365 {
366 struct ifnet_traffic_descriptor_inet td;
367 struct ifnet *ifp;
368 uint64_t qset_id;
369 struct nx_netif *nif;
370 boolean_t changed;
371 int err;
372
373 ifp = fsw->fsw_ifp;
374 changed = ifnet_sync_traffic_rule_genid(ifp, &fe->fe_tr_genid);
375 if (!changed && skip_if_no_change) {
376 return;
377 }
378 if (fe->fe_qset != NULL) {
379 nx_netif_qset_release(&fe->fe_qset);
380 ASSERT(fe->fe_qset == NULL);
381 }
382 if (ifp->if_traffic_rule_count == 0) {
383 DTRACE_SKYWALK2(no__rules, struct nx_flowswitch *, fsw,
384 struct flow_entry *, fe);
385 return;
386 }
387 err = convert_flowkey_to_inet_td(&fe->fe_key, &td);
388 ASSERT(err == 0);
389 err = nxctl_inet_traffic_rule_find_qset_id(ifp->if_xname, &td, &qset_id);
390 if (err != 0) {
391 DTRACE_SKYWALK3(qset__id__not__found,
392 struct nx_flowswitch *, fsw,
393 struct flow_entry *, fe,
394 struct ifnet_traffic_descriptor_inet *, &td);
395 return;
396 }
397 DTRACE_SKYWALK4(qset__id__found, struct nx_flowswitch *, fsw,
398 struct flow_entry *, fe, struct ifnet_traffic_descriptor_inet *,
399 &td, uint64_t, qset_id);
400 nif = NX_NETIF_PRIVATE(fsw->fsw_dev_ch->ch_na->na_nx);
401 ASSERT(fe->fe_qset == NULL);
402 fe->fe_qset = nx_netif_find_qset(nif, qset_id);
403 }
404
405 /* writer-lock must be owned for memory management functions */
406 struct flow_entry *
flow_entry_alloc(struct flow_owner * fo,struct nx_flow_req * req,int * perr)407 flow_entry_alloc(struct flow_owner *fo, struct nx_flow_req *req, int *perr)
408 {
409 SK_LOG_VAR(char dbgbuf[FLOWENTRY_DBGBUF_SIZE]);
410 nexus_port_t nx_port = req->nfr_nx_port;
411 struct flow_entry *__single fe = NULL;
412 struct flow_entry *__single parent_fe = NULL;
413 flowadv_idx_t fadv_idx = FLOWADV_IDX_NONE;
414 struct nexus_adapter *dev_na;
415 struct nx_netif *nif;
416 int err;
417
418 FOB_LOCK_ASSERT_HELD(FO_BUCKET(fo));
419 ASSERT(nx_port != NEXUS_PORT_ANY);
420 ASSERT(!fo->fo_nx_port_destroyed);
421
422 *perr = 0;
423
424 struct flow_key key __sk_aligned(16);
425 err = flow_req2key(req, &key);
426 if (__improbable(err != 0)) {
427 SK_ERR("invalid request (err %d)", err);
428 goto done;
429 }
430
431 struct flow_mgr *fm = fo->fo_fsw->fsw_flow_mgr;
432 fe = flow_mgr_find_conflicting_fe(fm, &key);
433 if (fe != NULL) {
434 if ((fe->fe_flags & FLOWENTF_PARENT) &&
435 uuid_compare(fe->fe_uuid, req->nfr_parent_flow_uuid) == 0) {
436 parent_fe = fe;
437 fe = NULL;
438 } else {
439 SK_ERR("entry \"%s\" already exists at fe 0x%llx "
440 "flags 0x%b %s(%d)", fe_as_string(fe,
441 dbgbuf, sizeof(dbgbuf)), SK_KVA(fe), fe->fe_flags,
442 FLOWENTF_BITS, fe->fe_proc_name,
443 fe->fe_pid);
444 /* don't return it */
445 flow_entry_release(&fe);
446 err = EEXIST;
447 goto done;
448 }
449 } else if (!uuid_is_null(req->nfr_parent_flow_uuid)) {
450 uuid_string_t uuid_str;
451 sk_uuid_unparse(req->nfr_parent_flow_uuid, uuid_str);
452 SK_ERR("parent entry \"%s\" does not exist", uuid_str);
453 err = ENOENT;
454 goto done;
455 }
456
457 if ((req->nfr_flags & NXFLOWREQF_FLOWADV) &&
458 (flow_owner_flowadv_index_alloc(fo, &fadv_idx) != 0)) {
459 SK_ERR("failed to alloc flowadv index for flow %s",
460 sk_uuid_unparse(req->nfr_flow_uuid, dbgbuf));
461 err = ENOMEM;
462 goto done;
463 }
464
465 fe = fe_alloc(TRUE);
466 if (__improbable(fe == NULL)) {
467 err = ENOMEM;
468 goto done;
469 }
470
471 fe->fe_key = key;
472 if (req->nfr_route != NULL) {
473 fe->fe_laddr_gencnt = req->nfr_route->fr_laddr_gencnt;
474 } else {
475 fe->fe_laddr_gencnt = req->nfr_saddr_gencnt;
476 }
477
478 if (__improbable(req->nfr_flags & NXFLOWREQF_LISTENER)) {
479 /* mark this as listener mode */
480 os_atomic_or(&fe->fe_flags, FLOWENTF_LISTENER, relaxed);
481 } else {
482 ASSERT((fe->fe_key.fk_ipver == IPVERSION &&
483 fe->fe_key.fk_src4.s_addr != INADDR_ANY) ||
484 (fe->fe_key.fk_ipver == IPV6_VERSION &&
485 !IN6_IS_ADDR_UNSPECIFIED(&fe->fe_key.fk_src6)));
486
487 /* mark this as connected mode */
488 os_atomic_or(&fe->fe_flags, FLOWENTF_CONNECTED, relaxed);
489 }
490
491 if (req->nfr_flags & NXFLOWREQF_NOWAKEFROMSLEEP) {
492 fe->fe_flags |= FLOWENTF_NOWAKEFROMSLEEP;
493 }
494 fe->fe_port_reservation = req->nfr_port_reservation;
495 req->nfr_port_reservation = NULL;
496 if (req->nfr_flags & NXFLOWREQF_EXT_PORT_RSV) {
497 fe->fe_flags |= FLOWENTF_EXTRL_PORT;
498 }
499 fe->fe_proto_reservation = req->nfr_proto_reservation;
500 req->nfr_proto_reservation = NULL;
501 if (req->nfr_flags & NXFLOWREQF_EXT_PROTO_RSV) {
502 fe->fe_flags |= FLOWENTF_EXTRL_PROTO;
503 }
504 fe->fe_ipsec_reservation = req->nfr_ipsec_reservation;
505 req->nfr_ipsec_reservation = NULL;
506
507 fe->fe_tx_process = dp_flow_tx_process;
508 fe->fe_rx_process = dp_flow_rx_process;
509
510 dev_na = fo->fo_fsw->fsw_dev_ch->ch_na;
511 nif = NX_NETIF_PRIVATE(dev_na->na_nx);
512 if (NX_LLINK_PROV(nif->nif_nx) &&
513 (fe->fe_key.fk_mask & (FKMASK_IPVER | FKMASK_PROTO | FKMASK_DST)) ==
514 (FKMASK_IPVER | FKMASK_PROTO | FKMASK_DST)) {
515 if (req->nfr_qset_id != 0) {
516 fe->fe_qset_select = FE_QSET_SELECT_FIXED;
517 fe->fe_qset_id = req->nfr_qset_id;
518 fe->fe_qset = nx_netif_find_qset(nif, req->nfr_qset_id);
519 } else {
520 fe->fe_qset_select = FE_QSET_SELECT_DYNAMIC;
521 fe->fe_qset_id = 0;
522 flow_qset_select_dynamic(fo->fo_fsw, fe, FALSE);
523 }
524 } else {
525 fe->fe_qset_select = FE_QSET_SELECT_NONE;
526 }
527 if (req->nfr_flags & NXFLOWREQF_LOW_LATENCY) {
528 os_atomic_or(&fe->fe_flags, FLOWENTF_LOW_LATENCY, relaxed);
529 }
530
531 fe->fe_transport_protocol = req->nfr_transport_protocol;
532 if (NX_FSW_TCP_RX_AGG_ENABLED() &&
533 (fo->fo_fsw->fsw_nx->nx_prov->nxprov_params->nxp_max_frags > 1) &&
534 (fe->fe_key.fk_proto == IPPROTO_TCP) &&
535 (fe->fe_key.fk_mask == FKMASK_5TUPLE)) {
536 fe->fe_rx_process = flow_rx_agg_tcp;
537 }
538 uuid_copy(fe->fe_uuid, req->nfr_flow_uuid);
539 if ((req->nfr_flags & NXFLOWREQF_LISTENER) == 0 &&
540 (req->nfr_flags & NXFLOWREQF_TRACK) != 0) {
541 switch (req->nfr_ip_protocol) {
542 case IPPROTO_TCP:
543 case IPPROTO_UDP:
544 os_atomic_or(&fe->fe_flags, FLOWENTF_TRACK, relaxed);
545 break;
546 default:
547 break;
548 }
549 }
550
551 if (req->nfr_flags & NXFLOWREQF_QOS_MARKING) {
552 os_atomic_or(&fe->fe_flags, FLOWENTF_QOS_MARKING, relaxed);
553 }
554
555 if (req->nfr_flags & NXFLOWREQF_PARENT) {
556 os_atomic_or(&fe->fe_flags, FLOWENTF_PARENT, relaxed);
557 TAILQ_INIT(&fe->fe_child_list);
558 lck_rw_init(&fe->fe_child_list_lock, &nexus_lock_group, &nexus_lock_attr);
559 }
560
561 if (req->nfr_route != NULL) {
562 fe->fe_route = req->nfr_route;
563 req->nfr_route = NULL;
564 }
565
566 fe->fe_nx_port = nx_port;
567 fe->fe_adv_idx = fadv_idx;
568
569 if (req->nfr_inp_flowhash != 0) {
570 /*
571 * BSD flow, use the inpcb flow hash value
572 */
573 fe->fe_flowid = req->nfr_inp_flowhash;
574 fe->fe_flags |= FLOWENTF_EXTRL_FLOWID;
575 } else {
576 fe->fe_flowid = flow_entry_calc_flowid(fe);
577 }
578
579 if (fe->fe_adv_idx != FLOWADV_IDX_NONE && fo->fo_nx_port_na != NULL) {
580 na_flowadv_entry_alloc(fo->fo_nx_port_na, fe->fe_uuid,
581 fe->fe_adv_idx, fe->fe_flowid);
582 }
583
584 if (KPKT_VALID_SVC(req->nfr_svc_class)) {
585 fe->fe_svc_class = (kern_packet_svc_class_t)req->nfr_svc_class;
586 } else {
587 fe->fe_svc_class = KPKT_SC_BE;
588 }
589
590 uuid_copy(fe->fe_eproc_uuid, req->nfr_euuid);
591 fe->fe_policy_id = req->nfr_policy_id;
592 fe->fe_skip_policy_id = req->nfr_skip_policy_id;
593
594 err = flow_mgr_flow_hash_mask_add(fm, fe->fe_key.fk_mask);
595 ASSERT(err == 0);
596
597 if (parent_fe != NULL) {
598 os_atomic_or(&fe->fe_flags, FLOWENTF_CHILD, relaxed);
599 flow_entry_set_demux_patterns(fe, req);
600 fe->fe_demux_pkt_data = sk_alloc_data(FLOW_DEMUX_MAX_LEN, Z_WAITOK | Z_NOFAIL, skmem_tag_flow_demux);
601 if (!flow_entry_add_child(parent_fe, fe)) {
602 goto done;
603 }
604 } else {
605 fe->fe_key_hash = flow_key_hash(&fe->fe_key);
606 err = cuckoo_hashtable_add_with_hash(fm->fm_flow_table, &fe->fe_cnode,
607 fe->fe_key_hash);
608 if (err != 0) {
609 SK_ERR("flow table add failed (err %d)", err);
610 flow_mgr_flow_hash_mask_del(fm, fe->fe_key.fk_mask);
611 goto done;
612 }
613 }
614
615 RB_INSERT(flow_entry_id_tree, &fo->fo_flow_entry_id_head, fe);
616 flow_entry_retain(fe); /* one refcnt in id_tree */
617
618 *(struct nx_flowswitch **)(uintptr_t)&fe->fe_fsw = fo->fo_fsw;
619 fe->fe_pid = fo->fo_pid;
620 if (req->nfr_epid != -1 && req->nfr_epid != fo->fo_pid) {
621 fe->fe_epid = req->nfr_epid;
622 proc_name(fe->fe_epid, fe->fe_eproc_name,
623 sizeof(fe->fe_eproc_name));
624 } else {
625 fe->fe_epid = -1;
626 }
627
628 (void) snprintf(fe->fe_proc_name, sizeof(fe->fe_proc_name), "%s",
629 fo->fo_name);
630
631 fe_stats_init(fe);
632 flow_stats_retain(fe->fe_stats);
633 req->nfr_flow_stats = fe->fe_stats;
634 fe->fe_rx_worker_tid = 0;
635
636 #if SK_LOG
637 SK_DF(SK_VERB_FLOW, "allocated entry \"%s\" fe 0x%llx flags 0x%b "
638 "[fo 0x%llx ]", fe_as_string(fe, dbgbuf,
639 sizeof(dbgbuf)), SK_KVA(fe), fe->fe_flags, FLOWENTF_BITS,
640 SK_KVA(fo));
641 #endif /* SK_LOG */
642
643 done:
644 if (parent_fe != NULL) {
645 flow_entry_release(&parent_fe);
646 }
647 if (err != 0) {
648 if (fadv_idx != FLOWADV_IDX_NONE) {
649 flow_owner_flowadv_index_free(fo, fadv_idx);
650 }
651 if (fe != NULL) {
652 flow_entry_release(&fe);
653 }
654 }
655 *perr = err;
656 return fe;
657 }
658
659 void
flow_entry_teardown(struct flow_owner * fo,struct flow_entry * fe)660 flow_entry_teardown(struct flow_owner *fo, struct flow_entry *fe)
661 {
662 #if SK_LOG
663 char dbgbuf[FLOWENTRY_DBGBUF_SIZE];
664 SK_DF(SK_VERB_FLOW, "entry \"%s\" fe 0x%llx flags 0x%b [fo 0x%llx] "
665 "non_via %d withdrawn %d", fe_as_string(fe, dbgbuf, sizeof(dbgbuf)),
666 SK_KVA(fe), fe->fe_flags, FLOWENTF_BITS, SK_KVA(fo),
667 fe->fe_want_nonviable, fe->fe_want_withdraw);
668 #endif /* SK_LOG */
669 struct nx_flowswitch *fsw = fo->fo_fsw;
670
671 FOB_LOCK_ASSERT_HELD(FO_BUCKET(fo));
672
673 ASSERT(!(fe->fe_flags & FLOWENTF_DESTROYED));
674 ASSERT(!(fe->fe_flags & FLOWENTF_LINGERING));
675 ASSERT(fsw != NULL);
676
677 if (os_atomic_cmpxchg(&fe->fe_want_nonviable, 1, 0, acq_rel)) {
678 ASSERT(fsw->fsw_pending_nonviable != 0);
679 os_atomic_dec(&fsw->fsw_pending_nonviable, relaxed);
680 os_atomic_or(&fe->fe_flags, FLOWENTF_NONVIABLE, relaxed);
681 }
682
683 /* always withdraw namespace during tear down */
684 if (!(fe->fe_flags & FLOWENTF_EXTRL_PORT) &&
685 !(fe->fe_flags & FLOWENTF_WITHDRAWN)) {
686 os_atomic_or(&fe->fe_flags, FLOWENTF_WITHDRAWN, relaxed);
687 os_atomic_store(&fe->fe_want_withdraw, 0, release);
688 /* local port is now inactive; not eligible for offload */
689 flow_namespace_withdraw(&fe->fe_port_reservation);
690 }
691
692 /* we may get here multiple times, so check */
693 if (!(fe->fe_flags & FLOWENTF_TORN_DOWN)) {
694 os_atomic_or(&fe->fe_flags, FLOWENTF_TORN_DOWN, relaxed);
695 if (fe->fe_adv_idx != FLOWADV_IDX_NONE) {
696 if (fo->fo_nx_port_na != NULL) {
697 na_flowadv_entry_free(fo->fo_nx_port_na,
698 fe->fe_uuid, fe->fe_adv_idx, fe->fe_flowid);
699 }
700 flow_owner_flowadv_index_free(fo, fe->fe_adv_idx);
701 fe->fe_adv_idx = FLOWADV_IDX_NONE;
702 }
703 }
704 ASSERT(fe->fe_adv_idx == FLOWADV_IDX_NONE);
705 ASSERT(fe->fe_flags & FLOWENTF_TORN_DOWN);
706
707 /* mark child flow as nonviable */
708 if (fe->fe_flags & FLOWENTF_PARENT) {
709 flow_entry_remove_all_children(fe, fsw);
710 }
711 }
712
713 void
flow_entry_destroy(struct flow_owner * fo,struct flow_entry * fe,bool nolinger,void * close_params)714 flow_entry_destroy(struct flow_owner *fo, struct flow_entry *fe, bool nolinger,
715 void *close_params)
716 {
717 struct flow_mgr *fm = fo->fo_fsw->fsw_flow_mgr;
718 int err;
719
720 FOB_LOCK_ASSERT_HELD(FO_BUCKET(fo));
721
722 /*
723 * regular flow: one in flow_table, one in id_tree, one here
724 * child flow: one in id_tree, one here
725 */
726 ASSERT(flow_entry_refcnt(fe) > 2 ||
727 ((fe->fe_flags & FLOWENTF_CHILD) && flow_entry_refcnt(fe) > 1));
728
729 flow_entry_teardown(fo, fe);
730
731 err = flow_mgr_flow_hash_mask_del(fm, fe->fe_key.fk_mask);
732 ASSERT(err == 0);
733
734 /* only regular or parent flows have entries in flow_table */
735 if (__probable(!(fe->fe_flags & FLOWENTF_CHILD))) {
736 uint32_t hash;
737 hash = flow_key_hash(&fe->fe_key);
738 cuckoo_hashtable_del(fm->fm_flow_table, &fe->fe_cnode, hash);
739 }
740
741 RB_REMOVE(flow_entry_id_tree, &fo->fo_flow_entry_id_head, fe);
742 struct flow_entry *__single tfe = fe;
743 flow_entry_release(&tfe);
744
745 ASSERT(!(fe->fe_flags & FLOWENTF_DESTROYED));
746 os_atomic_or(&fe->fe_flags, FLOWENTF_DESTROYED, relaxed);
747
748 if (fe->fe_transport_protocol == IPPROTO_QUIC) {
749 if (!nolinger && close_params != NULL) {
750 /*
751 * -fbounds-safety: We can't annotate close_params (last
752 * argument of this function) with
753 * __sized_by(QUIC_STATELESS_RESET_TOKEN_SIZE) because
754 * there are callsites that pass NULL to this. Until
755 * __sized_by_or_null is available (rdar://75598414),
756 * forge this for now.
757 */
758 uint8_t *quic_close_params = __unsafe_forge_bidi_indexable(uint8_t *,
759 close_params, QUIC_STATELESS_RESET_TOKEN_SIZE);
760 flow_track_abort_quic(fe, quic_close_params);
761 }
762 flow_entry_release(&fe);
763 } else if (nolinger || !(fe->fe_flags & FLOWENTF_WAIT_CLOSE)) {
764 flow_entry_release(&fe);
765 } else {
766 fsw_linger_insert(fe);
767 }
768 }
769
770 uint32_t
flow_entry_refcnt(struct flow_entry * fe)771 flow_entry_refcnt(struct flow_entry *fe)
772 {
773 return os_ref_get_count(&fe->fe_refcnt);
774 }
775
776 void
flow_entry_retain(struct flow_entry * fe)777 flow_entry_retain(struct flow_entry *fe)
778 {
779 os_ref_retain(&fe->fe_refcnt);
780 }
781
782 void
flow_entry_release(struct flow_entry ** pfe)783 flow_entry_release(struct flow_entry **pfe)
784 {
785 struct flow_entry *fe = *pfe;
786 ASSERT(fe != NULL);
787 *pfe = NULL; /* caller lose reference */
788 #if SK_LOG
789 if (__improbable(sk_verbose != 0)) {
790 char dbgbuf[FLOWENTRY_DBGBUF_SIZE];
791 SK_DF(SK_VERB_FLOW, "entry \"%s\" fe 0x%llx flags 0x%b",
792 fe_as_string(fe, dbgbuf, sizeof(dbgbuf)), SK_KVA(fe),
793 fe->fe_flags, FLOWENTF_BITS);
794 }
795 #endif /* SK_LOG */
796
797 if (__improbable(os_ref_release(&fe->fe_refcnt) == 0)) {
798 fe->fe_nx_port = NEXUS_PORT_ANY;
799 if (fe->fe_route != NULL) {
800 flow_route_release(fe->fe_route);
801 fe->fe_route = NULL;
802 }
803 if (fe->fe_qset != NULL) {
804 nx_netif_qset_release(&fe->fe_qset);
805 ASSERT(fe->fe_qset == NULL);
806 }
807 if (fe->fe_demux_patterns != NULL) {
808 sk_free_type_array_counted_by(struct kern_flow_demux_pattern,
809 fe->fe_demux_pattern_count, fe->fe_demux_patterns);
810 fe->fe_demux_patterns = NULL;
811 fe->fe_demux_pattern_count = 0;
812 }
813 if (fe->fe_demux_pkt_data != NULL) {
814 size_t demux_pkt_data_size = FLOW_DEMUX_MAX_LEN;
815 sk_free_data_sized_by(fe->fe_demux_pkt_data, demux_pkt_data_size);
816 fe->fe_demux_pkt_data = NULL;
817 }
818 fe_free(fe);
819 }
820 }
821
822 struct flow_entry_dead *
flow_entry_dead_alloc(zalloc_flags_t how)823 flow_entry_dead_alloc(zalloc_flags_t how)
824 {
825 struct flow_entry_dead *fed;
826
827 fed = zalloc_flags(sk_fed_zone, how | Z_ZERO);
828 if (fed != NULL) {
829 SK_DF(SK_VERB_MEM, "fed 0x%llx ALLOC", SK_KVA(fed));
830 }
831 return fed;
832 }
833
834 void
flow_entry_dead_free(struct flow_entry_dead * fed)835 flow_entry_dead_free(struct flow_entry_dead *fed)
836 {
837 SK_DF(SK_VERB_MEM, "fed 0x%llx FREE", SK_KVA(fed));
838 zfree(sk_fed_zone, fed);
839 }
840
841 static void
fe_stats_init(struct flow_entry * fe)842 fe_stats_init(struct flow_entry *fe)
843 {
844 struct nx_flowswitch *fsw = fe->fe_fsw;
845 struct sk_stats_flow *sf = &fe->fe_stats->fs_stats;
846
847 ASSERT(fe->fe_stats != NULL);
848 ASSERT(os_ref_get_count(&fe->fe_stats->fs_refcnt) >= 1);
849
850 bzero(sf, sizeof(*sf));
851 uuid_copy(sf->sf_nx_uuid, fsw->fsw_nx->nx_uuid);
852 uuid_copy(sf->sf_uuid, fe->fe_uuid);
853 (void) strbufcpy(sf->sf_if_name, fsw->fsw_flow_mgr->fm_name);
854 sf->sf_if_index = fsw->fsw_ifp->if_index;
855 sf->sf_pid = fe->fe_pid;
856 sf->sf_epid = fe->fe_epid;
857 (void) snprintf(sf->sf_proc_name, sizeof(sf->sf_proc_name), "%s",
858 fe->fe_proc_name);
859 (void) snprintf(sf->sf_eproc_name, sizeof(sf->sf_eproc_name), "%s",
860 fe->fe_eproc_name);
861
862 sf->sf_nx_port = fe->fe_nx_port;
863 sf->sf_key = fe->fe_key;
864 sf->sf_protocol = fe->fe_transport_protocol;
865 sf->sf_svc_class = (packet_svc_class_t)fe->fe_svc_class;
866 sf->sf_adv_idx = fe->fe_adv_idx;
867
868 if (fe->fe_flags & FLOWENTF_TRACK) {
869 sf->sf_flags |= SFLOWF_TRACK;
870 }
871 if (fe->fe_flags & FLOWENTF_LISTENER) {
872 sf->sf_flags |= SFLOWF_LISTENER;
873 }
874 if (fe->fe_route != NULL && fe->fe_route->fr_flags & FLOWRTF_ONLINK) {
875 sf->sf_flags |= SFLOWF_ONLINK;
876 }
877
878 fe_stats_update(fe);
879 }
880
881 static void
fe_stats_update(struct flow_entry * fe)882 fe_stats_update(struct flow_entry *fe)
883 {
884 struct sk_stats_flow *sf = &fe->fe_stats->fs_stats;
885
886 ASSERT(fe->fe_stats != NULL);
887 ASSERT(os_ref_get_count(&fe->fe_stats->fs_refcnt) >= 1);
888
889 if (fe->fe_flags & FLOWENTF_CONNECTED) {
890 sf->sf_flags |= SFLOWF_CONNECTED;
891 }
892 if (fe->fe_flags & FLOWENTF_QOS_MARKING) {
893 sf->sf_flags |= SFLOWF_QOS_MARKING;
894 }
895 if (fe->fe_flags & FLOWENTF_WAIT_CLOSE) {
896 sf->sf_flags |= SFLOWF_WAIT_CLOSE;
897 }
898 if (fe->fe_flags & FLOWENTF_CLOSE_NOTIFY) {
899 sf->sf_flags |= SFLOWF_CLOSE_NOTIFY;
900 }
901 if (fe->fe_flags & FLOWENTF_ABORTED) {
902 sf->sf_flags |= SFLOWF_ABORTED;
903 }
904 if (fe->fe_flags & FLOWENTF_NONVIABLE) {
905 sf->sf_flags |= SFLOWF_NONVIABLE;
906 }
907 if (fe->fe_flags & FLOWENTF_WITHDRAWN) {
908 sf->sf_flags |= SFLOWF_WITHDRAWN;
909 }
910 if (fe->fe_flags & FLOWENTF_TORN_DOWN) {
911 sf->sf_flags |= SFLOWF_TORN_DOWN;
912 }
913 if (fe->fe_flags & FLOWENTF_DESTROYED) {
914 sf->sf_flags |= SFLOWF_DESTROYED;
915 }
916 if (fe->fe_flags & FLOWENTF_LINGERING) {
917 sf->sf_flags |= SFLOWF_LINGERING;
918 }
919 if (fe->fe_flags & FLOWENTF_LOW_LATENCY) {
920 sf->sf_flags |= SFLOWF_LOW_LATENCY;
921 }
922 if (fe->fe_flags & FLOWENTF_PARENT) {
923 sf->sf_flags |= SFLOWF_PARENT;
924 }
925 if (fe->fe_flags & FLOWENTF_CHILD) {
926 sf->sf_flags |= SFLOWF_CHILD;
927 }
928 if (fe->fe_flags & FLOWENTF_NOWAKEFROMSLEEP) {
929 sf->sf_flags |= SFLOWF_NOWAKEFROMSLEEP;
930 } else {
931 sf->sf_flags &= ~SFLOWF_NOWAKEFROMSLEEP;
932 }
933
934 sf->sf_bucket_idx = SFLOW_BUCKET_NONE;
935
936 sf->sf_ltrack.sft_state = fe->fe_ltrack.fse_state;
937 sf->sf_ltrack.sft_seq = fe->fe_ltrack.fse_seqlo;
938 sf->sf_ltrack.sft_max_win = fe->fe_ltrack.fse_max_win;
939 sf->sf_ltrack.sft_wscale = fe->fe_ltrack.fse_wscale;
940 sf->sf_rtrack.sft_state = fe->fe_rtrack.fse_state;
941 sf->sf_rtrack.sft_seq = fe->fe_rtrack.fse_seqlo;
942 sf->sf_rtrack.sft_max_win = fe->fe_rtrack.fse_max_win;
943 }
944
945 void
flow_entry_stats_get(struct flow_entry * fe,struct sk_stats_flow * sf)946 flow_entry_stats_get(struct flow_entry *fe, struct sk_stats_flow *sf)
947 {
948 _CASSERT(sizeof(fe->fe_stats->fs_stats) == sizeof(*sf));
949
950 fe_stats_update(fe);
951 bcopy(&fe->fe_stats->fs_stats, sf, sizeof(*sf));
952 }
953
954 struct flow_entry *
fe_alloc(boolean_t can_block)955 fe_alloc(boolean_t can_block)
956 {
957 struct flow_entry *fe;
958
959 _CASSERT((offsetof(struct flow_entry, fe_key) % 16) == 0);
960
961 fe = skmem_cache_alloc(sk_fe_cache,
962 can_block ? SKMEM_SLEEP : SKMEM_NOSLEEP);
963 if (fe == NULL) {
964 return NULL;
965 }
966
967 /*
968 * fe_key is 16-bytes aligned which requires fe to begin on
969 * a 16-bytes boundary as well. This alignment is specified
970 * at sk_fe_cache creation time and we assert here.
971 */
972 ASSERT(IS_P2ALIGNED(fe, 16));
973 bzero(fe, sk_fe_size);
974
975 fe->fe_stats = flow_stats_alloc(can_block);
976 if (fe->fe_stats == NULL) {
977 skmem_cache_free(sk_fe_cache, fe);
978 return NULL;
979 }
980
981 SK_DF(SK_VERB_MEM, "fe 0x%llx ALLOC", SK_KVA(fe));
982
983 os_ref_init(&fe->fe_refcnt, &flow_entry_refgrp);
984
985 lck_mtx_init(&fe->fe_rx_pktq_lock, &nexus_lock_group, &nexus_lock_attr);
986 KPKTQ_INIT(&fe->fe_rx_pktq);
987 KPKTQ_INIT(&fe->fe_tx_pktq);
988
989 return fe;
990 }
991
992 static void
fe_free(struct flow_entry * fe)993 fe_free(struct flow_entry *fe)
994 {
995 ASSERT(fe->fe_flags & FLOWENTF_TORN_DOWN);
996 ASSERT(fe->fe_flags & FLOWENTF_DESTROYED);
997 ASSERT(!(fe->fe_flags & FLOWENTF_LINGERING));
998 ASSERT(fe->fe_route == NULL);
999
1000 ASSERT(fe->fe_stats != NULL);
1001 flow_stats_release(fe->fe_stats);
1002 fe->fe_stats = NULL;
1003
1004 /* only at very last existence of flow releases namespace reservation */
1005 if (!(fe->fe_flags & FLOWENTF_EXTRL_PORT) &&
1006 NETNS_TOKEN_VALID(&fe->fe_port_reservation)) {
1007 flow_namespace_destroy(&fe->fe_port_reservation);
1008 ASSERT(!NETNS_TOKEN_VALID(&fe->fe_port_reservation));
1009 }
1010 fe->fe_port_reservation = NULL;
1011
1012 if (!(fe->fe_flags & FLOWENTF_EXTRL_PROTO) &&
1013 protons_token_is_valid(fe->fe_proto_reservation)) {
1014 protons_release(&fe->fe_proto_reservation);
1015 }
1016 fe->fe_proto_reservation = NULL;
1017
1018 if (key_custom_ipsec_token_is_valid(fe->fe_ipsec_reservation)) {
1019 key_release_custom_ipsec(&fe->fe_ipsec_reservation);
1020 }
1021 fe->fe_ipsec_reservation = NULL;
1022
1023 if (!(fe->fe_flags & FLOWENTF_EXTRL_FLOWID) && (fe->fe_flowid != 0)) {
1024 flowidns_release_flowid(fe->fe_flowid);
1025 fe->fe_flowid = 0;
1026 }
1027
1028 skmem_cache_free(sk_fe_cache, fe);
1029 }
1030
1031 static __inline__ int
fe_id_cmp(const struct flow_entry * a,const struct flow_entry * b)1032 fe_id_cmp(const struct flow_entry *a, const struct flow_entry *b)
1033 {
1034 return uuid_compare(a->fe_uuid, b->fe_uuid);
1035 }
1036
1037 #if SK_LOG
1038 SK_NO_INLINE_ATTRIBUTE
1039 char *
fk_as_string(const struct flow_key * fk,char * __counted_by (dsz)dst,size_t dsz)1040 fk_as_string(const struct flow_key *fk, char *__counted_by(dsz)dst, size_t dsz)
1041 {
1042 int af;
1043 char src_s[MAX_IPv6_STR_LEN];
1044 char dst_s[MAX_IPv6_STR_LEN];
1045
1046 af = fk->fk_ipver == 4 ? AF_INET : AF_INET6;
1047
1048 (void) inet_ntop(af, &fk->fk_src, src_s, sizeof(src_s));
1049 (void) inet_ntop(af, &fk->fk_dst, dst_s, sizeof(dst_s));
1050 (void) snprintf(dst, dsz,
1051 "ipver=%u,src=%s,dst=%s,proto=0x%02u,sport=%u,dport=%u "
1052 "mask=%08x,hash=%08x",
1053 fk->fk_ipver, src_s, dst_s, fk->fk_proto, ntohs(fk->fk_sport),
1054 ntohs(fk->fk_dport), fk->fk_mask, flow_key_hash(fk));
1055
1056 return dst;
1057 }
1058
1059 SK_NO_INLINE_ATTRIBUTE
1060 char *
fe_as_string(const struct flow_entry * fe,char * __counted_by (dsz)dst,size_t dsz)1061 fe_as_string(const struct flow_entry *fe, char *__counted_by(dsz)dst, size_t dsz)
1062 {
1063 char keybuf[FLOWKEY_DBGBUF_SIZE]; /* just for debug message */
1064 uuid_string_t uuidstr;
1065
1066 fk_as_string(&fe->fe_key, keybuf, sizeof(keybuf));
1067
1068 (void) snprintf(dst, dsz,
1069 "fe 0x%llx proc %s nx_port %d flow_uuid %s %s tp_proto=0x%02u",
1070 SK_KVA(fe), fe->fe_proc_name, (int)fe->fe_nx_port,
1071 sk_uuid_unparse(fe->fe_uuid, uuidstr),
1072 keybuf, fe->fe_transport_protocol);
1073
1074 return dst;
1075 }
1076 #endif /* SK_LOG */
1077