1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <skywalk/nexus/upipe/nx_user_pipe.h>
31 #include <skywalk/nexus/kpipe/nx_kernel_pipe.h>
32 #include <skywalk/nexus/flowswitch/nx_flowswitch.h>
33 #include <skywalk/nexus/netif/nx_netif.h>
34 #include <skywalk/nexus/monitor/nx_monitor.h>
35
36 static STAILQ_HEAD(, nxdom) nexus_domains =
37 STAILQ_HEAD_INITIALIZER(nexus_domains);
38
39 static void nxdom_attach(struct nxdom *);
40 static void nxdom_detach(struct nxdom *);
41 static void nxdom_init(struct nxdom *);
42 static void nxdom_terminate(struct nxdom *);
43 static void nxdom_fini(struct nxdom *);
44 static void nxdom_del_provider_final(struct kern_nexus_domain_provider *);
45
46 static int nxdom_prov_ext_init(struct kern_nexus_domain_provider *);
47 static void nxdom_prov_ext_fini(struct kern_nexus_domain_provider *);
48 static struct kern_nexus_domain_provider *nxdom_prov_alloc(zalloc_flags_t);
49 static void nxdom_prov_free(struct kern_nexus_domain_provider *);
50
51 static uint32_t nxprov_bound_var(uint32_t *, uint32_t, uint32_t, uint32_t,
52 const char *);
53 static void nxprov_detaching_enqueue(struct kern_nexus_domain_provider *);
54 static struct kern_nexus_domain_provider *nxprov_detaching_dequeue(void);
55 static void nxprov_detacher(void *, wait_result_t);
56 static int nxprov_detacher_cont(int);
57
58 static struct nexus_controller *ncd_alloc(zalloc_flags_t);
59 static void ncd_free(struct nexus_controller *);
60
61 static struct nexus_attr *nxa_alloc(zalloc_flags_t);
62 static void nxa_free(struct nexus_attr *);
63
64 static int _kern_nexus_ifattach(struct nxctl *nxctl, const uuid_t nx_uuid,
65 struct ifnet *ifp, const uuid_t nx_uuid_attachee, boolean_t host,
66 uuid_t *nx_if_uuid);
67
68 static ZONE_DEFINE(ncd_zone, SKMEM_ZONE_PREFIX ".nx.kern.ctl.desc",
69 sizeof(struct nexus_controller), ZC_ZFREE_CLEARMEM);
70
71 static ZONE_DEFINE(nxdom_prov_zone, SKMEM_ZONE_PREFIX ".nx.kern.dom.prov",
72 sizeof(struct kern_nexus_domain_provider), ZC_ZFREE_CLEARMEM);
73
74 static ZONE_DEFINE(nxa_zone, SKMEM_ZONE_PREFIX ".nx.kern.attr",
75 sizeof(struct nexus_attr), ZC_ZFREE_CLEARMEM);
76
77 static int __nxdom_inited = 0;
78 static STAILQ_HEAD(, kern_nexus_domain_provider) nxprov_detaching_head =
79 STAILQ_HEAD_INITIALIZER(nxprov_detaching_head);
80 static uint32_t nxprov_detaching_cnt;
81 static void *nxprov_detach_wchan; /* wait channel for detacher */
82
83 /*
84 * Array of default nexus domain providers. Initialized once during
85 * domain attach time; no lock is needed to read as they can be treated
86 * as immutables, since default providers imply built-in ones and they
87 * never detach in practice.
88 */
89 struct kern_nexus_domain_provider *nxdom_prov_default[NEXUS_TYPE_MAX];
90
91 void
nxdom_attach_all(void)92 nxdom_attach_all(void)
93 {
94 struct nxdom *nxdom;
95 thread_t tp = THREAD_NULL;
96
97 SK_LOCK_ASSERT_HELD();
98 ASSERT(!__nxdom_inited);
99 ASSERT(STAILQ_EMPTY(&nexus_domains));
100
101 #if CONFIG_NEXUS_FLOWSWITCH
102 nxdom_attach(&nx_flowswitch_dom_s);
103 #endif /* CONFIG_NEXUS_FLOWSWITCH */
104 #if CONFIG_NEXUS_USER_PIPE
105 nxdom_attach(&nx_upipe_dom_s);
106 #endif /* CONFIG_NEXUS_USER_PIPE */
107 #if CONFIG_NEXUS_KERNEL_PIPE
108 nxdom_attach(&nx_kpipe_dom_s);
109 #endif /* CONFIG_NEXUS_KERNEL_PIPE */
110 #if CONFIG_NEXUS_NETIF
111 nxdom_attach(&nx_netif_dom_s);
112 #endif /* CONFIG_NEXUS_NETIF */
113 #if CONFIG_NEXUS_MONITOR
114 nxdom_attach(&nx_monitor_dom_s);
115 #endif /* CONFIG_NEXUS_MONITOR */
116
117 /* ask domains to initialize */
118 STAILQ_FOREACH(nxdom, &nexus_domains, nxdom_link)
119 nxdom_init(nxdom);
120
121 if (kernel_thread_start(nxprov_detacher, NULL, &tp) != KERN_SUCCESS) {
122 panic_plain("%s: couldn't create detacher thread", __func__);
123 /* NOTREACHED */
124 __builtin_unreachable();
125 }
126 thread_deallocate(tp);
127
128 __nxdom_inited = 1;
129 }
130
131 void
nxdom_detach_all(void)132 nxdom_detach_all(void)
133 {
134 struct nxdom *nxdom, *tnxdom;
135
136 SK_LOCK_ASSERT_HELD();
137
138 if (__nxdom_inited) {
139 STAILQ_FOREACH_SAFE(nxdom, &nexus_domains, nxdom_link, tnxdom) {
140 nxdom_terminate(nxdom);
141 nxdom_fini(nxdom);
142 nxdom_detach(nxdom);
143 }
144
145 /*
146 * TODO: [email protected] -- terminate detacher thread.
147 */
148
149 __nxdom_inited = 0;
150 }
151 ASSERT(STAILQ_EMPTY(&nexus_domains));
152 }
153
154 #define ASSERT_NXDOM_PARAMS(_dom, _var) do { \
155 ASSERT(NXDOM_MIN(_dom, _var) <= NXDOM_MAX(_dom, _var)); \
156 ASSERT(NXDOM_DEF(_dom, _var) >= NXDOM_MIN(_dom, _var)); \
157 ASSERT(NXDOM_DEF(_dom, _var) <= NXDOM_MAX(_dom, _var)); \
158 } while (0)
159
160 static void
nxdom_attach(struct nxdom * nxdom)161 nxdom_attach(struct nxdom *nxdom)
162 {
163 struct nxdom *nxdom1;
164
165 SK_LOCK_ASSERT_HELD();
166 ASSERT(!(nxdom->nxdom_flags & NEXUSDOMF_ATTACHED));
167
168 STAILQ_FOREACH(nxdom1, &nexus_domains, nxdom_link) {
169 if (nxdom1->nxdom_type == nxdom->nxdom_type) {
170 /* type must be unique; this is a programming error */
171 VERIFY(0);
172 /* NOTREACHED */
173 __builtin_unreachable();
174 }
175 }
176
177 /* verify this is a valid type */
178 switch (nxdom->nxdom_type) {
179 case NEXUS_TYPE_USER_PIPE:
180 case NEXUS_TYPE_KERNEL_PIPE:
181 case NEXUS_TYPE_NET_IF:
182 case NEXUS_TYPE_FLOW_SWITCH:
183 case NEXUS_TYPE_MONITOR:
184 break;
185
186 default:
187 VERIFY(0);
188 /* NOTREACHED */
189 __builtin_unreachable();
190 }
191
192 /* verify this is a valid metadata type */
193 switch (nxdom->nxdom_md_type) {
194 case NEXUS_META_TYPE_QUANTUM:
195 case NEXUS_META_TYPE_PACKET:
196 break;
197
198 default:
199 VERIFY(0);
200 /* NOTREACHED */
201 __builtin_unreachable();
202 }
203
204 /* verify this is a valid metadata subtype */
205 switch (nxdom->nxdom_md_subtype) {
206 case NEXUS_META_SUBTYPE_PAYLOAD:
207 case NEXUS_META_SUBTYPE_RAW:
208 break;
209
210 default:
211 VERIFY(0);
212 /* NOTREACHED */
213 __builtin_unreachable();
214 }
215
216 #if (DEVELOPMENT || DEBUG)
217 /*
218 * Override the default ring sizes for flowswitch if configured
219 * via boot-args. Each nexus provider instance can still change
220 * the values if so desired.
221 */
222 if (nxdom->nxdom_type == NEXUS_TYPE_FLOW_SWITCH) {
223 if (sk_txring_sz != 0) {
224 if (sk_txring_sz < NXDOM_MIN(nxdom, tx_slots)) {
225 sk_txring_sz = NXDOM_MIN(nxdom, tx_slots);
226 } else if (sk_txring_sz > NXDOM_MAX(nxdom, tx_slots)) {
227 sk_txring_sz = NXDOM_MAX(nxdom, tx_slots);
228 }
229 NXDOM_DEF(nxdom, tx_slots) = sk_txring_sz;
230 }
231 if (sk_rxring_sz != 0) {
232 if (sk_rxring_sz < NXDOM_MIN(nxdom, rx_slots)) {
233 sk_rxring_sz = NXDOM_MIN(nxdom, rx_slots);
234 } else if (sk_rxring_sz > NXDOM_MAX(nxdom, rx_slots)) {
235 sk_rxring_sz = NXDOM_MAX(nxdom, rx_slots);
236 }
237 NXDOM_DEF(nxdom, rx_slots) = sk_rxring_sz;
238 }
239 }
240 /*
241 * Override the default ring sizes for netif if configured
242 * via boot-args. Each nexus provider instance can still change
243 * the values if so desired.
244 */
245 if (nxdom->nxdom_type == NEXUS_TYPE_NET_IF) {
246 if (sk_net_txring_sz != 0) {
247 if (sk_net_txring_sz < NXDOM_MIN(nxdom, tx_slots)) {
248 sk_net_txring_sz = NXDOM_MIN(nxdom, tx_slots);
249 } else if (sk_net_txring_sz > NXDOM_MAX(nxdom, tx_slots)) {
250 sk_net_txring_sz = NXDOM_MAX(nxdom, tx_slots);
251 }
252 NXDOM_DEF(nxdom, tx_slots) = sk_net_txring_sz;
253 }
254 if (sk_net_rxring_sz != 0) {
255 if (sk_net_rxring_sz < NXDOM_MIN(nxdom, rx_slots)) {
256 sk_net_rxring_sz = NXDOM_MIN(nxdom, rx_slots);
257 } else if (sk_net_rxring_sz > NXDOM_MAX(nxdom, rx_slots)) {
258 sk_net_rxring_sz = NXDOM_MAX(nxdom, rx_slots);
259 }
260 NXDOM_DEF(nxdom, rx_slots) = sk_net_rxring_sz;
261 }
262 }
263
264 #endif /* DEVELOPMENT || DEBUG */
265
266 /* verify that parameters are sane */
267 ASSERT(NXDOM_MAX(nxdom, ports) > 0);
268 ASSERT(NXDOM_MAX(nxdom, ports) <= NEXUS_PORT_MAX);
269 ASSERT_NXDOM_PARAMS(nxdom, ports);
270 ASSERT_NXDOM_PARAMS(nxdom, tx_rings);
271 ASSERT_NXDOM_PARAMS(nxdom, rx_rings);
272 ASSERT(NXDOM_MAX(nxdom, tx_slots) > 0);
273 ASSERT_NXDOM_PARAMS(nxdom, tx_slots);
274 ASSERT(NXDOM_MAX(nxdom, rx_slots) > 0);
275 ASSERT_NXDOM_PARAMS(nxdom, rx_slots);
276 ASSERT_NXDOM_PARAMS(nxdom, buf_size);
277 ASSERT_NXDOM_PARAMS(nxdom, meta_size);
278 ASSERT_NXDOM_PARAMS(nxdom, pipes);
279 ASSERT_NXDOM_PARAMS(nxdom, extensions);
280
281 /* these must exist */
282 ASSERT(nxdom->nxdom_bind_port != NULL);
283 ASSERT(nxdom->nxdom_unbind_port != NULL);
284 ASSERT(nxdom->nxdom_connect != NULL);
285 ASSERT(nxdom->nxdom_disconnect != NULL);
286 ASSERT(nxdom->nxdom_defunct != NULL);
287 ASSERT(nxdom->nxdom_defunct_finalize != NULL);
288
289 STAILQ_INSERT_TAIL(&nexus_domains, nxdom, nxdom_link);
290 nxdom->nxdom_flags |= NEXUSDOMF_ATTACHED;
291 }
292
293 #undef VERIFY_NXDOM_PARAMS
294
295 static void
nxdom_detach(struct nxdom * nxdom)296 nxdom_detach(struct nxdom *nxdom)
297 {
298 SK_LOCK_ASSERT_HELD();
299 ASSERT(nxdom->nxdom_flags & NEXUSDOMF_ATTACHED);
300
301 STAILQ_REMOVE(&nexus_domains, nxdom, nxdom, nxdom_link);
302 nxdom->nxdom_flags &= ~NEXUSDOMF_ATTACHED;
303 }
304
305 static void
nxdom_init(struct nxdom * nxdom)306 nxdom_init(struct nxdom *nxdom)
307 {
308 ASSERT(nxdom->nxdom_flags & NEXUSDOMF_ATTACHED);
309
310 SK_LOCK_ASSERT_HELD();
311
312 if (!(nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED)) {
313 if (nxdom->nxdom_init != NULL) {
314 nxdom->nxdom_init(nxdom);
315 }
316 nxdom->nxdom_flags |= NEXUSDOMF_INITIALIZED;
317 }
318 }
319
320 static void
nxdom_terminate(struct nxdom * nxdom)321 nxdom_terminate(struct nxdom *nxdom)
322 {
323 ASSERT(nxdom->nxdom_flags & NEXUSDOMF_ATTACHED);
324
325 SK_LOCK_ASSERT_HELD();
326
327 if ((nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED) &&
328 !(nxdom->nxdom_flags & NEXUSDOMF_TERMINATED)) {
329 if (nxdom->nxdom_terminate != NULL) {
330 nxdom->nxdom_terminate(nxdom);
331 }
332 nxdom->nxdom_flags |= NEXUSDOMF_TERMINATED;
333 }
334 }
335
336 static void
nxdom_fini(struct nxdom * nxdom)337 nxdom_fini(struct nxdom *nxdom)
338 {
339 ASSERT(nxdom->nxdom_flags & NEXUSDOMF_ATTACHED);
340
341 if (nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED) {
342 if (nxdom->nxdom_fini != NULL) {
343 nxdom->nxdom_fini(nxdom);
344 }
345 nxdom->nxdom_flags &= ~NEXUSDOMF_INITIALIZED;
346 }
347 }
348
349 int
nxdom_prov_add(struct nxdom * nxdom,struct kern_nexus_domain_provider * nxdom_prov)350 nxdom_prov_add(struct nxdom *nxdom,
351 struct kern_nexus_domain_provider *nxdom_prov)
352 {
353 struct kern_nexus_domain_provider *nxprov1;
354 nexus_type_t type = nxdom->nxdom_type;
355 boolean_t builtin;
356 int err = 0;
357
358 SK_LOCK_ASSERT_HELD();
359 ASSERT(type < NEXUS_TYPE_MAX);
360
361 builtin = !(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT);
362
363 STAILQ_FOREACH(nxprov1, &nxdom->nxdom_prov_head, nxdom_prov_link) {
364 /*
365 * We can be a little more strict in the kernel and
366 * avoid namespace collision (even though each domain
367 * provider has UUID; this also guarantees that external
368 * providers won't conflict with the builtin ones.
369 */
370 if (strcmp(nxprov1->nxdom_prov_name,
371 nxdom_prov->nxdom_prov_name) == 0) {
372 return EEXIST;
373 }
374 }
375
376 VERIFY(!(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_ATTACHED));
377 VERIFY(!(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_INITIALIZED));
378
379 uuid_generate_random(nxdom_prov->nxdom_prov_uuid);
380 nxdom_prov->nxdom_prov_dom = nxdom;
381 if (nxdom_prov->nxdom_prov_init != NULL) {
382 err = nxdom_prov->nxdom_prov_init(nxdom_prov);
383 }
384
385 if (err == 0) {
386 nxdom_prov->nxdom_prov_flags |=
387 (NXDOMPROVF_ATTACHED | NXDOMPROVF_INITIALIZED);
388 STAILQ_INSERT_TAIL(&nxdom->nxdom_prov_head, nxdom_prov,
389 nxdom_prov_link);
390 /* for being in the list */
391 nxdom_prov_retain_locked(nxdom_prov);
392
393 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_DEFAULT) {
394 VERIFY(builtin && nxdom_prov_default[type] == NULL);
395 nxdom_prov_default[type] = nxdom_prov;
396 /* for being in the array */
397 nxdom_prov_retain_locked(nxdom_prov);
398 }
399
400 SK_D("nxdom_prov 0x%llx (%s) dom %s",
401 SK_KVA(nxdom_prov), nxdom_prov->nxdom_prov_name,
402 nxdom->nxdom_name);
403 } else {
404 uuid_clear(nxdom_prov->nxdom_prov_uuid);
405 nxdom_prov->nxdom_prov_dom = NULL;
406 }
407
408 return err;
409 }
410
411 void
nxdom_prov_del(struct kern_nexus_domain_provider * nxdom_prov)412 nxdom_prov_del(struct kern_nexus_domain_provider *nxdom_prov)
413 {
414 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
415 nexus_type_t type = nxdom->nxdom_type;
416
417 SK_LOCK_ASSERT_HELD();
418 ASSERT(type < NEXUS_TYPE_MAX);
419 ASSERT(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_ATTACHED);
420
421 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_DETACHING) {
422 return;
423 }
424
425 SK_D("nxdom_prov 0x%llx (%s:%s)", SK_KVA(nxdom_prov), nxdom->nxdom_name,
426 nxdom_prov->nxdom_prov_name);
427
428 /* keep the reference around for the detaching list (see below) */
429 STAILQ_REMOVE(&nxdom->nxdom_prov_head, nxdom_prov,
430 kern_nexus_domain_provider, nxdom_prov_link);
431 nxdom_prov->nxdom_prov_flags &= ~NXDOMPROVF_ATTACHED;
432 nxdom_prov->nxdom_prov_flags |= NXDOMPROVF_DETACHING;
433
434 /* there can only be one default and it must match this one */
435 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_DEFAULT) {
436 ASSERT(!(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT));
437 VERIFY(nxdom_prov_default[type] == nxdom_prov);
438 nxdom_prov_default[type] = NULL;
439 /*
440 * Release reference held for the array; this must
441 * not be the last reference, as there is still at
442 * least one which we kept for the detaching list.
443 */
444 VERIFY(!nxdom_prov_release_locked(nxdom_prov));
445 }
446
447 /* add to detaching list and wake up detacher */
448 nxprov_detaching_enqueue(nxdom_prov);
449 }
450
451 static void
nxdom_del_provider_final(struct kern_nexus_domain_provider * nxdom_prov)452 nxdom_del_provider_final(struct kern_nexus_domain_provider *nxdom_prov)
453 {
454 #if (DEBUG || DEVELOPMENT)
455 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
456 #endif /* DEBUG || DEVELOPMENT */
457
458 SK_LOCK_ASSERT_HELD();
459
460 ASSERT((nxdom_prov->nxdom_prov_flags & (NXDOMPROVF_ATTACHED |
461 NXDOMPROVF_DETACHING)) == NXDOMPROVF_DETACHING);
462 ASSERT(nxdom != NULL);
463
464 SK_D("nxdom_prov 0x%llx (%s:%s)", SK_KVA(nxdom_prov), nxdom->nxdom_name,
465 nxdom_prov->nxdom_prov_name);
466
467 nxdom_prov->nxdom_prov_flags &= ~NXDOMPROVF_DETACHING;
468
469 /*
470 * Release reference held for detaching list; if this is the last
471 * reference, the domain provider's nxdom_prov_fini() callback will
472 * be called (if applicable) within the detacher thread's context.
473 * Otherwise, this will occur when the last nexus provider for that
474 * domain provider has been released.
475 */
476 (void) nxdom_prov_release_locked(nxdom_prov);
477 }
478
479 struct nxdom *
nxdom_find(nexus_type_t type)480 nxdom_find(nexus_type_t type)
481 {
482 struct nxdom *nxdom;
483
484 SK_LOCK_ASSERT_HELD();
485 ASSERT(type < NEXUS_TYPE_MAX);
486
487 STAILQ_FOREACH(nxdom, &nexus_domains, nxdom_link) {
488 if (nxdom->nxdom_type == type) {
489 break;
490 }
491 }
492
493 return nxdom;
494 }
495
496 struct kern_nexus_domain_provider *
nxdom_prov_find(const struct nxdom * nxdom,const char * name)497 nxdom_prov_find(const struct nxdom *nxdom, const char *name)
498 {
499 struct kern_nexus_domain_provider *nxdom_prov = NULL;
500
501 SK_LOCK_ASSERT_HELD();
502
503 if (name != NULL) {
504 STAILQ_FOREACH(nxdom_prov, &nxdom->nxdom_prov_head,
505 nxdom_prov_link) {
506 if (strcmp(nxdom_prov->nxdom_prov_name, name) == 0) {
507 break;
508 }
509 }
510 }
511
512 if (nxdom_prov != NULL) {
513 nxdom_prov_retain_locked(nxdom_prov); /* for caller */
514 }
515 return nxdom_prov;
516 }
517
518 struct kern_nexus_domain_provider *
nxdom_prov_find_uuid(const uuid_t dom_prov_uuid)519 nxdom_prov_find_uuid(const uuid_t dom_prov_uuid)
520 {
521 struct kern_nexus_domain_provider *nxdom_prov = NULL;
522 struct nxdom *nxdom;
523
524 SK_LOCK_ASSERT_HELD();
525 ASSERT(dom_prov_uuid != NULL && !uuid_is_null(dom_prov_uuid));
526
527 STAILQ_FOREACH(nxdom, &nexus_domains, nxdom_link) {
528 STAILQ_FOREACH(nxdom_prov, &nxdom->nxdom_prov_head,
529 nxdom_prov_link) {
530 ASSERT(!uuid_is_null(nxdom_prov->nxdom_prov_uuid));
531 if (uuid_compare(nxdom_prov->nxdom_prov_uuid,
532 dom_prov_uuid) == 0) {
533 break;
534 }
535 }
536 if (nxdom_prov != NULL) {
537 nxdom_prov_retain_locked(nxdom_prov); /* for caller */
538 break;
539 }
540 }
541
542 return nxdom_prov;
543 }
544
545 errno_t
kern_nexus_register_domain_provider(const nexus_type_t type,const nexus_domain_provider_name_t name,const struct kern_nexus_domain_provider_init * init,const uint32_t init_len,uuid_t * dom_prov_uuid)546 kern_nexus_register_domain_provider(const nexus_type_t type,
547 const nexus_domain_provider_name_t name,
548 const struct kern_nexus_domain_provider_init *init,
549 const uint32_t init_len, uuid_t *dom_prov_uuid)
550 {
551 struct kern_nexus_domain_provider *nxdom_prov = NULL;
552 struct nxdom *nxdom;
553 errno_t err = 0;
554
555 _CASSERT(sizeof(*init) == sizeof(nxdom_prov->nxdom_prov_ext));
556
557 if (type >= NEXUS_TYPE_MAX || dom_prov_uuid == NULL) {
558 return EINVAL;
559 }
560
561 uuid_clear(*dom_prov_uuid);
562
563 if (name == NULL || init == NULL || init_len < sizeof(*init) ||
564 init->nxdpi_version != KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION) {
565 return EINVAL;
566 }
567
568 /*
569 * init, fini are required.
570 */
571 if (init->nxdpi_init == NULL || init->nxdpi_fini == NULL) {
572 return EINVAL;
573 }
574
575 SK_LOCK();
576 if (nxdom_prov_default[type] == NULL) {
577 err = ENXIO;
578 goto done;
579 }
580
581 nxdom = nxdom_find(type);
582 if (nxdom == NULL) {
583 err = ENXIO;
584 goto done;
585 }
586
587 /*
588 * Allow only kernel pipe and netif external domain providers for
589 * now, until we understand the implications and requirements for
590 * supporting other domain types. For all other types, using
591 * the built-in domain providers and registering nexus should
592 * suffice.
593 */
594 if (nxdom->nxdom_type != NEXUS_TYPE_KERNEL_PIPE &&
595 nxdom->nxdom_type != NEXUS_TYPE_NET_IF) {
596 err = EINVAL;
597 goto done;
598 }
599
600 nxdom_prov = nxdom_prov_alloc(Z_WAITOK);
601
602 /*
603 * Point all callback routines to the default provider for this
604 * domain; for nxdom_prov{init,fini}, refer to externally-provided
605 * callback routines, if applicable.
606 */
607 bcopy(init, &nxdom_prov->nxdom_prov_ext, sizeof(*init));
608 bcopy(&nxdom_prov_default[type]->nxdom_prov_cb,
609 &nxdom_prov->nxdom_prov_cb, sizeof(struct nxdom_prov_cb));
610 nxdom_prov->nxdom_prov_flags |= NXDOMPROVF_EXT;
611 nxdom_prov->nxdom_prov_init = nxdom_prov_ext_init;
612 nxdom_prov->nxdom_prov_fini = nxdom_prov_ext_fini;
613 (void) snprintf(nxdom_prov->nxdom_prov_name,
614 sizeof(nxdom_prov->nxdom_prov_name), "%s", name);
615
616 ASSERT(!(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_DEFAULT));
617 err = nxdom_prov_add(nxdom, nxdom_prov);
618 if (err != 0) {
619 nxdom_prov_free(nxdom_prov);
620 nxdom_prov = NULL;
621 }
622
623 done:
624 if (nxdom_prov != NULL) {
625 ASSERT(err == 0 && !uuid_is_null(nxdom_prov->nxdom_prov_uuid));
626 uuid_copy(*dom_prov_uuid, nxdom_prov->nxdom_prov_uuid);
627 }
628 SK_UNLOCK();
629
630 return err;
631 }
632
633 errno_t
kern_nexus_deregister_domain_provider(const uuid_t dom_prov_uuid)634 kern_nexus_deregister_domain_provider(const uuid_t dom_prov_uuid)
635 {
636 struct kern_nexus_domain_provider *nxdom_prov = NULL;
637 errno_t err = 0;
638
639 if (dom_prov_uuid == NULL || uuid_is_null(dom_prov_uuid)) {
640 return EINVAL;
641 }
642
643 SK_LOCK();
644 nxdom_prov = nxdom_prov_find_uuid(dom_prov_uuid);
645 if (nxdom_prov == NULL) {
646 err = ENXIO;
647 goto done;
648 }
649
650 /* don't allow external request for built-in domain providers */
651 if (!(nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT)) {
652 err = EINVAL;
653 goto done;
654 }
655
656 /* schedule this to be deleted */
657 nxdom_prov_del(nxdom_prov);
658 done:
659 /* release reference from nxdom_prov_find_uuid */
660 if (nxdom_prov != NULL) {
661 (void) nxdom_prov_release_locked(nxdom_prov);
662 }
663 SK_UNLOCK();
664
665 return err;
666 }
667
668 errno_t
kern_nexus_get_default_domain_provider(const nexus_type_t type,uuid_t * dom_prov_uuid)669 kern_nexus_get_default_domain_provider(const nexus_type_t type,
670 uuid_t *dom_prov_uuid)
671 {
672 struct kern_nexus_domain_provider *nxdom_prov;
673
674 if (type >= NEXUS_TYPE_MAX || dom_prov_uuid == NULL) {
675 return EINVAL;
676 }
677
678 uuid_clear(*dom_prov_uuid);
679
680 /* no lock is needed; array is immutable */
681 if ((nxdom_prov = nxdom_prov_default[type]) == NULL) {
682 return ENXIO;
683 }
684
685 uuid_copy(*dom_prov_uuid, nxdom_prov->nxdom_prov_uuid);
686
687 return 0;
688 }
689
690 static int
nxdom_prov_ext_init(struct kern_nexus_domain_provider * nxdom_prov)691 nxdom_prov_ext_init(struct kern_nexus_domain_provider *nxdom_prov)
692 {
693 int err = 0;
694
695 SK_D("initializing %s", nxdom_prov->nxdom_prov_name);
696
697 ASSERT(nxdom_prov->nxdom_prov_ext.nxdpi_init != NULL);
698 if ((err = nxdom_prov->nxdom_prov_ext.nxdpi_init(nxdom_prov)) == 0) {
699 nxdom_prov->nxdom_prov_flags |= NXDOMPROVF_EXT_INITED;
700 }
701
702 return err;
703 }
704
705 static void
nxdom_prov_ext_fini(struct kern_nexus_domain_provider * nxdom_prov)706 nxdom_prov_ext_fini(struct kern_nexus_domain_provider *nxdom_prov)
707 {
708 SK_D("destroying %s", nxdom_prov->nxdom_prov_name);
709
710 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT_INITED) {
711 ASSERT(nxdom_prov->nxdom_prov_ext.nxdpi_fini != NULL);
712 nxdom_prov->nxdom_prov_ext.nxdpi_fini(nxdom_prov);
713 nxdom_prov->nxdom_prov_flags &= ~NXDOMPROVF_EXT_INITED;
714 }
715 }
716
717 static struct nexus_attr *
nxa_alloc(zalloc_flags_t how)718 nxa_alloc(zalloc_flags_t how)
719 {
720 return zalloc_flags(nxa_zone, how | Z_ZERO);
721 }
722
723 static void
nxa_free(struct nexus_attr * nxa)724 nxa_free(struct nexus_attr *nxa)
725 {
726 SK_DF(SK_VERB_MEM, "nxa 0x%llx FREE", SK_KVA(nxa));
727 zfree(nxa_zone, nxa);
728 }
729
730 errno_t
kern_nexus_attr_create(nexus_attr_t * nxa)731 kern_nexus_attr_create(nexus_attr_t *nxa)
732 {
733 errno_t err = 0;
734
735 if (nxa == NULL) {
736 err = EINVAL;
737 } else {
738 *nxa = nxa_alloc(Z_WAITOK);
739 }
740 return err;
741 }
742
743 errno_t
kern_nexus_attr_clone(const nexus_attr_t nxa,nexus_attr_t * nnxa)744 kern_nexus_attr_clone(const nexus_attr_t nxa, nexus_attr_t *nnxa)
745 {
746 errno_t err = 0;
747
748 if (nnxa == NULL) {
749 err = EINVAL;
750 } else {
751 err = kern_nexus_attr_create(nnxa);
752 if (err == 0 && nxa != NULL) {
753 ASSERT(*nnxa != NULL);
754 bcopy(nxa, *nnxa, sizeof(**nnxa));
755 }
756 }
757 return err;
758 }
759
760 errno_t
kern_nexus_attr_set(const nexus_attr_t nxa,const nexus_attr_type_t type,const uint64_t value)761 kern_nexus_attr_set(const nexus_attr_t nxa,
762 const nexus_attr_type_t type, const uint64_t value)
763 {
764 return __nexus_attr_set(nxa, type, value);
765 }
766
767 errno_t
kern_nexus_attr_get(nexus_attr_t nxa,const nexus_attr_type_t type,uint64_t * value)768 kern_nexus_attr_get(nexus_attr_t nxa, const nexus_attr_type_t type,
769 uint64_t *value)
770 {
771 return __nexus_attr_get(nxa, type, value);
772 }
773
774 void
kern_nexus_attr_destroy(nexus_attr_t nxa)775 kern_nexus_attr_destroy(nexus_attr_t nxa)
776 {
777 nxa_free(nxa);
778 }
779
780 static struct nexus_controller *
ncd_alloc(zalloc_flags_t how)781 ncd_alloc(zalloc_flags_t how)
782 {
783 return zalloc_flags(ncd_zone, how | Z_ZERO);
784 }
785
786 static void
ncd_free(struct nexus_controller * ncd)787 ncd_free(struct nexus_controller *ncd)
788 {
789 SK_DF(SK_VERB_MEM, "ncd 0x%llx FREE", SK_KVA(ncd));
790 zfree(ncd_zone, ncd);
791 }
792
793 nexus_controller_t
kern_nexus_shared_controller(void)794 kern_nexus_shared_controller(void)
795 {
796 return &kernnxctl;
797 }
798
799 errno_t
kern_nexus_controller_create(nexus_controller_t * ncd)800 kern_nexus_controller_create(nexus_controller_t *ncd)
801 {
802 struct nxctl *nxctl = NULL;
803 uuid_t nxctl_uuid;
804 errno_t err = 0;
805
806 uuid_generate_random(nxctl_uuid);
807
808 if (ncd == NULL) {
809 err = EINVAL;
810 goto done;
811 } else {
812 *ncd = NULL;
813 }
814
815 nxctl = nxctl_create(kernproc, NULL, nxctl_uuid, &err);
816 if (nxctl == NULL) {
817 ASSERT(err != 0);
818 goto done;
819 }
820
821 *ncd = ncd_alloc(Z_WAITOK);
822 (*ncd)->ncd_nxctl = nxctl; /* ref from nxctl_create */
823
824 done:
825 if (err != 0) {
826 if (nxctl != NULL) {
827 nxctl_dtor(nxctl);
828 nxctl = NULL;
829 }
830 if (ncd != NULL && *ncd != NULL) {
831 ncd_free(*ncd);
832 *ncd = NULL;
833 }
834 }
835
836 return err;
837 }
838
839 #define NXPI_INVALID_CB_PAIRS(cb1, cb2) \
840 (!(init->nxpi_##cb1 == NULL && init->nxpi_##cb2 == NULL) && \
841 ((init->nxpi_##cb1 == NULL) ^ (init->nxpi_##cb2 == NULL)))
842
843 static errno_t
nexus_controller_register_provider_validate_init_params(const struct kern_nexus_provider_init * init,uint32_t init_len,nexus_type_t nxdom_type)844 nexus_controller_register_provider_validate_init_params(
845 const struct kern_nexus_provider_init *init, uint32_t init_len,
846 nexus_type_t nxdom_type)
847 {
848 errno_t err = 0;
849 struct kern_nexus_netif_provider_init *netif_init;
850
851 _CASSERT(__builtin_offsetof(struct kern_nexus_provider_init,
852 nxpi_version) == 0);
853 _CASSERT(sizeof(init->nxpi_version) == sizeof(uint32_t));
854
855 if (init == NULL) {
856 return 0;
857 }
858
859 if (init_len < sizeof(uint32_t)) {
860 return EINVAL;
861 }
862
863 switch (init->nxpi_version) {
864 case KERN_NEXUS_PROVIDER_VERSION_1:
865 if (init_len != sizeof(struct kern_nexus_provider_init)) {
866 err = EINVAL;
867 break;
868 }
869 /*
870 * sync_{tx,rx} callbacks are required; the rest of the
871 * callback pairs are optional, but must be symmetrical.
872 */
873 if (init->nxpi_sync_tx == NULL || init->nxpi_sync_rx == NULL ||
874 init->nxpi_pre_connect == NULL ||
875 init->nxpi_connected == NULL ||
876 init->nxpi_pre_disconnect == NULL ||
877 init->nxpi_disconnected == NULL ||
878 NXPI_INVALID_CB_PAIRS(ring_init, ring_fini) ||
879 NXPI_INVALID_CB_PAIRS(slot_init, slot_fini)) {
880 err = EINVAL;
881 break;
882 }
883 /*
884 * Tx doorbell interface is only supported for netif and
885 * Tx doorbell is mandatory for netif
886 */
887 if (((init->nxpi_tx_doorbell != NULL) &&
888 (nxdom_type != NEXUS_TYPE_NET_IF)) ||
889 ((nxdom_type == NEXUS_TYPE_NET_IF) &&
890 (init->nxpi_tx_doorbell == NULL))) {
891 err = EINVAL;
892 break;
893 }
894 /*
895 * Capabilities configuration interface is only supported for
896 * netif.
897 */
898 if ((init->nxpi_config_capab != NULL) &&
899 (nxdom_type != NEXUS_TYPE_NET_IF)) {
900 err = EINVAL;
901 break;
902 }
903 break;
904
905 case KERN_NEXUS_PROVIDER_VERSION_NETIF:
906 if (init_len != sizeof(struct kern_nexus_netif_provider_init)) {
907 err = EINVAL;
908 break;
909 }
910 if (nxdom_type != NEXUS_TYPE_NET_IF) {
911 err = EINVAL;
912 break;
913 }
914 netif_init =
915 __DECONST(struct kern_nexus_netif_provider_init *, init);
916 if (netif_init->nxnpi_pre_connect == NULL ||
917 netif_init->nxnpi_connected == NULL ||
918 netif_init->nxnpi_pre_disconnect == NULL ||
919 netif_init->nxnpi_disconnected == NULL ||
920 netif_init->nxnpi_qset_init == NULL ||
921 netif_init->nxnpi_qset_fini == NULL ||
922 netif_init->nxnpi_queue_init == NULL ||
923 netif_init->nxnpi_queue_fini == NULL ||
924 netif_init->nxnpi_tx_qset_notify == NULL ||
925 netif_init->nxnpi_config_capab == NULL) {
926 err = EINVAL;
927 break;
928 }
929 break;
930
931 default:
932 err = EINVAL;
933 break;
934 }
935 return err;
936 }
937
938 errno_t
kern_nexus_controller_register_provider(const nexus_controller_t ncd,const uuid_t dom_prov_uuid,const nexus_name_t name,const struct kern_nexus_provider_init * init,uint32_t init_len,const nexus_attr_t nxa,uuid_t * prov_uuid)939 kern_nexus_controller_register_provider(const nexus_controller_t ncd,
940 const uuid_t dom_prov_uuid, const nexus_name_t name,
941 const struct kern_nexus_provider_init *init, uint32_t init_len,
942 const nexus_attr_t nxa, uuid_t *prov_uuid)
943 {
944 struct kern_nexus_domain_provider *nxdom_prov = NULL;
945 struct kern_nexus_provider *nxprov = NULL;
946 nexus_type_t nxdom_type;
947 struct nxprov_reg reg;
948 struct nxctl *nxctl;
949 errno_t err = 0;
950
951 if (prov_uuid == NULL) {
952 return EINVAL;
953 }
954
955 uuid_clear(*prov_uuid);
956
957 if (ncd == NULL ||
958 dom_prov_uuid == NULL || uuid_is_null(dom_prov_uuid)) {
959 return EINVAL;
960 }
961
962 nxctl = ncd->ncd_nxctl;
963 NXCTL_LOCK(nxctl);
964 SK_LOCK();
965 nxdom_prov = nxdom_prov_find_uuid(dom_prov_uuid);
966 if (nxdom_prov == NULL) {
967 SK_UNLOCK();
968 err = ENXIO;
969 goto done;
970 }
971
972 nxdom_type = nxdom_prov->nxdom_prov_dom->nxdom_type;
973 ASSERT(nxdom_type < NEXUS_TYPE_MAX);
974
975 err = nexus_controller_register_provider_validate_init_params(init,
976 init_len, nxdom_type);
977 if (err != 0) {
978 SK_UNLOCK();
979 err = EINVAL;
980 goto done;
981 }
982
983 if ((err = __nexus_provider_reg_prepare(®, name,
984 nxdom_type, nxa)) != 0) {
985 SK_UNLOCK();
986 goto done;
987 }
988
989 if (init && init->nxpi_version == KERN_NEXUS_PROVIDER_VERSION_NETIF) {
990 reg.nxpreg_params.nxp_flags |= NXPF_NETIF_LLINK;
991 }
992
993 /* callee will hold reference on nxdom_prov upon success */
994 if ((nxprov = nxprov_create_kern(nxctl, nxdom_prov, ®,
995 init, &err)) == NULL) {
996 SK_UNLOCK();
997 ASSERT(err != 0);
998 goto done;
999 }
1000 SK_UNLOCK();
1001
1002 uuid_copy(*prov_uuid, nxprov->nxprov_uuid);
1003
1004 done:
1005 SK_LOCK_ASSERT_NOTHELD();
1006 NXCTL_UNLOCK(nxctl);
1007
1008 if (err != 0 && nxprov != NULL) {
1009 err = nxprov_close(nxprov, FALSE);
1010 }
1011
1012 /* release extra ref from nxprov_create_kern */
1013 if (nxprov != NULL) {
1014 nxprov_release(nxprov);
1015 }
1016 /* release extra ref from nxdom_prov_find_uuid */
1017 if (nxdom_prov != NULL) {
1018 (void) nxdom_prov_release(nxdom_prov);
1019 }
1020
1021 return err;
1022 }
1023
1024 #undef NXPI_INVALID_CB_PAIRS
1025
1026 errno_t
kern_nexus_controller_deregister_provider(const nexus_controller_t ncd,const uuid_t prov_uuid)1027 kern_nexus_controller_deregister_provider(const nexus_controller_t ncd,
1028 const uuid_t prov_uuid)
1029 {
1030 errno_t err;
1031
1032 if (ncd == NULL || prov_uuid == NULL || uuid_is_null(prov_uuid)) {
1033 err = EINVAL;
1034 } else {
1035 struct nxctl *nxctl = ncd->ncd_nxctl;
1036 NXCTL_LOCK(nxctl);
1037 err = nxprov_destroy(nxctl, prov_uuid);
1038 NXCTL_UNLOCK(nxctl);
1039 }
1040 return err;
1041 }
1042
1043 errno_t
kern_nexus_controller_alloc_provider_instance(const nexus_controller_t ncd,const uuid_t prov_uuid,const void * nx_ctx,nexus_ctx_release_fn_t nx_ctx_release,uuid_t * nx_uuid,const struct kern_nexus_init * init)1044 kern_nexus_controller_alloc_provider_instance(const nexus_controller_t ncd,
1045 const uuid_t prov_uuid, const void *nx_ctx,
1046 nexus_ctx_release_fn_t nx_ctx_release, uuid_t *nx_uuid,
1047 const struct kern_nexus_init *init)
1048 {
1049 struct kern_nexus *nx = NULL;
1050 struct nxctl *nxctl;
1051 errno_t err = 0;
1052
1053 if (ncd == NULL || prov_uuid == NULL || uuid_is_null(prov_uuid) ||
1054 nx_uuid == NULL || init == NULL ||
1055 init->nxi_version != KERN_NEXUS_CURRENT_VERSION ||
1056 (init->nxi_rx_pbufpool != NULL &&
1057 init->nxi_rx_pbufpool != init->nxi_tx_pbufpool)) {
1058 err = EINVAL;
1059 goto done;
1060 }
1061
1062 nxctl = ncd->ncd_nxctl;
1063 NXCTL_LOCK(nxctl);
1064 nx = nx_create(nxctl, prov_uuid, NEXUS_TYPE_UNDEFINED, nx_ctx,
1065 nx_ctx_release, init->nxi_tx_pbufpool, init->nxi_rx_pbufpool, &err);
1066 NXCTL_UNLOCK(nxctl);
1067 if (nx == NULL) {
1068 ASSERT(err != 0);
1069 goto done;
1070 }
1071 ASSERT(err == 0);
1072 uuid_copy(*nx_uuid, nx->nx_uuid);
1073
1074 done:
1075 /* release extra ref from nx_create */
1076 if (nx != NULL) {
1077 (void) nx_release(nx);
1078 }
1079
1080 return err;
1081 }
1082
1083 errno_t
kern_nexus_controller_alloc_net_provider_instance(const nexus_controller_t ncd,const uuid_t prov_uuid,const void * nx_ctx,nexus_ctx_release_fn_t nx_ctx_release,uuid_t * nx_uuid,const struct kern_nexus_net_init * init,struct ifnet ** pifp)1084 kern_nexus_controller_alloc_net_provider_instance(
1085 const nexus_controller_t ncd, const uuid_t prov_uuid, const void *nx_ctx,
1086 nexus_ctx_release_fn_t nx_ctx_release, uuid_t *nx_uuid,
1087 const struct kern_nexus_net_init *init, struct ifnet **pifp)
1088 {
1089 struct kern_nexus *nx = NULL;
1090 struct ifnet *ifp = NULL;
1091 struct nxctl *nxctl;
1092 boolean_t nxctl_locked = FALSE;
1093 errno_t err = 0;
1094
1095 if (ncd == NULL || prov_uuid == NULL || uuid_is_null(prov_uuid) ||
1096 nx_uuid == NULL || init == NULL ||
1097 init->nxneti_version != KERN_NEXUS_NET_CURRENT_VERSION ||
1098 init->nxneti_eparams == NULL || pifp == NULL) {
1099 err = EINVAL;
1100 goto done;
1101 }
1102
1103 /*
1104 * Skywalk native interface doesn't support legacy model.
1105 */
1106 if ((init->nxneti_eparams->start != NULL) ||
1107 (init->nxneti_eparams->flags & IFNET_INIT_LEGACY) ||
1108 (init->nxneti_eparams->flags & IFNET_INIT_INPUT_POLL)) {
1109 err = EINVAL;
1110 goto done;
1111 }
1112
1113 /* create an embryonic ifnet */
1114 err = ifnet_allocate_extended(init->nxneti_eparams, &ifp);
1115 if (err != 0) {
1116 goto done;
1117 }
1118
1119 nxctl = ncd->ncd_nxctl;
1120 NXCTL_LOCK(nxctl);
1121 nxctl_locked = TRUE;
1122
1123 nx = nx_create(nxctl, prov_uuid, NEXUS_TYPE_NET_IF, nx_ctx,
1124 nx_ctx_release, init->nxneti_tx_pbufpool, init->nxneti_rx_pbufpool,
1125 &err);
1126 if (nx == NULL) {
1127 ASSERT(err != 0);
1128 goto done;
1129 }
1130
1131 if (NX_LLINK_PROV(nx)) {
1132 if (init->nxneti_llink == NULL) {
1133 SK_ERR("logical link configuration required");
1134 err = EINVAL;
1135 goto done;
1136 }
1137 err = nx_netif_default_llink_config(NX_NETIF_PRIVATE(nx),
1138 init->nxneti_llink);
1139 if (err != 0) {
1140 goto done;
1141 }
1142 }
1143
1144 /* prepare this ifnet instance if needed */
1145 if (init->nxneti_prepare != NULL) {
1146 err = init->nxneti_prepare(nx, ifp);
1147 if (err != 0) {
1148 goto done;
1149 }
1150 }
1151
1152 /* attach embryonic ifnet to nexus */
1153 err = _kern_nexus_ifattach(nxctl, nx->nx_uuid, ifp, NULL, FALSE, NULL);
1154
1155 if (err != 0) {
1156 goto done;
1157 }
1158
1159 /* and finalize the ifnet attach */
1160 ASSERT(nxctl_locked);
1161 NXCTL_UNLOCK(nxctl);
1162 nxctl_locked = FALSE;
1163
1164 err = ifnet_attach(ifp, init->nxneti_lladdr);
1165 if (err != 0) {
1166 goto done;
1167 }
1168
1169 ASSERT(err == 0);
1170 /*
1171 * Return ifnet reference held by ifnet_allocate_extended();
1172 * caller is expected to retain this reference until its ifnet
1173 * detach callback is called.
1174 */
1175 *pifp = ifp;
1176 uuid_copy(*nx_uuid, nx->nx_uuid);
1177
1178 done:
1179 if (nxctl_locked) {
1180 NXCTL_UNLOCK(nxctl);
1181 }
1182
1183 /* release extra ref from nx_create */
1184 if (nx != NULL) {
1185 SK_LOCK();
1186 if (err != 0) {
1187 (void) nx_close(nx, TRUE);
1188 }
1189 (void) nx_release_locked(nx);
1190 SK_UNLOCK();
1191 }
1192 if (err != 0 && ifp != NULL) {
1193 ifnet_release(ifp);
1194 }
1195
1196 return err;
1197 }
1198
1199 errno_t
kern_nexus_controller_free_provider_instance(const nexus_controller_t ncd,const uuid_t nx_uuid)1200 kern_nexus_controller_free_provider_instance(const nexus_controller_t ncd,
1201 const uuid_t nx_uuid)
1202 {
1203 errno_t err;
1204
1205 if (ncd == NULL || nx_uuid == NULL || uuid_is_null(nx_uuid)) {
1206 err = EINVAL;
1207 } else {
1208 struct nxctl *nxctl = ncd->ncd_nxctl;
1209 NXCTL_LOCK(nxctl);
1210 err = nx_destroy(nxctl, nx_uuid);
1211 NXCTL_UNLOCK(nxctl);
1212 }
1213 return err;
1214 }
1215
1216 errno_t
kern_nexus_controller_bind_provider_instance(const nexus_controller_t ncd,const uuid_t nx_uuid,nexus_port_t * port,const pid_t pid,const uuid_t exec_uuid,const void * key,const uint32_t key_len,const uint32_t bind_flags)1217 kern_nexus_controller_bind_provider_instance(const nexus_controller_t ncd,
1218 const uuid_t nx_uuid, nexus_port_t *port, const pid_t pid,
1219 const uuid_t exec_uuid, const void *key, const uint32_t key_len,
1220 const uint32_t bind_flags)
1221 {
1222 struct nx_bind_req nbr;
1223 struct sockopt sopt;
1224 struct nxctl *nxctl;
1225 int err = 0;
1226
1227 if (ncd == NULL || nx_uuid == NULL || uuid_is_null(nx_uuid) ||
1228 port == NULL) {
1229 return EINVAL;
1230 }
1231
1232 __nexus_bind_req_prepare(&nbr, nx_uuid, *port, pid, exec_uuid,
1233 key, key_len, bind_flags);
1234
1235 bzero(&sopt, sizeof(sopt));
1236 sopt.sopt_dir = SOPT_SET;
1237 sopt.sopt_name = NXOPT_NEXUS_BIND;
1238 sopt.sopt_val = (user_addr_t)&nbr;
1239 sopt.sopt_valsize = sizeof(nbr);
1240 sopt.sopt_p = kernproc;
1241
1242 nxctl = ncd->ncd_nxctl;
1243 NXCTL_LOCK(nxctl);
1244 err = nxctl_set_opt(nxctl, &sopt);
1245 NXCTL_UNLOCK(nxctl);
1246
1247 if (err == 0) {
1248 *port = nbr.nb_port;
1249 }
1250
1251 return err;
1252 }
1253
1254 errno_t
kern_nexus_controller_unbind_provider_instance(const nexus_controller_t ncd,const uuid_t nx_uuid,const nexus_port_t port)1255 kern_nexus_controller_unbind_provider_instance(const nexus_controller_t ncd,
1256 const uuid_t nx_uuid, const nexus_port_t port)
1257 {
1258 struct nx_unbind_req nbu;
1259 struct sockopt sopt;
1260 struct nxctl *nxctl;
1261 int err = 0;
1262
1263 if (ncd == NULL || nx_uuid == NULL || uuid_is_null(nx_uuid)) {
1264 return EINVAL;
1265 }
1266
1267 __nexus_unbind_req_prepare(&nbu, nx_uuid, port);
1268
1269 bzero(&sopt, sizeof(sopt));
1270 sopt.sopt_dir = SOPT_SET;
1271 sopt.sopt_name = NXOPT_NEXUS_UNBIND;
1272 sopt.sopt_val = (user_addr_t)&nbu;
1273 sopt.sopt_valsize = sizeof(nbu);
1274 sopt.sopt_p = kernproc;
1275
1276 nxctl = ncd->ncd_nxctl;
1277 NXCTL_LOCK(nxctl);
1278 err = nxctl_set_opt(nxctl, &sopt);
1279 NXCTL_UNLOCK(nxctl);
1280
1281 return err;
1282 }
1283
1284 errno_t
kern_nexus_controller_read_provider_attr(const nexus_controller_t ncd,const uuid_t prov_uuid,nexus_attr_t nxa)1285 kern_nexus_controller_read_provider_attr(const nexus_controller_t ncd,
1286 const uuid_t prov_uuid, nexus_attr_t nxa)
1287 {
1288 struct nxprov_reg_ent nre;
1289 struct nxprov_params *p = &nre.npre_prov_params;
1290 struct sockopt sopt;
1291 struct nxctl *nxctl;
1292 int err = 0;
1293
1294 if (ncd == NULL || prov_uuid == NULL || uuid_is_null(prov_uuid) ||
1295 nxa == NULL) {
1296 return EINVAL;
1297 }
1298
1299 bzero(&nre, sizeof(nre));
1300 bcopy(prov_uuid, nre.npre_prov_uuid, sizeof(uuid_t));
1301
1302 bzero(&sopt, sizeof(sopt));
1303 sopt.sopt_dir = SOPT_GET;
1304 sopt.sopt_name = NXOPT_NEXUS_PROV_ENTRY;
1305 sopt.sopt_val = (user_addr_t)&nre;
1306 sopt.sopt_valsize = sizeof(nre);
1307 sopt.sopt_p = kernproc;
1308
1309 nxctl = ncd->ncd_nxctl;
1310 NXCTL_LOCK(nxctl);
1311 err = nxctl_get_opt(nxctl, &sopt);
1312 NXCTL_UNLOCK(nxctl);
1313
1314 if (err == 0) {
1315 __nexus_attr_from_params(nxa, p);
1316 }
1317
1318 return err;
1319 }
1320
1321 void
kern_nexus_controller_destroy(nexus_controller_t ncd)1322 kern_nexus_controller_destroy(nexus_controller_t ncd)
1323 {
1324 struct nxctl *nxctl;
1325
1326 if (ncd == NULL) {
1327 return;
1328 }
1329
1330 nxctl = ncd->ncd_nxctl;
1331 ASSERT(nxctl != NULL);
1332 ncd->ncd_nxctl = NULL;
1333 nxctl_dtor(nxctl);
1334
1335 ncd_free(ncd);
1336 }
1337
1338 void *
kern_nexus_get_context(const kern_nexus_t nx)1339 kern_nexus_get_context(const kern_nexus_t nx)
1340 {
1341 return nx->nx_ctx;
1342 }
1343
1344 void
kern_nexus_stop(const kern_nexus_t nx)1345 kern_nexus_stop(const kern_nexus_t nx)
1346 {
1347 SK_LOCK();
1348 nx_stop(nx);
1349 SK_UNLOCK();
1350 }
1351
1352 errno_t
kern_nexus_get_pbufpool(const kern_nexus_t nx,kern_pbufpool_t * ptx_pp,kern_pbufpool_t * prx_pp)1353 kern_nexus_get_pbufpool(const kern_nexus_t nx, kern_pbufpool_t *ptx_pp,
1354 kern_pbufpool_t *prx_pp)
1355 {
1356 kern_pbufpool_t tpp = NULL, rpp = NULL;
1357 int err = 0;
1358
1359 if (ptx_pp == NULL && prx_pp == NULL) {
1360 return EINVAL;
1361 }
1362
1363 if (NX_DOM_PROV(nx)->nxdom_prov_nx_mem_info == NULL) {
1364 err = ENOTSUP;
1365 } else {
1366 err = NX_DOM_PROV(nx)->nxdom_prov_nx_mem_info(nx, &tpp, &rpp);
1367 }
1368
1369 if (ptx_pp != NULL) {
1370 *ptx_pp = tpp;
1371 }
1372 if (prx_pp != NULL) {
1373 *prx_pp = rpp;
1374 }
1375
1376 return err;
1377 }
1378
1379 static int
_kern_nexus_ifattach(struct nxctl * nxctl,const uuid_t nx_uuid,struct ifnet * ifp,const uuid_t nx_uuid_attachee,boolean_t host,uuid_t * nx_if_uuid)1380 _kern_nexus_ifattach(struct nxctl *nxctl, const uuid_t nx_uuid,
1381 struct ifnet *ifp, const uuid_t nx_uuid_attachee, boolean_t host,
1382 uuid_t *nx_if_uuid)
1383 {
1384 struct nx_cfg_req ncr;
1385 struct nx_spec_req nsr;
1386 struct sockopt sopt;
1387 int err = 0;
1388
1389 NXCTL_LOCK_ASSERT_HELD(nxctl);
1390
1391 if (nx_uuid == NULL || uuid_is_null(nx_uuid)) {
1392 return EINVAL;
1393 }
1394
1395 bzero(&nsr, sizeof(nsr));
1396 if (ifp != NULL) {
1397 if (nx_uuid_attachee != NULL) {
1398 return EINVAL;
1399 }
1400
1401 nsr.nsr_flags = NXSPECREQ_IFP;
1402 nsr.nsr_ifp = ifp;
1403 } else {
1404 if (nx_uuid_attachee == NULL) {
1405 return EINVAL;
1406 }
1407
1408 nsr.nsr_flags = NXSPECREQ_UUID;
1409 if (host) {
1410 nsr.nsr_flags |= NXSPECREQ_HOST;
1411 }
1412
1413 uuid_copy(nsr.nsr_uuid, nx_uuid_attachee);
1414 }
1415 __nexus_config_req_prepare(&ncr, nx_uuid, NXCFG_CMD_ATTACH,
1416 &nsr, sizeof(nsr));
1417
1418 bzero(&sopt, sizeof(sopt));
1419 sopt.sopt_dir = SOPT_SET;
1420 sopt.sopt_name = NXOPT_NEXUS_CONFIG;
1421 sopt.sopt_val = (user_addr_t)&ncr;
1422 sopt.sopt_valsize = sizeof(ncr);
1423 sopt.sopt_p = kernproc;
1424
1425 err = nxctl_set_opt(nxctl, &sopt);
1426 if (err == 0 && nx_if_uuid != NULL) {
1427 uuid_copy(*nx_if_uuid, nsr.nsr_if_uuid);
1428 }
1429
1430 return err;
1431 }
1432
1433 int
kern_nexus_ifattach(nexus_controller_t ncd,const uuid_t nx_uuid,struct ifnet * ifp,const uuid_t nx_uuid_attachee,boolean_t host,uuid_t * nx_if_uuid)1434 kern_nexus_ifattach(nexus_controller_t ncd, const uuid_t nx_uuid,
1435 struct ifnet *ifp, const uuid_t nx_uuid_attachee, boolean_t host,
1436 uuid_t *nx_if_uuid)
1437 {
1438 struct nxctl *nxctl;
1439 int err = 0;
1440
1441 if (ncd == NULL) {
1442 return EINVAL;
1443 }
1444
1445 nxctl = ncd->ncd_nxctl;
1446 ASSERT(nxctl != NULL);
1447 NXCTL_LOCK(nxctl);
1448 err = _kern_nexus_ifattach(nxctl, nx_uuid, ifp, nx_uuid_attachee,
1449 host, nx_if_uuid);
1450 NXCTL_UNLOCK(nxctl);
1451
1452 return err;
1453 }
1454
1455 int
kern_nexus_ifdetach(const nexus_controller_t ncd,const uuid_t nx_uuid,const uuid_t nx_if_uuid)1456 kern_nexus_ifdetach(const nexus_controller_t ncd,
1457 const uuid_t nx_uuid, const uuid_t nx_if_uuid)
1458 {
1459 struct nx_cfg_req ncr;
1460 struct nx_spec_req nsr;
1461 struct sockopt sopt;
1462 struct nxctl *nxctl;
1463 int err = 0;
1464
1465 if (ncd == NULL || nx_uuid == NULL || uuid_is_null(nx_uuid) ||
1466 nx_if_uuid == NULL || uuid_is_null(nx_if_uuid)) {
1467 return EINVAL;
1468 }
1469
1470 bzero(&nsr, sizeof(nsr));
1471 uuid_copy(nsr.nsr_if_uuid, nx_if_uuid);
1472
1473 __nexus_config_req_prepare(&ncr, nx_uuid, NXCFG_CMD_DETACH,
1474 &nsr, sizeof(nsr));
1475
1476 bzero(&sopt, sizeof(sopt));
1477 sopt.sopt_dir = SOPT_SET;
1478 sopt.sopt_name = NXOPT_NEXUS_CONFIG;
1479 sopt.sopt_val = (user_addr_t)&ncr;
1480 sopt.sopt_valsize = sizeof(ncr);
1481 sopt.sopt_p = kernproc;
1482
1483 nxctl = ncd->ncd_nxctl;
1484 NXCTL_LOCK(nxctl);
1485 err = nxctl_set_opt(nxctl, &sopt);
1486 NXCTL_UNLOCK(nxctl);
1487
1488 return err;
1489 }
1490
1491 int
kern_nexus_get_netif_instance(struct ifnet * ifp,uuid_t nx_uuid)1492 kern_nexus_get_netif_instance(struct ifnet *ifp, uuid_t nx_uuid)
1493 {
1494 struct nexus_netif_adapter *if_na;
1495 int err = 0;
1496
1497 SK_LOCK();
1498 if_na = ifp->if_na;
1499 if (if_na != NULL) {
1500 uuid_copy(nx_uuid, if_na->nifna_up.na_nx->nx_uuid);
1501 } else {
1502 err = ENXIO;
1503 }
1504 SK_UNLOCK();
1505 if (err != 0) {
1506 uuid_clear(nx_uuid);
1507 }
1508
1509 return err;
1510 }
1511
1512 int
kern_nexus_get_flowswitch_instance(struct ifnet * ifp,uuid_t nx_uuid)1513 kern_nexus_get_flowswitch_instance(struct ifnet *ifp, uuid_t nx_uuid)
1514 {
1515 struct nexus_netif_adapter *if_na;
1516 struct nx_flowswitch *fsw = NULL;
1517 int err = 0;
1518
1519 SK_LOCK();
1520 if_na = ifp->if_na;
1521 if (if_na != NULL) {
1522 fsw = ifp->if_na->nifna_netif->nif_fsw;
1523 }
1524 if (fsw != NULL) {
1525 uuid_copy(nx_uuid, fsw->fsw_nx->nx_uuid);
1526 } else {
1527 err = ENXIO;
1528 }
1529 SK_UNLOCK();
1530 if (err != 0) {
1531 uuid_clear(nx_uuid);
1532 }
1533
1534 return err;
1535 }
1536
1537 static void
kern_nexus_netagent_add(struct kern_nexus * nx,void * arg0)1538 kern_nexus_netagent_add(struct kern_nexus *nx, void *arg0)
1539 {
1540 #pragma unused(arg0)
1541 nx_fsw_netagent_add(nx);
1542 }
1543
1544 static void
kern_nexus_netagent_remove(struct kern_nexus * nx,void * arg0)1545 kern_nexus_netagent_remove(struct kern_nexus *nx, void *arg0)
1546 {
1547 #pragma unused(arg0)
1548 nx_fsw_netagent_remove(nx);
1549 }
1550
1551 static void
kern_nexus_netagent_update(struct kern_nexus * nx,void * arg0)1552 kern_nexus_netagent_update(struct kern_nexus *nx, void *arg0)
1553 {
1554 #pragma unused(arg0)
1555 nx_fsw_netagent_update(nx);
1556 }
1557
1558 void
kern_nexus_register_netagents(void)1559 kern_nexus_register_netagents(void)
1560 {
1561 kern_nexus_walktree(kern_nexus_netagent_add, NULL, FALSE);
1562 }
1563
1564 void
kern_nexus_deregister_netagents(void)1565 kern_nexus_deregister_netagents(void)
1566 {
1567 kern_nexus_walktree(kern_nexus_netagent_remove, NULL, FALSE);
1568 }
1569
1570 void
kern_nexus_update_netagents(void)1571 kern_nexus_update_netagents(void)
1572 {
1573 kern_nexus_walktree(kern_nexus_netagent_update, NULL, FALSE);
1574 }
1575
1576 static int
_interface_add_remove_netagent(struct ifnet * ifp,bool add)1577 _interface_add_remove_netagent(struct ifnet *ifp, bool add)
1578 {
1579 struct nexus_netif_adapter *if_na;
1580 int err = ENXIO;
1581
1582 SK_LOCK();
1583 if_na = ifp->if_na;
1584 if (if_na != NULL) {
1585 struct nx_flowswitch *fsw;
1586
1587 fsw = if_na->nifna_netif->nif_fsw;
1588 if (fsw != NULL) {
1589 if (add) {
1590 err = nx_fsw_netagent_add(fsw->fsw_nx);
1591 } else {
1592 err = nx_fsw_netagent_remove(fsw->fsw_nx);
1593 }
1594 }
1595 }
1596 SK_UNLOCK();
1597 return err;
1598 }
1599
1600 int
kern_nexus_interface_add_netagent(struct ifnet * ifp)1601 kern_nexus_interface_add_netagent(struct ifnet *ifp)
1602 {
1603 return _interface_add_remove_netagent(ifp, true);
1604 }
1605
1606 int
kern_nexus_interface_remove_netagent(struct ifnet * ifp)1607 kern_nexus_interface_remove_netagent(struct ifnet *ifp)
1608 {
1609 return _interface_add_remove_netagent(ifp, false);
1610 }
1611
1612 int
kern_nexus_set_netif_input_tbr_rate(struct ifnet * ifp,uint64_t rate)1613 kern_nexus_set_netif_input_tbr_rate(struct ifnet *ifp, uint64_t rate)
1614 {
1615 /* input tbr is only functional with active netif attachment */
1616 if (ifp->if_na == NULL) {
1617 if (rate != 0) {
1618 return EINVAL;
1619 } else {
1620 return 0;
1621 }
1622 }
1623
1624 ifp->if_na->nifna_netif->nif_input_rate = rate;
1625 return 0;
1626 }
1627
1628 int
kern_nexus_set_if_netem_params(const nexus_controller_t ncd,const uuid_t nx_uuid,void * data,size_t data_len)1629 kern_nexus_set_if_netem_params(const nexus_controller_t ncd,
1630 const uuid_t nx_uuid, void *data, size_t data_len)
1631 {
1632 struct nx_cfg_req ncr;
1633 struct sockopt sopt;
1634 struct nxctl *nxctl;
1635 int err = 0;
1636
1637 if (nx_uuid == NULL || uuid_is_null(nx_uuid) ||
1638 data_len < sizeof(struct if_netem_params)) {
1639 return EINVAL;
1640 }
1641
1642 __nexus_config_req_prepare(&ncr, nx_uuid, NXCFG_CMD_NETEM,
1643 data, data_len);
1644 bzero(&sopt, sizeof(sopt));
1645 sopt.sopt_dir = SOPT_SET;
1646 sopt.sopt_name = NXOPT_NEXUS_CONFIG;
1647 sopt.sopt_val = (user_addr_t)&ncr;
1648 sopt.sopt_valsize = sizeof(ncr);
1649 sopt.sopt_p = kernproc;
1650
1651 nxctl = ncd->ncd_nxctl;
1652 NXCTL_LOCK(nxctl);
1653 err = nxctl_set_opt(nxctl, &sopt);
1654 NXCTL_UNLOCK(nxctl);
1655
1656 return err;
1657 }
1658
1659 static int
_kern_nexus_flow_config(const nexus_controller_t ncd,const uuid_t nx_uuid,const nxcfg_cmd_t cmd,void * data,size_t data_len)1660 _kern_nexus_flow_config(const nexus_controller_t ncd, const uuid_t nx_uuid,
1661 const nxcfg_cmd_t cmd, void *data, size_t data_len)
1662 {
1663 struct nx_cfg_req ncr;
1664 struct sockopt sopt;
1665 struct nxctl *nxctl;
1666 int err = 0;
1667
1668 if (nx_uuid == NULL || uuid_is_null(nx_uuid) ||
1669 data_len < sizeof(struct nx_flow_req)) {
1670 return EINVAL;
1671 }
1672
1673 __nexus_config_req_prepare(&ncr, nx_uuid, cmd, data, data_len);
1674
1675 bzero(&sopt, sizeof(sopt));
1676 sopt.sopt_dir = SOPT_SET;
1677 sopt.sopt_name = NXOPT_NEXUS_CONFIG;
1678 sopt.sopt_val = (user_addr_t)&ncr;
1679 sopt.sopt_valsize = sizeof(ncr);
1680 sopt.sopt_p = kernproc;
1681
1682 nxctl = ncd->ncd_nxctl;
1683 NXCTL_LOCK(nxctl);
1684 err = nxctl_set_opt(nxctl, &sopt);
1685 NXCTL_UNLOCK(nxctl);
1686
1687 return err;
1688 }
1689
1690 int
kern_nexus_flow_add(const nexus_controller_t ncd,const uuid_t nx_uuid,void * data,size_t data_len)1691 kern_nexus_flow_add(const nexus_controller_t ncd, const uuid_t nx_uuid,
1692 void *data, size_t data_len)
1693 {
1694 return _kern_nexus_flow_config(ncd, nx_uuid, NXCFG_CMD_FLOW_ADD, data,
1695 data_len);
1696 }
1697
1698 int
kern_nexus_flow_del(const nexus_controller_t ncd,const uuid_t nx_uuid,void * data,size_t data_len)1699 kern_nexus_flow_del(const nexus_controller_t ncd, const uuid_t nx_uuid,
1700 void *data, size_t data_len)
1701 {
1702 return _kern_nexus_flow_config(ncd, nx_uuid, NXCFG_CMD_FLOW_DEL, data,
1703 data_len);
1704 }
1705
1706 static struct kern_nexus_domain_provider *
nxdom_prov_alloc(zalloc_flags_t how)1707 nxdom_prov_alloc(zalloc_flags_t how)
1708 {
1709 SK_LOCK_ASSERT_HELD();
1710
1711 return zalloc_flags(nxdom_prov_zone, how | Z_ZERO);
1712 }
1713
1714 static void
nxdom_prov_free(struct kern_nexus_domain_provider * nxdom_prov)1715 nxdom_prov_free(struct kern_nexus_domain_provider *nxdom_prov)
1716 {
1717 SK_LOCK_ASSERT_HELD();
1718
1719 ASSERT(nxdom_prov->nxdom_prov_refcnt == 0);
1720 ASSERT(!(nxdom_prov->nxdom_prov_flags &
1721 (NXDOMPROVF_ATTACHED | NXDOMPROVF_DETACHING)));
1722
1723 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_INITIALIZED) {
1724 /*
1725 * Tell the domain provider that we're done with this
1726 * instance, and it is now free to go away.
1727 */
1728 if (nxdom_prov->nxdom_prov_fini != NULL) {
1729 nxdom_prov->nxdom_prov_fini(nxdom_prov);
1730 }
1731 nxdom_prov->nxdom_prov_flags &= ~NXDOMPROVF_INITIALIZED;
1732 }
1733 uuid_clear(nxdom_prov->nxdom_prov_uuid);
1734 nxdom_prov->nxdom_prov_dom = NULL;
1735
1736 SK_DF(SK_VERB_MEM, "nxdom_prov 0x%llx %s", SK_KVA(nxdom_prov),
1737 ((nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT) ?
1738 "FREE" : "DESTROY"));
1739 if (nxdom_prov->nxdom_prov_flags & NXDOMPROVF_EXT) {
1740 zfree(nxdom_prov_zone, nxdom_prov);
1741 }
1742 }
1743
1744 void
nxdom_prov_retain_locked(struct kern_nexus_domain_provider * nxdom_prov)1745 nxdom_prov_retain_locked(struct kern_nexus_domain_provider *nxdom_prov)
1746 {
1747 SK_LOCK_ASSERT_HELD();
1748
1749 nxdom_prov->nxdom_prov_refcnt++;
1750 ASSERT(nxdom_prov->nxdom_prov_refcnt != 0);
1751 }
1752
1753 void
nxdom_prov_retain(struct kern_nexus_domain_provider * nxdom_prov)1754 nxdom_prov_retain(struct kern_nexus_domain_provider *nxdom_prov)
1755 {
1756 SK_LOCK();
1757 nxdom_prov_retain_locked(nxdom_prov);
1758 SK_UNLOCK();
1759 }
1760
1761 static int
nxdom_prov_params_default(struct kern_nexus_domain_provider * nxdom_prov,const uint32_t req,const struct nxprov_params * nxp0,struct nxprov_params * nxp,struct skmem_region_params srp[SKMEM_REGIONS],uint32_t pp_region_config_flags)1762 nxdom_prov_params_default(struct kern_nexus_domain_provider *nxdom_prov,
1763 const uint32_t req, const struct nxprov_params *nxp0,
1764 struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS],
1765 uint32_t pp_region_config_flags)
1766 {
1767 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
1768
1769 return nxprov_params_adjust(nxdom_prov, req, nxp0, nxp, srp,
1770 nxdom, nxdom, nxdom, pp_region_config_flags, NULL);
1771 }
1772
1773 int
nxdom_prov_validate_params(struct kern_nexus_domain_provider * nxdom_prov,const struct nxprov_reg * reg,struct nxprov_params * nxp,struct skmem_region_params srp[SKMEM_REGIONS],const uint32_t oflags,uint32_t pp_region_config_flags)1774 nxdom_prov_validate_params(struct kern_nexus_domain_provider *nxdom_prov,
1775 const struct nxprov_reg *reg, struct nxprov_params *nxp,
1776 struct skmem_region_params srp[SKMEM_REGIONS], const uint32_t oflags,
1777 uint32_t pp_region_config_flags)
1778 {
1779 const struct nxprov_params *nxp0 = ®->nxpreg_params;
1780 const uint32_t req = reg->nxpreg_requested;
1781 int i, err = 0;
1782
1783 ASSERT(reg->nxpreg_version == NXPROV_REG_CURRENT_VERSION &&
1784 nxp0->nxp_namelen != 0 &&
1785 nxp0->nxp_namelen <= sizeof(nexus_name_t));
1786
1787 /* fill in with default values and let the nexus override them */
1788 bzero(nxp, sizeof(*nxp));
1789 bcopy(&nxp0->nxp_name, &nxp->nxp_name, sizeof(nxp->nxp_name));
1790 nxp->nxp_name[sizeof(nxp->nxp_name) - 1] = '\0';
1791 nxp->nxp_namelen = nxp0->nxp_namelen;
1792 nxp->nxp_type = nxp0->nxp_type;
1793 nxp->nxp_md_type = nxdom_prov->nxdom_prov_dom->nxdom_md_type;
1794 nxp->nxp_md_subtype = nxdom_prov->nxdom_prov_dom->nxdom_md_subtype;
1795 nxp->nxp_flags = (nxp0->nxp_flags & NXPF_MASK);
1796 nxp->nxp_flags |= oflags; /* override */
1797 nxp->nxp_format = nxp0->nxp_format;
1798 nxp->nxp_ifindex = nxp0->nxp_ifindex;
1799 nxp->nxp_reject_on_close = nxp0->nxp_reject_on_close;
1800
1801 /* inherit default region parameters */
1802 for (i = 0; i < SKMEM_REGIONS; i++) {
1803 srp[i] = *skmem_get_default(i);
1804 }
1805
1806 if (nxdom_prov->nxdom_prov_params != NULL) {
1807 err = nxdom_prov->nxdom_prov_params(nxdom_prov, req, nxp0,
1808 nxp, srp, pp_region_config_flags);
1809 } else {
1810 err = nxdom_prov_params_default(nxdom_prov, req, nxp0,
1811 nxp, srp, pp_region_config_flags);
1812 }
1813 return err;
1814 }
1815
1816 boolean_t
nxdom_prov_release_locked(struct kern_nexus_domain_provider * nxdom_prov)1817 nxdom_prov_release_locked(struct kern_nexus_domain_provider *nxdom_prov)
1818 {
1819 int oldref = nxdom_prov->nxdom_prov_refcnt;
1820
1821 SK_LOCK_ASSERT_HELD();
1822
1823 ASSERT(nxdom_prov->nxdom_prov_refcnt != 0);
1824 if (--nxdom_prov->nxdom_prov_refcnt == 0) {
1825 nxdom_prov_free(nxdom_prov);
1826 }
1827
1828 return oldref == 1;
1829 }
1830
1831 boolean_t
nxdom_prov_release(struct kern_nexus_domain_provider * nxdom_prov)1832 nxdom_prov_release(struct kern_nexus_domain_provider *nxdom_prov)
1833 {
1834 boolean_t lastref;
1835
1836 SK_LOCK();
1837 lastref = nxdom_prov_release_locked(nxdom_prov);
1838 SK_UNLOCK();
1839
1840 return lastref;
1841 }
1842
1843 static uint32_t
nxprov_bound_var(uint32_t * v,uint32_t dflt,uint32_t lo,uint32_t hi,const char * msg)1844 nxprov_bound_var(uint32_t *v, uint32_t dflt, uint32_t lo, uint32_t hi,
1845 const char *msg)
1846 {
1847 #pragma unused(msg)
1848 uint32_t oldv = *v;
1849 const char *op = NULL;
1850
1851 if (dflt < lo) {
1852 dflt = lo;
1853 }
1854 if (dflt > hi) {
1855 dflt = hi;
1856 }
1857 if (oldv < lo) {
1858 *v = dflt;
1859 op = "bump";
1860 } else if (oldv > hi) {
1861 *v = hi;
1862 op = "clamp";
1863 }
1864 #if SK_LOG
1865 if (op != NULL && msg != NULL) {
1866 SK_ERR("%s %s to %u (was %u)", op, msg, *v, oldv);
1867 }
1868 #endif /* SK_LOG */
1869 return *v;
1870 }
1871
1872 #define NXPROV_PARAMS_ADJUST(flag, param) do { \
1873 uint32_t _v0, _v; \
1874 if (req & (flag)) \
1875 _v = nxp0->nxp_##param; \
1876 else \
1877 _v = NXDOM_DEF(nxdom_def, param); \
1878 _v0 = _v; \
1879 if (nxprov_bound_var(&_v, NXDOM_DEF(nxdom_def, param), \
1880 NXDOM_MIN(nxdom_min, param), NXDOM_MAX(nxdom_max, param), \
1881 "nxp_" #param) < _v0) { \
1882 err = ENOMEM; \
1883 goto error; \
1884 } \
1885 nxp->nxp_##param = _v; \
1886 } while (0)
1887
1888 #define MUL(x, y, z) do { \
1889 if (__builtin_mul_overflow((x), (y), (z))) { \
1890 overflowline = __LINE__; \
1891 goto error; \
1892 } \
1893 } while (0)
1894
1895 #define ADD(x, y, z) do { \
1896 if (__builtin_add_overflow((x), (y), (z))) { \
1897 overflowline = __LINE__; \
1898 goto error; \
1899 } \
1900 } while (0)
1901
1902 int
nxprov_params_adjust(struct kern_nexus_domain_provider * nxdom_prov,const uint32_t req,const struct nxprov_params * nxp0,struct nxprov_params * nxp,struct skmem_region_params srp[SKMEM_REGIONS],const struct nxdom * nxdom_def,const struct nxdom * nxdom_min,const struct nxdom * nxdom_max,uint32_t pp_region_config_flags,int (* adjust_fn)(const struct kern_nexus_domain_provider *,const struct nxprov_params *,struct nxprov_adjusted_params *))1903 nxprov_params_adjust(struct kern_nexus_domain_provider *nxdom_prov,
1904 const uint32_t req, const struct nxprov_params *nxp0,
1905 struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS],
1906 const struct nxdom *nxdom_def, const struct nxdom *nxdom_min,
1907 const struct nxdom *nxdom_max, uint32_t pp_region_config_flags,
1908 int (*adjust_fn)(const struct kern_nexus_domain_provider *,
1909 const struct nxprov_params *, struct nxprov_adjusted_params *))
1910 {
1911 uint32_t buf_cnt;
1912 uint32_t stats_size;
1913 uint32_t flowadv_max;
1914 uint32_t nexusadv_size;
1915 uint32_t capabs;
1916 uint32_t tx_rings, rx_rings;
1917 uint32_t alloc_rings = 0, free_rings = 0, ev_rings = 0;
1918 uint32_t tx_slots, rx_slots;
1919 uint32_t alloc_slots = 0, free_slots = 0, ev_slots = 0;
1920 uint32_t buf_size, buf_region_segment_size, max_buffers = 0;
1921 uint32_t tmp1, tmp2, tmp3, tmp4xpipes, tmpsumrings;
1922 uint32_t tmpsumall, tmp4xpipesplusrings;
1923 uint32_t large_buf_size;
1924 int overflowline = 0;
1925 int err = 0;
1926
1927 NXPROV_PARAMS_ADJUST(NXPREQ_TX_RINGS, tx_rings);
1928 NXPROV_PARAMS_ADJUST(NXPREQ_RX_RINGS, rx_rings);
1929 NXPROV_PARAMS_ADJUST(NXPREQ_TX_SLOTS, tx_slots);
1930 NXPROV_PARAMS_ADJUST(NXPREQ_RX_SLOTS, rx_slots);
1931 NXPROV_PARAMS_ADJUST(NXPREQ_BUF_SIZE, buf_size);
1932 NXPROV_PARAMS_ADJUST(NXPREQ_LARGE_BUF_SIZE, large_buf_size);
1933 NXPROV_PARAMS_ADJUST(NXPREQ_STATS_SIZE, stats_size);
1934 NXPROV_PARAMS_ADJUST(NXPREQ_FLOWADV_MAX, flowadv_max);
1935 NXPROV_PARAMS_ADJUST(NXPREQ_NEXUSADV_SIZE, nexusadv_size);
1936 NXPROV_PARAMS_ADJUST(NXPREQ_PIPES, pipes);
1937 NXPROV_PARAMS_ADJUST(NXPREQ_EXTENSIONS, extensions);
1938 NXPROV_PARAMS_ADJUST(NXPREQ_MHINTS, mhints);
1939 NXPROV_PARAMS_ADJUST(NXPREQ_CAPABILITIES, capabilities);
1940 NXPROV_PARAMS_ADJUST(NXPREQ_QMAP, qmap);
1941 NXPROV_PARAMS_ADJUST(NXPREQ_MAX_FRAGS, max_frags);
1942
1943 capabs = NXDOM_DEF(nxdom_def, capabilities);
1944 if (req & NXPREQ_USER_CHANNEL) {
1945 if (nxp->nxp_flags & NXPF_USER_CHANNEL) {
1946 capabs |= NXPCAP_USER_CHANNEL;
1947 } else {
1948 capabs &= ~NXPCAP_USER_CHANNEL;
1949 }
1950 } else {
1951 if (capabs & NXPCAP_USER_CHANNEL) {
1952 nxp->nxp_flags |= NXPF_USER_CHANNEL;
1953 } else {
1954 nxp->nxp_flags &= ~NXPF_USER_CHANNEL;
1955 }
1956 }
1957
1958 if (NXDOM_MIN(nxdom_min, capabilities) != 0 &&
1959 !(capabs & NXDOM_MIN(nxdom_min, capabilities))) {
1960 SK_ERR("%s: caps 0x%b < min 0x%b",
1961 nxdom_prov->nxdom_prov_name, capabs, NXPCAP_BITS,
1962 NXDOM_MIN(nxdom_min, capabilities), NXPCAP_BITS);
1963 err = EINVAL;
1964 goto error;
1965 } else if (NXDOM_MAX(nxdom_max, capabilities) != 0 &&
1966 (capabs & ~NXDOM_MAX(nxdom_max, capabilities))) {
1967 SK_ERR("%s: caps 0x%b > max 0x%b",
1968 nxdom_prov->nxdom_prov_name, capabs, NXPCAP_BITS,
1969 NXDOM_MAX(nxdom_max, capabilities), NXPCAP_BITS);
1970 err = EINVAL;
1971 goto error;
1972 }
1973
1974 stats_size = nxp->nxp_stats_size;
1975 flowadv_max = nxp->nxp_flowadv_max;
1976 nexusadv_size = nxp->nxp_nexusadv_size;
1977 tx_rings = nxp->nxp_tx_rings;
1978 rx_rings = nxp->nxp_rx_rings;
1979 tx_slots = nxp->nxp_tx_slots;
1980 rx_slots = nxp->nxp_rx_slots;
1981 buf_size = nxp->nxp_buf_size;
1982 large_buf_size = nxp->nxp_large_buf_size;
1983 buf_region_segment_size = skmem_usr_buf_seg_size;
1984 ASSERT(pp_region_config_flags & PP_REGION_CONFIG_MD_MAGAZINE_ENABLE);
1985
1986 if (adjust_fn != NULL) {
1987 struct nxprov_adjusted_params adj = {
1988 .adj_md_subtype = &nxp->nxp_md_subtype,
1989 .adj_stats_size = &stats_size,
1990 .adj_flowadv_max = &flowadv_max,
1991 .adj_nexusadv_size = &nexusadv_size,
1992 .adj_caps = &capabs,
1993 .adj_tx_rings = &tx_rings,
1994 .adj_rx_rings = &rx_rings,
1995 .adj_tx_slots = &tx_slots,
1996 .adj_rx_slots = &rx_slots,
1997 .adj_alloc_rings = &alloc_rings,
1998 .adj_free_rings = &free_rings,
1999 .adj_alloc_slots = &alloc_slots,
2000 .adj_free_slots = &free_slots,
2001 .adj_buf_size = &buf_size,
2002 .adj_buf_region_segment_size = &buf_region_segment_size,
2003 .adj_pp_region_config_flags = &pp_region_config_flags,
2004 .adj_max_frags = &nxp->nxp_max_frags,
2005 .adj_event_rings = &ev_rings,
2006 .adj_event_slots = &ev_slots,
2007 .adj_max_buffers = &max_buffers,
2008 .adj_large_buf_size = &large_buf_size,
2009 };
2010 err = adjust_fn(nxdom_prov, nxp, &adj);
2011 if (err != 0) {
2012 goto error;
2013 }
2014
2015 ASSERT(capabs >= NXDOM_MIN(nxdom_min, capabilities));
2016 ASSERT(capabs <= NXDOM_MAX(nxdom_max, capabilities));
2017 }
2018
2019 if (nxp->nxp_max_frags > UINT16_MAX) {
2020 SK_ERR("invalid configuration for max frags %d",
2021 nxp->nxp_max_frags);
2022 err = EINVAL;
2023 }
2024
2025 if (nxp->nxp_type == NEXUS_TYPE_USER_PIPE) {
2026 if (tx_rings != rx_rings) {
2027 SK_ERR("invalid configuration: {rx,tx} rings must be"
2028 "in pairs for user pipe rx_rings(%d) tx_rings(%d)",
2029 rx_rings, tx_rings);
2030 err = EINVAL;
2031 }
2032 } else {
2033 if (nxp->nxp_pipes != 0) {
2034 SK_ERR("invalid configuration: pipe configuration is"
2035 "only valid for user pipe nexus, type %d, pipes %d",
2036 nxp->nxp_type, nxp->nxp_pipes);
2037 err = EINVAL;
2038 }
2039 }
2040 if (err != 0) {
2041 goto error;
2042 }
2043
2044 /* leading and trailing guard pages (if applicable) */
2045 if (sk_guard) {
2046 srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_size = SKMEM_PAGE_SIZE;
2047 srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_cnt = sk_headguard_sz;
2048 skmem_region_params_config(&srp[SKMEM_REGION_GUARD_HEAD]);
2049 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_size = SKMEM_PAGE_SIZE;
2050 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_cnt = sk_tailguard_sz;
2051 skmem_region_params_config(&srp[SKMEM_REGION_GUARD_TAIL]);
2052 } else {
2053 srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_size = 0;
2054 srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_cnt = 0;
2055 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_size = 0;
2056 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_cnt = 0;
2057 }
2058
2059 /* update to the adjusted/configured values */
2060 nxp->nxp_buf_size = buf_size;
2061 nxp->nxp_tx_slots = tx_slots;
2062 nxp->nxp_rx_slots = rx_slots;
2063 nxp->nxp_large_buf_size = large_buf_size;
2064
2065 SK_D("nxdom \"%s\" (0x%llx) type %d",
2066 nxdom_prov->nxdom_prov_dom->nxdom_name,
2067 SK_KVA(nxdom_prov->nxdom_prov_dom),
2068 nxdom_prov->nxdom_prov_dom->nxdom_type);
2069 SK_D("nxp \"%s\" (0x%llx) flags 0x%b",
2070 nxp->nxp_name, SK_KVA(nxp), nxp->nxp_flags, NXPF_BITS);
2071 SK_D(" req 0x%b rings %u/%u/%u/%u/%u slots %u/%u/%u/%u/%u buf %u "
2072 "type %u subtype %u stats %u flowadv_max %u nexusadv_size %u "
2073 "capabs 0x%b pipes %u extensions %u max_frags %u headguard %u "
2074 "tailguard %u large_buf %u", req, NXPREQ_BITS, tx_rings, rx_rings,
2075 alloc_rings, free_rings, ev_rings, tx_slots, rx_slots, alloc_slots,
2076 free_slots, ev_slots, nxp->nxp_buf_size, nxp->nxp_md_type,
2077 nxp->nxp_md_subtype, stats_size, flowadv_max, nexusadv_size,
2078 capabs, NXPCAP_BITS, nxp->nxp_pipes, nxp->nxp_extensions,
2079 nxp->nxp_max_frags, srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_size *
2080 srp[SKMEM_REGION_GUARD_HEAD].srp_r_obj_cnt,
2081 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_size *
2082 srp[SKMEM_REGION_GUARD_TAIL].srp_r_obj_cnt,
2083 nxp->nxp_large_buf_size);
2084
2085 /*
2086 * tmp4xpipes = 4 * nxp->nxp_pipes
2087 */
2088 MUL(4, nxp->nxp_pipes, &tmp4xpipes);
2089
2090 /*
2091 * tmp4xpipesplusrings = tx_rings + (4 * nxp->nxp_pipes)
2092 */
2093 VERIFY((tmp4xpipes == 0) || (rx_rings == tx_rings));
2094 ADD(tx_rings, tmp4xpipes, &tmp4xpipesplusrings);
2095
2096 /*
2097 * tmpsumrings = tx_rings + rx_rings + alloc_rings + free_rings +
2098 * ev_rings
2099 */
2100 ADD(tx_rings, rx_rings, &tmpsumrings);
2101 ADD(tmpsumrings, alloc_rings, &tmpsumrings);
2102 ADD(tmpsumrings, free_rings, &tmpsumrings);
2103 ADD(tmpsumrings, ev_rings, &tmpsumrings);
2104
2105 /*
2106 * tmpsumall = (tx_rings + rx_rings +
2107 * alloc_rings + free_rings + ev_rings + (4 * nxp->nxp_pipes))
2108 */
2109 ADD(tmpsumrings, tmp4xpipes, &tmpsumall);
2110
2111 /* possibly increase them to fit user request */
2112 VERIFY(CHANNEL_SCHEMA_SIZE(tmpsumrings) <= UINT32_MAX);
2113 srp[SKMEM_REGION_SCHEMA].srp_r_obj_size =
2114 (uint32_t)CHANNEL_SCHEMA_SIZE(tmpsumrings);
2115 /* worst case is one channel bound to each ring pair */
2116 srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = tmp4xpipesplusrings;
2117
2118 skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]);
2119
2120 srp[SKMEM_REGION_RING].srp_r_obj_size =
2121 sizeof(struct __user_channel_ring);
2122 /* each pipe endpoint needs two tx rings and two rx rings */
2123 srp[SKMEM_REGION_RING].srp_r_obj_cnt = tmpsumall;
2124 skmem_region_params_config(&srp[SKMEM_REGION_RING]);
2125
2126 /*
2127 * For each pipe we only need the buffers for the "real" rings.
2128 * On the other end, the pipe ring dimension may be different from
2129 * the parent port ring dimension. As a compromise, we allocate twice
2130 * the space actually needed if the pipe rings were the same size as
2131 * the parent rings.
2132 *
2133 * buf_cnt = ((4 * nxp->nxp_pipes) + rx_rings) * rx_slots +
2134 * ((4 * nxp->nxp_pipes) + tx_rings) * tx_slots +
2135 * (ev_rings * ev_slots);
2136 */
2137 if (nxp->nxp_type == NEXUS_TYPE_USER_PIPE) {
2138 MUL(tmp4xpipesplusrings, rx_slots, &tmp1);
2139 MUL(tmp4xpipesplusrings, tx_slots, &tmp2);
2140 ASSERT(ev_rings == 0);
2141 tmp3 = 0;
2142 } else {
2143 MUL(rx_rings, rx_slots, &tmp1);
2144 MUL(tx_rings, tx_slots, &tmp2);
2145 MUL(ev_rings, ev_slots, &tmp3);
2146 }
2147 ADD(tmp1, tmp2, &buf_cnt);
2148 ADD(tmp3, buf_cnt, &buf_cnt);
2149
2150 if (nxp->nxp_max_frags > 1) {
2151 pp_region_config_flags |= PP_REGION_CONFIG_BUFLET;
2152 buf_cnt = MIN((((uint32_t)P2ROUNDUP(NX_MAX_AGGR_PKT_SIZE,
2153 nxp->nxp_buf_size) / nxp->nxp_buf_size) * buf_cnt),
2154 (buf_cnt * nxp->nxp_max_frags));
2155 }
2156
2157 if (max_buffers != 0) {
2158 buf_cnt = MIN(max_buffers, buf_cnt);
2159 }
2160
2161 if ((nxp->nxp_flags & NXPF_USER_CHANNEL) == 0) {
2162 pp_region_config_flags |= PP_REGION_CONFIG_KERNEL_ONLY;
2163 }
2164
2165 /* # of metadata objects is same as the # of buffer objects */
2166 ASSERT(buf_region_segment_size != 0);
2167 pp_regions_params_adjust(srp, nxp->nxp_md_type, nxp->nxp_md_subtype,
2168 buf_cnt, (uint16_t)nxp->nxp_max_frags, nxp->nxp_buf_size,
2169 nxp->nxp_large_buf_size, buf_cnt, buf_region_segment_size,
2170 pp_region_config_flags);
2171
2172 /* statistics region size */
2173 if (stats_size != 0) {
2174 srp[SKMEM_REGION_USTATS].srp_r_obj_size = stats_size;
2175 srp[SKMEM_REGION_USTATS].srp_r_obj_cnt = 1;
2176 skmem_region_params_config(&srp[SKMEM_REGION_USTATS]);
2177 } else {
2178 srp[SKMEM_REGION_USTATS].srp_r_obj_size = 0;
2179 srp[SKMEM_REGION_USTATS].srp_r_obj_cnt = 0;
2180 srp[SKMEM_REGION_USTATS].srp_c_obj_size = 0;
2181 srp[SKMEM_REGION_USTATS].srp_c_obj_cnt = 0;
2182 }
2183
2184 /* flow advisory region size */
2185 if (flowadv_max != 0) {
2186 _CASSERT(NX_FLOWADV_DEFAULT * sizeof(struct __flowadv_entry) <=
2187 SKMEM_MIN_SEG_SIZE);
2188 MUL(sizeof(struct __flowadv_entry), flowadv_max, &tmp1);
2189 srp[SKMEM_REGION_FLOWADV].srp_r_obj_size = tmp1;
2190 srp[SKMEM_REGION_FLOWADV].srp_r_obj_cnt = 1;
2191 skmem_region_params_config(&srp[SKMEM_REGION_FLOWADV]);
2192 } else {
2193 srp[SKMEM_REGION_FLOWADV].srp_r_obj_size = 0;
2194 srp[SKMEM_REGION_FLOWADV].srp_r_obj_cnt = 0;
2195 srp[SKMEM_REGION_FLOWADV].srp_c_obj_size = 0;
2196 srp[SKMEM_REGION_FLOWADV].srp_c_obj_cnt = 0;
2197 }
2198
2199 /* nexus advisory region size */
2200 if (nexusadv_size != 0) {
2201 srp[SKMEM_REGION_NEXUSADV].srp_r_obj_size = nexusadv_size +
2202 sizeof(struct __kern_nexus_adv_metadata);
2203 srp[SKMEM_REGION_NEXUSADV].srp_r_obj_cnt = 1;
2204 skmem_region_params_config(&srp[SKMEM_REGION_NEXUSADV]);
2205 } else {
2206 srp[SKMEM_REGION_NEXUSADV].srp_r_obj_size = 0;
2207 srp[SKMEM_REGION_NEXUSADV].srp_r_obj_cnt = 0;
2208 srp[SKMEM_REGION_NEXUSADV].srp_c_obj_size = 0;
2209 srp[SKMEM_REGION_NEXUSADV].srp_c_obj_cnt = 0;
2210 }
2211
2212 /* sysctls region is not applicable to nexus */
2213 srp[SKMEM_REGION_SYSCTLS].srp_r_obj_size = 0;
2214 srp[SKMEM_REGION_SYSCTLS].srp_r_obj_cnt = 0;
2215 srp[SKMEM_REGION_SYSCTLS].srp_c_obj_size = 0;
2216 srp[SKMEM_REGION_SYSCTLS].srp_c_obj_cnt = 0;
2217
2218 /*
2219 * Since the tx/alloc/event slots share the same region and cache,
2220 * we will use the same object size for both types of slots.
2221 */
2222 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size =
2223 (MAX(MAX(tx_slots, alloc_slots), ev_slots)) * SLOT_DESC_SZ;
2224 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt = tx_rings + alloc_rings +
2225 ev_rings;
2226 skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]);
2227
2228 /* USD and KSD objects share the same size and count */
2229 srp[SKMEM_REGION_TXAUSD].srp_r_obj_size =
2230 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size;
2231 srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt =
2232 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt;
2233 skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]);
2234
2235 /*
2236 * Since the rx/free slots share the same region and cache,
2237 * we will use the same object size for both types of slots.
2238 */
2239 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size =
2240 MAX(rx_slots, free_slots) * SLOT_DESC_SZ;
2241 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt = rx_rings + free_rings;
2242 skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]);
2243
2244 /* USD and KSD objects share the same size and count */
2245 srp[SKMEM_REGION_RXFUSD].srp_r_obj_size =
2246 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size;
2247 srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt =
2248 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt;
2249 skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]);
2250
2251 /* update these based on the adjusted/configured values */
2252 nxp->nxp_meta_size = srp[SKMEM_REGION_KMD].srp_c_obj_size;
2253 nxp->nxp_stats_size = stats_size;
2254 nxp->nxp_flowadv_max = flowadv_max;
2255 nxp->nxp_nexusadv_size = nexusadv_size;
2256 nxp->nxp_capabilities = capabs;
2257
2258 error:
2259 if (overflowline) {
2260 err = EOVERFLOW;
2261 SK_ERR("math overflow in %s on line %d",
2262 __func__, overflowline);
2263 }
2264 return err;
2265 }
2266
2267 #undef ADD
2268 #undef MUL
2269 #undef NXPROV_PARAMS_ADJUST
2270
2271 static void
nxprov_detaching_enqueue(struct kern_nexus_domain_provider * nxdom_prov)2272 nxprov_detaching_enqueue(struct kern_nexus_domain_provider *nxdom_prov)
2273 {
2274 SK_LOCK_ASSERT_HELD();
2275
2276 ASSERT((nxdom_prov->nxdom_prov_flags & (NXDOMPROVF_ATTACHED |
2277 NXDOMPROVF_DETACHING)) == NXDOMPROVF_DETACHING);
2278
2279 ++nxprov_detaching_cnt;
2280 ASSERT(nxprov_detaching_cnt != 0);
2281 /*
2282 * Insert this to the detaching list; caller is expected to
2283 * have held a reference, most likely the same one that was
2284 * used for the per-domain provider list.
2285 */
2286 STAILQ_INSERT_TAIL(&nxprov_detaching_head, nxdom_prov,
2287 nxdom_prov_detaching_link);
2288 wakeup((caddr_t)&nxprov_detach_wchan);
2289 }
2290
2291 static struct kern_nexus_domain_provider *
nxprov_detaching_dequeue(void)2292 nxprov_detaching_dequeue(void)
2293 {
2294 struct kern_nexus_domain_provider *nxdom_prov;
2295
2296 SK_LOCK_ASSERT_HELD();
2297
2298 nxdom_prov = STAILQ_FIRST(&nxprov_detaching_head);
2299 ASSERT(nxprov_detaching_cnt != 0 || nxdom_prov == NULL);
2300 if (nxdom_prov != NULL) {
2301 ASSERT((nxdom_prov->nxdom_prov_flags & (NXDOMPROVF_ATTACHED |
2302 NXDOMPROVF_DETACHING)) == NXDOMPROVF_DETACHING);
2303 ASSERT(nxprov_detaching_cnt != 0);
2304 --nxprov_detaching_cnt;
2305 STAILQ_REMOVE(&nxprov_detaching_head, nxdom_prov,
2306 kern_nexus_domain_provider, nxdom_prov_detaching_link);
2307 }
2308 return nxdom_prov;
2309 }
2310
2311 __attribute__((noreturn))
2312 static void
nxprov_detacher(void * v,wait_result_t w)2313 nxprov_detacher(void *v, wait_result_t w)
2314 {
2315 #pragma unused(v, w)
2316 SK_LOCK();
2317 (void) msleep0(&nxprov_detach_wchan, &sk_lock, (PZERO - 1),
2318 __func__, 0, nxprov_detacher_cont);
2319 /*
2320 * msleep0() shouldn't have returned as PCATCH was not set;
2321 * therefore assert in this case.
2322 */
2323 SK_UNLOCK();
2324 VERIFY(0);
2325 /* NOTREACHED */
2326 __builtin_unreachable();
2327 }
2328
2329 static int
nxprov_detacher_cont(int err)2330 nxprov_detacher_cont(int err)
2331 {
2332 #pragma unused(err)
2333 struct kern_nexus_domain_provider *nxdom_prov;
2334
2335 for (;;) {
2336 SK_LOCK_ASSERT_HELD();
2337 while (nxprov_detaching_cnt == 0) {
2338 (void) msleep0(&nxprov_detach_wchan, &sk_lock,
2339 (PZERO - 1), __func__, 0, nxprov_detacher_cont);
2340 /* NOTREACHED */
2341 }
2342
2343 ASSERT(STAILQ_FIRST(&nxprov_detaching_head) != NULL);
2344
2345 nxdom_prov = nxprov_detaching_dequeue();
2346 if (nxdom_prov != NULL) {
2347 nxdom_del_provider_final(nxdom_prov);
2348 }
2349 }
2350 }
2351