1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54 #include <skywalk/os_skywalk_private.h>
55 #include <skywalk/nexus/upipe/nx_user_pipe.h>
56
57 #define NX_UPIPE_RINGSIZE 128 /* default ring size */
58 #define NX_UPIPE_MAXRINGS NX_MAX_NUM_RING_PAIR
59 #define NX_UPIPE_MINSLOTS 2 /* XXX same as above */
60 #define NX_UPIPE_MAXSLOTS 4096 /* XXX same as above */
61 #define NX_UPIPE_BUFSIZE (2 * 1024)
62 #define NX_UPIPE_MINBUFSIZE 1024
63 #define NX_UPIPE_MAXBUFSIZE (16 * 1024)
64 #define NX_UPIPE_MHINTS NEXUS_MHINTS_NORMAL
65
66 static int nx_upipe_na_alloc(struct nexus_adapter *, uint32_t);
67 static struct nexus_upipe_adapter *nx_upipe_find(struct nexus_adapter *,
68 uint32_t);
69 static int nx_upipe_na_add(struct nexus_adapter *,
70 struct nexus_upipe_adapter *);
71 static void nx_upipe_na_remove(struct nexus_adapter *,
72 struct nexus_upipe_adapter *);
73 static int nx_upipe_na_txsync(struct __kern_channel_ring *,
74 struct proc *, uint32_t);
75 static int nx_upipe_na_txsync_locked(struct __kern_channel_ring *,
76 struct proc *, uint32_t, int *, boolean_t);
77 static int nx_upipe_na_rxsync(struct __kern_channel_ring *,
78 struct proc *, uint32_t);
79 static int nx_upipe_na_krings_create(struct nexus_adapter *,
80 struct kern_channel *);
81 static int nx_upipe_na_activate(struct nexus_adapter *, na_activate_mode_t);
82 static void nx_upipe_na_krings_delete(struct nexus_adapter *,
83 struct kern_channel *, boolean_t);
84 static void nx_upipe_na_dtor(struct nexus_adapter *);
85
86 static void nx_upipe_dom_init(struct nxdom *);
87 static void nx_upipe_dom_terminate(struct nxdom *);
88 static void nx_upipe_dom_fini(struct nxdom *);
89 static int nx_upipe_dom_bind_port(struct kern_nexus *, nexus_port_t *,
90 struct nxbind *, void *);
91 static int nx_upipe_dom_unbind_port(struct kern_nexus *, nexus_port_t);
92 static int nx_upipe_dom_connect(struct kern_nexus_domain_provider *,
93 struct kern_nexus *, struct kern_channel *, struct chreq *,
94 struct kern_channel *, struct nxbind *, struct proc *);
95 static void nx_upipe_dom_disconnect(struct kern_nexus_domain_provider *,
96 struct kern_nexus *, struct kern_channel *);
97 static void nx_upipe_dom_defunct(struct kern_nexus_domain_provider *,
98 struct kern_nexus *, struct kern_channel *, struct proc *);
99 static void nx_upipe_dom_defunct_finalize(struct kern_nexus_domain_provider *,
100 struct kern_nexus *, struct kern_channel *, boolean_t);
101
102 static int nx_upipe_prov_init(struct kern_nexus_domain_provider *);
103 static int nx_upipe_prov_params_adjust(
104 const struct kern_nexus_domain_provider *, const struct nxprov_params *,
105 struct nxprov_adjusted_params *);
106 static int nx_upipe_prov_params(struct kern_nexus_domain_provider *,
107 const uint32_t, const struct nxprov_params *, struct nxprov_params *,
108 struct skmem_region_params[SKMEM_REGIONS], uint32_t);
109 static int nx_upipe_prov_mem_new(struct kern_nexus_domain_provider *,
110 struct kern_nexus *, struct nexus_adapter *);
111 static void nx_upipe_prov_fini(struct kern_nexus_domain_provider *);
112 static int nx_upipe_prov_nx_ctor(struct kern_nexus *);
113 static void nx_upipe_prov_nx_dtor(struct kern_nexus *);
114
115 static struct nexus_upipe_adapter *na_upipe_alloc(zalloc_flags_t);
116 static void na_upipe_free(struct nexus_adapter *);
117
118 static struct nx_upipe *nx_upipe_alloc(zalloc_flags_t);
119 static void nx_upipe_free(struct nx_upipe *);
120
121 #if (DEVELOPMENT || DEBUG)
122 static uint32_t nx_upipe_mhints = 0;
123 SYSCTL_NODE(_kern_skywalk, OID_AUTO, upipe, CTLFLAG_RW | CTLFLAG_LOCKED,
124 0, "Skywalk upipe tuning");
125 SYSCTL_UINT(_kern_skywalk_upipe, OID_AUTO, nx_mhints,
126 CTLFLAG_RW | CTLFLAG_LOCKED, &nx_upipe_mhints, 0,
127 "upipe nexus memory usage hints");
128 #endif /* (DEVELOPMENT || DEBUG) */
129
130 struct nxdom nx_upipe_dom_s = {
131 .nxdom_prov_head =
132 STAILQ_HEAD_INITIALIZER(nx_upipe_dom_s.nxdom_prov_head),
133 .nxdom_type = NEXUS_TYPE_USER_PIPE,
134 .nxdom_md_type = NEXUS_META_TYPE_QUANTUM,
135 .nxdom_md_subtype = NEXUS_META_SUBTYPE_PAYLOAD,
136 .nxdom_name = "upipe",
137 .nxdom_ports = {
138 .nb_def = 2,
139 .nb_min = 2,
140 .nb_max = 2,
141 },
142 .nxdom_tx_rings = {
143 .nb_def = 1,
144 .nb_min = 1,
145 .nb_max = NX_UPIPE_MAXRINGS,
146 },
147 .nxdom_rx_rings = {
148 .nb_def = 1,
149 .nb_min = 1,
150 .nb_max = NX_UPIPE_MAXRINGS,
151 },
152 .nxdom_tx_slots = {
153 .nb_def = NX_UPIPE_RINGSIZE,
154 .nb_min = NX_UPIPE_MINSLOTS,
155 .nb_max = NX_UPIPE_MAXSLOTS,
156 },
157 .nxdom_rx_slots = {
158 .nb_def = NX_UPIPE_RINGSIZE,
159 .nb_min = NX_UPIPE_MINSLOTS,
160 .nb_max = NX_UPIPE_MAXSLOTS,
161 },
162 .nxdom_buf_size = {
163 .nb_def = NX_UPIPE_BUFSIZE,
164 .nb_min = NX_UPIPE_MINBUFSIZE,
165 .nb_max = NX_UPIPE_MAXBUFSIZE,
166 },
167 .nxdom_large_buf_size = {
168 .nb_def = 0,
169 .nb_min = 0,
170 .nb_max = 0,
171 },
172 .nxdom_meta_size = {
173 .nb_def = NX_METADATA_OBJ_MIN_SZ,
174 .nb_min = NX_METADATA_OBJ_MIN_SZ,
175 .nb_max = NX_METADATA_USR_MAX_SZ,
176 },
177 .nxdom_stats_size = {
178 .nb_def = 0,
179 .nb_min = 0,
180 .nb_max = NX_STATS_MAX_SZ,
181 },
182 .nxdom_pipes = {
183 .nb_def = 0,
184 .nb_min = 0,
185 .nb_max = NX_UPIPE_MAXPIPES,
186 },
187 .nxdom_mhints = {
188 .nb_def = NX_UPIPE_MHINTS,
189 .nb_min = NEXUS_MHINTS_NORMAL,
190 .nb_max = (NEXUS_MHINTS_NORMAL | NEXUS_MHINTS_WILLNEED |
191 NEXUS_MHINTS_LOWLATENCY | NEXUS_MHINTS_HIUSE),
192 },
193 .nxdom_flowadv_max = {
194 .nb_def = 0,
195 .nb_min = 0,
196 .nb_max = NX_FLOWADV_MAX,
197 },
198 .nxdom_nexusadv_size = {
199 .nb_def = 0,
200 .nb_min = 0,
201 .nb_max = NX_NEXUSADV_MAX_SZ,
202 },
203 .nxdom_capabilities = {
204 .nb_def = NXPCAP_USER_CHANNEL,
205 .nb_min = NXPCAP_USER_CHANNEL,
206 .nb_max = NXPCAP_USER_CHANNEL,
207 },
208 .nxdom_qmap = {
209 .nb_def = NEXUS_QMAP_TYPE_INVALID,
210 .nb_min = NEXUS_QMAP_TYPE_INVALID,
211 .nb_max = NEXUS_QMAP_TYPE_INVALID,
212 },
213 .nxdom_max_frags = {
214 .nb_def = NX_PBUF_FRAGS_DEFAULT,
215 .nb_min = NX_PBUF_FRAGS_MIN,
216 .nb_max = NX_PBUF_FRAGS_DEFAULT,
217 },
218 .nxdom_init = nx_upipe_dom_init,
219 .nxdom_terminate = nx_upipe_dom_terminate,
220 .nxdom_fini = nx_upipe_dom_fini,
221 .nxdom_find_port = NULL,
222 .nxdom_port_is_reserved = NULL,
223 .nxdom_bind_port = nx_upipe_dom_bind_port,
224 .nxdom_unbind_port = nx_upipe_dom_unbind_port,
225 .nxdom_connect = nx_upipe_dom_connect,
226 .nxdom_disconnect = nx_upipe_dom_disconnect,
227 .nxdom_defunct = nx_upipe_dom_defunct,
228 .nxdom_defunct_finalize = nx_upipe_dom_defunct_finalize,
229 };
230
231 static struct kern_nexus_domain_provider nx_upipe_prov_s = {
232 .nxdom_prov_name = NEXUS_PROVIDER_USER_PIPE,
233 .nxdom_prov_flags = NXDOMPROVF_DEFAULT,
234 .nxdom_prov_cb = {
235 .dp_cb_init = nx_upipe_prov_init,
236 .dp_cb_fini = nx_upipe_prov_fini,
237 .dp_cb_params = nx_upipe_prov_params,
238 .dp_cb_mem_new = nx_upipe_prov_mem_new,
239 .dp_cb_config = NULL,
240 .dp_cb_nx_ctor = nx_upipe_prov_nx_ctor,
241 .dp_cb_nx_dtor = nx_upipe_prov_nx_dtor,
242 .dp_cb_nx_mem_info = NULL,
243 .dp_cb_nx_mib_get = NULL,
244 .dp_cb_nx_stop = NULL,
245 },
246 };
247
248 static ZONE_DEFINE(na_upipe_zone, SKMEM_ZONE_PREFIX ".na.upipe",
249 sizeof(struct nexus_upipe_adapter), ZC_ZFREE_CLEARMEM);
250
251 static ZONE_DEFINE(nx_upipe_zone, SKMEM_ZONE_PREFIX ".nx.upipe",
252 sizeof(struct nx_upipe), ZC_ZFREE_CLEARMEM);
253
254 #define SKMEM_TAG_PIPES "com.apple.skywalk.pipes"
255 static SKMEM_TAG_DEFINE(skmem_tag_pipes, SKMEM_TAG_PIPES);
256
257 static void
nx_upipe_dom_init(struct nxdom * nxdom)258 nx_upipe_dom_init(struct nxdom *nxdom)
259 {
260 SK_LOCK_ASSERT_HELD();
261 ASSERT(!(nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED));
262
263 (void) nxdom_prov_add(nxdom, &nx_upipe_prov_s);
264 }
265
266 static void
nx_upipe_dom_terminate(struct nxdom * nxdom)267 nx_upipe_dom_terminate(struct nxdom *nxdom)
268 {
269 struct kern_nexus_domain_provider *nxdom_prov, *tnxdp;
270
271 STAILQ_FOREACH_SAFE(nxdom_prov, &nxdom->nxdom_prov_head,
272 nxdom_prov_link, tnxdp) {
273 (void) nxdom_prov_del(nxdom_prov);
274 }
275 }
276
277 static void
nx_upipe_dom_fini(struct nxdom * nxdom)278 nx_upipe_dom_fini(struct nxdom *nxdom)
279 {
280 #pragma unused(nxdom)
281 }
282
283 static int
nx_upipe_prov_init(struct kern_nexus_domain_provider * nxdom_prov)284 nx_upipe_prov_init(struct kern_nexus_domain_provider *nxdom_prov)
285 {
286 #pragma unused(nxdom_prov)
287 SK_D("initializing %s", nxdom_prov->nxdom_prov_name);
288 return 0;
289 }
290
291 static int
nx_upipe_prov_params_adjust(const struct kern_nexus_domain_provider * nxdom_prov,const struct nxprov_params * nxp,struct nxprov_adjusted_params * adj)292 nx_upipe_prov_params_adjust(const struct kern_nexus_domain_provider *nxdom_prov,
293 const struct nxprov_params *nxp, struct nxprov_adjusted_params *adj)
294 {
295 #pragma unused(nxdom_prov, nxp)
296 /*
297 * User pipe requires double the amount of rings.
298 * The ring counts must also be symmetrical.
299 */
300 if (*(adj->adj_tx_rings) != *(adj->adj_rx_rings)) {
301 SK_ERR("rings: tx (%u) != rx (%u)", *(adj->adj_tx_rings),
302 *(adj->adj_rx_rings));
303 return EINVAL;
304 }
305 *(adj->adj_tx_rings) *= 2;
306 *(adj->adj_rx_rings) *= 2;
307 return 0;
308 }
309
310 static int
nx_upipe_prov_params(struct kern_nexus_domain_provider * nxdom_prov,const uint32_t req,const struct nxprov_params * nxp0,struct nxprov_params * nxp,struct skmem_region_params srp[SKMEM_REGIONS],uint32_t pp_region_config_flags)311 nx_upipe_prov_params(struct kern_nexus_domain_provider *nxdom_prov,
312 const uint32_t req, const struct nxprov_params *nxp0,
313 struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS],
314 uint32_t pp_region_config_flags)
315 {
316 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
317 int err;
318
319 err = nxprov_params_adjust(nxdom_prov, req, nxp0, nxp, srp,
320 nxdom, nxdom, nxdom, pp_region_config_flags,
321 nx_upipe_prov_params_adjust);
322 #if (DEVELOPMENT || DEBUG)
323 /* sysctl override */
324 if ((err == 0) && (nx_upipe_mhints != 0)) {
325 nxp->nxp_mhints = nx_upipe_mhints;
326 }
327 #endif /* (DEVELOPMENT || DEBUG) */
328 return err;
329 }
330
331 static int
nx_upipe_prov_mem_new(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct nexus_adapter * na)332 nx_upipe_prov_mem_new(struct kern_nexus_domain_provider *nxdom_prov,
333 struct kern_nexus *nx, struct nexus_adapter *na)
334 {
335 #pragma unused(nxdom_prov)
336 int err = 0;
337
338 SK_DF(SK_VERB_USER_PIPE,
339 "nx 0x%llx (\"%s\":\"%s\") na \"%s\" (0x%llx)", SK_KVA(nx),
340 NX_DOM(nx)->nxdom_name, nxdom_prov->nxdom_prov_name, na->na_name,
341 SK_KVA(na));
342
343 ASSERT(na->na_arena == NULL);
344 ASSERT(NX_USER_CHANNEL_PROV(nx));
345 /*
346 * The underlying nexus adapters already share the same memory
347 * allocator, and thus we don't care about storing the pp in
348 * the nexus.
349 *
350 * This means that clients calling kern_nexus_get_pbufpool()
351 * will get NULL, but this is fine since we don't expose the
352 * user pipe to external kernel clients.
353 */
354 na->na_arena = skmem_arena_create_for_nexus(na,
355 NX_PROV(nx)->nxprov_region_params, NULL, NULL, 0, NULL, &err);
356 ASSERT(na->na_arena != NULL || err != 0);
357
358 return err;
359 }
360
361 static void
nx_upipe_prov_fini(struct kern_nexus_domain_provider * nxdom_prov)362 nx_upipe_prov_fini(struct kern_nexus_domain_provider *nxdom_prov)
363 {
364 #pragma unused(nxdom_prov)
365 SK_D("destroying %s", nxdom_prov->nxdom_prov_name);
366 }
367
368 static int
nx_upipe_prov_nx_ctor(struct kern_nexus * nx)369 nx_upipe_prov_nx_ctor(struct kern_nexus *nx)
370 {
371 SK_LOCK_ASSERT_HELD();
372 ASSERT(nx->nx_arg == NULL);
373
374 SK_D("nexus 0x%llx (%s)", SK_KVA(nx), NX_DOM_PROV(nx)->nxdom_prov_name);
375
376 nx->nx_arg = nx_upipe_alloc(Z_WAITOK);
377 SK_D("create new upipe 0x%llx for nexus 0x%llx",
378 SK_KVA(NX_UPIPE_PRIVATE(nx)), SK_KVA(nx));
379
380 return 0;
381 }
382
383 static void
nx_upipe_prov_nx_dtor(struct kern_nexus * nx)384 nx_upipe_prov_nx_dtor(struct kern_nexus *nx)
385 {
386 struct nx_upipe *u = NX_UPIPE_PRIVATE(nx);
387
388 SK_LOCK_ASSERT_HELD();
389
390 SK_D("nexus 0x%llx (%s) upipe 0x%llx", SK_KVA(nx),
391 NX_DOM_PROV(nx)->nxdom_prov_name, SK_KVA(u));
392
393 if (u->nup_cli_nxb != NULL) {
394 nxb_free(u->nup_cli_nxb);
395 u->nup_cli_nxb = NULL;
396 }
397 if (u->nup_srv_nxb != NULL) {
398 nxb_free(u->nup_srv_nxb);
399 u->nup_srv_nxb = NULL;
400 }
401
402 SK_DF(SK_VERB_USER_PIPE, "marking upipe 0x%llx as free", SK_KVA(u));
403 nx_upipe_free(u);
404 nx->nx_arg = NULL;
405 }
406
407 static struct nexus_upipe_adapter *
na_upipe_alloc(zalloc_flags_t how)408 na_upipe_alloc(zalloc_flags_t how)
409 {
410 struct nexus_upipe_adapter *pna;
411
412 _CASSERT(offsetof(struct nexus_upipe_adapter, pna_up) == 0);
413
414 pna = zalloc_flags(na_upipe_zone, how | Z_ZERO);
415 if (pna) {
416 pna->pna_up.na_type = NA_USER_PIPE;
417 pna->pna_up.na_free = na_upipe_free;
418 }
419 return pna;
420 }
421
422 static void
na_upipe_free(struct nexus_adapter * na)423 na_upipe_free(struct nexus_adapter *na)
424 {
425 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
426
427 ASSERT(pna->pna_up.na_refcount == 0);
428 SK_DF(SK_VERB_MEM, "pna 0x%llx FREE", SK_KVA(pna));
429 bzero(pna, sizeof(*pna));
430 zfree(na_upipe_zone, pna);
431 }
432
433 static int
nx_upipe_dom_bind_port(struct kern_nexus * nx,nexus_port_t * nx_port,struct nxbind * nxb0,void * info)434 nx_upipe_dom_bind_port(struct kern_nexus *nx, nexus_port_t *nx_port,
435 struct nxbind *nxb0, void *info)
436 {
437 #pragma unused(info)
438 struct nx_upipe *u = NX_UPIPE_PRIVATE(nx);
439 struct nxbind *nxb = NULL;
440 int error = 0;
441
442 ASSERT(nx_port != NULL);
443 ASSERT(nxb0 != NULL);
444
445 switch (*nx_port) {
446 case NEXUS_PORT_USER_PIPE_CLIENT:
447 case NEXUS_PORT_USER_PIPE_SERVER:
448 if ((*nx_port == NEXUS_PORT_USER_PIPE_CLIENT &&
449 u->nup_cli_nxb != NULL) ||
450 (*nx_port == NEXUS_PORT_USER_PIPE_SERVER &&
451 u->nup_srv_nxb != NULL)) {
452 error = EEXIST;
453 break;
454 }
455
456 nxb = nxb_alloc(Z_WAITOK);
457 nxb_move(nxb0, nxb);
458 if (*nx_port == NEXUS_PORT_USER_PIPE_CLIENT) {
459 u->nup_cli_nxb = nxb;
460 } else {
461 u->nup_srv_nxb = nxb;
462 }
463
464 ASSERT(error == 0);
465 break;
466
467 default:
468 error = EDOM;
469 break;
470 }
471
472 return error;
473 }
474
475 static int
nx_upipe_dom_unbind_port(struct kern_nexus * nx,nexus_port_t nx_port)476 nx_upipe_dom_unbind_port(struct kern_nexus *nx, nexus_port_t nx_port)
477 {
478 struct nx_upipe *u = NX_UPIPE_PRIVATE(nx);
479 struct nxbind *nxb = NULL;
480 int error = 0;
481
482 ASSERT(nx_port != NEXUS_PORT_ANY);
483
484 switch (nx_port) {
485 case NEXUS_PORT_USER_PIPE_CLIENT:
486 case NEXUS_PORT_USER_PIPE_SERVER:
487 if ((nx_port == NEXUS_PORT_USER_PIPE_CLIENT &&
488 u->nup_cli_nxb == NULL) ||
489 (nx_port == NEXUS_PORT_USER_PIPE_SERVER &&
490 u->nup_srv_nxb == NULL)) {
491 error = ENOENT;
492 break;
493 }
494
495 if (nx_port == NEXUS_PORT_USER_PIPE_CLIENT) {
496 nxb = u->nup_cli_nxb;
497 u->nup_cli_nxb = NULL;
498 } else {
499 nxb = u->nup_srv_nxb;
500 u->nup_srv_nxb = NULL;
501 }
502 nxb_free(nxb);
503 ASSERT(error == 0);
504 break;
505
506 default:
507 error = EDOM;
508 break;
509 }
510
511 return error;
512 }
513
514 static int
nx_upipe_dom_connect(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,struct chreq * chr,struct kern_channel * ch0,struct nxbind * nxb,struct proc * p)515 nx_upipe_dom_connect(struct kern_nexus_domain_provider *nxdom_prov,
516 struct kern_nexus *nx, struct kern_channel *ch, struct chreq *chr,
517 struct kern_channel *ch0, struct nxbind *nxb, struct proc *p)
518 {
519 #pragma unused(nxdom_prov)
520 nexus_port_t port = chr->cr_port;
521 int err = 0;
522
523 SK_LOCK_ASSERT_HELD();
524
525 ASSERT(NX_DOM_PROV(nx) == nxdom_prov);
526 ASSERT(nx->nx_prov->nxprov_params->nxp_type ==
527 nxdom_prov->nxdom_prov_dom->nxdom_type &&
528 nx->nx_prov->nxprov_params->nxp_type == NEXUS_TYPE_USER_PIPE);
529
530 /*
531 * XXX: channel in user packet pool mode is not supported for
532 * user-pipe for now.
533 */
534 if (chr->cr_mode & CHMODE_USER_PACKET_POOL) {
535 SK_ERR("User packet pool mode not supported for upipe");
536 err = ENOTSUP;
537 goto done;
538 }
539
540 if (chr->cr_mode & CHMODE_EVENT_RING) {
541 SK_ERR("event ring is not supported for upipe");
542 err = ENOTSUP;
543 goto done;
544 }
545
546 if (chr->cr_mode & CHMODE_LOW_LATENCY) {
547 SK_ERR("low latency is not supported for upipe");
548 err = ENOTSUP;
549 goto done;
550 }
551
552 if (port == NEXUS_PORT_USER_PIPE_SERVER) {
553 chr->cr_real_endpoint = CH_ENDPOINT_USER_PIPE_MASTER;
554 } else if (port == NEXUS_PORT_USER_PIPE_CLIENT) {
555 chr->cr_real_endpoint = CH_ENDPOINT_USER_PIPE_SLAVE;
556 } else {
557 err = EINVAL;
558 goto done;
559 }
560
561 chr->cr_endpoint = chr->cr_real_endpoint;
562 chr->cr_ring_set = RING_SET_DEFAULT;
563 chr->cr_pipe_id = 0;
564 (void) snprintf(chr->cr_name, sizeof(chr->cr_name), "upipe:%llu:%.*s",
565 nx->nx_id, (int)nx->nx_prov->nxprov_params->nxp_namelen,
566 nx->nx_prov->nxprov_params->nxp_name);
567
568 err = na_connect(nx, ch, chr, ch0, nxb, p);
569 done:
570 return err;
571 }
572
573 static void
nx_upipe_dom_disconnect(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch)574 nx_upipe_dom_disconnect(struct kern_nexus_domain_provider *nxdom_prov,
575 struct kern_nexus *nx, struct kern_channel *ch)
576 {
577 #pragma unused(nxdom_prov)
578 SK_LOCK_ASSERT_HELD();
579
580 SK_D("channel 0x%llx -!- nexus 0x%llx (%s:\"%s\":%u:%d)", SK_KVA(ch),
581 SK_KVA(nx), nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
582 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
583
584 na_disconnect(nx, ch);
585 /*
586 * Set NXF_REJECT on the nexus which would cause any channel on the
587 * peer adapter to cease to function.
588 */
589 if (NX_PROV(nx)->nxprov_params->nxp_reject_on_close) {
590 atomic_bitset_32(&nx->nx_flags, NXF_REJECT);
591 }
592 }
593
594 static void
nx_upipe_dom_defunct(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,struct proc * p)595 nx_upipe_dom_defunct(struct kern_nexus_domain_provider *nxdom_prov,
596 struct kern_nexus *nx, struct kern_channel *ch, struct proc *p)
597 {
598 #pragma unused(nxdom_prov, nx)
599 struct nexus_adapter *na = ch->ch_na;
600 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
601 ring_id_t qfirst = ch->ch_first[NR_TX];
602 ring_id_t qlast = ch->ch_last[NR_TX];
603 uint32_t i;
604
605 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
606 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
607 ASSERT(na->na_type == NA_USER_PIPE);
608
609 /*
610 * Inform the peer receiver thread in nx_upipe_na_rxsync() or the
611 * peer transmit thread in nx_upipe_na_txsync() about
612 * this endpoint going defunct. We utilize the TX ring's
613 * lock for serialization, since that is what's being used
614 * by the receiving endpoint.
615 */
616 for (i = qfirst; i < qlast; i++) {
617 /*
618 * For maintaining lock ordering between the two channels of
619 * user pipe.
620 */
621 if (pna->pna_role == CH_ENDPOINT_USER_PIPE_MASTER) {
622 (void) kr_enter(&NAKR(na, NR_TX)[i], TRUE);
623 (void) kr_enter(NAKR(na, NR_RX)[i].ckr_pipe, TRUE);
624 } else {
625 (void) kr_enter(NAKR(na, NR_RX)[i].ckr_pipe, TRUE);
626 (void) kr_enter(&NAKR(na, NR_TX)[i], TRUE);
627 }
628 }
629
630 na_ch_rings_defunct(ch, p);
631
632 for (i = qfirst; i < qlast; i++) {
633 if (pna->pna_role == CH_ENDPOINT_USER_PIPE_MASTER) {
634 (void) kr_exit(NAKR(na, NR_RX)[i].ckr_pipe);
635 (void) kr_exit(&NAKR(na, NR_TX)[i]);
636 } else {
637 (void) kr_exit(&NAKR(na, NR_TX)[i]);
638 (void) kr_exit(NAKR(na, NR_RX)[i].ckr_pipe);
639 }
640 }
641 }
642
643 static void
nx_upipe_dom_defunct_finalize(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,boolean_t locked)644 nx_upipe_dom_defunct_finalize(struct kern_nexus_domain_provider *nxdom_prov,
645 struct kern_nexus *nx, struct kern_channel *ch, boolean_t locked)
646 {
647 #pragma unused(nxdom_prov)
648 struct nexus_upipe_adapter *pna =
649 (struct nexus_upipe_adapter *)ch->ch_na;
650
651 if (!locked) {
652 SK_LOCK_ASSERT_NOTHELD();
653 SK_LOCK();
654 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
655 } else {
656 SK_LOCK_ASSERT_HELD();
657 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
658 }
659
660 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
661 ASSERT(ch->ch_na->na_type == NA_USER_PIPE);
662
663 /*
664 * At this point, we know that the arena shared by the master and
665 * slave adapters has no more valid mappings on the channels opened
666 * to them. We need to invoke na_defunct() on both adapters to
667 * release any remaining slots attached to their rings.
668 *
669 * Note that the 'ch' that we pass in here is irrelevant as we
670 * don't support user packet pool for user pipe.
671 */
672 na_defunct(nx, ch, &pna->pna_up, locked);
673 if (pna->pna_peer != NULL) {
674 na_defunct(nx, ch, &pna->pna_peer->pna_up, locked);
675 }
676
677 /*
678 * And if their parent adapter (the memory owner) is a pseudo
679 * nexus adapter that we initially created in nx_upipe_na_find(),
680 * invoke na_defunct() on it now to do the final teardown on
681 * the arena.
682 */
683 if (pna->pna_parent->na_type == NA_PSEUDO) {
684 na_defunct(nx, ch, pna->pna_parent, locked);
685 }
686
687 SK_D("%s(%d): ch 0x%llx -/- nx 0x%llx (%s:\"%s\":%u:%d)",
688 ch->ch_name, ch->ch_pid, SK_KVA(ch), SK_KVA(nx),
689 nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
690 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
691
692 if (!locked) {
693 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
694 SK_UNLOCK();
695 } else {
696 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
697 SK_LOCK_ASSERT_HELD();
698 }
699 }
700
701 /* allocate the pipe array in the parent adapter */
702 static int
nx_upipe_na_alloc(struct nexus_adapter * na,uint32_t npipes)703 nx_upipe_na_alloc(struct nexus_adapter *na, uint32_t npipes)
704 {
705 struct nexus_upipe_adapter **npa;
706
707 if (npipes <= na->na_max_pipes) {
708 /* we already have more entries that requested */
709 return 0;
710 }
711 if (npipes < na->na_next_pipe || npipes > NX_UPIPE_MAXPIPES) {
712 return EINVAL;
713 }
714
715 npa = sk_realloc_type_array(struct nexus_upipe_adapter *,
716 na->na_max_pipes, npipes, na->na_pipes, Z_WAITOK, skmem_tag_pipes);
717 if (npa == NULL) {
718 return ENOMEM;
719 }
720
721 na->na_pipes = npa;
722 na->na_max_pipes = npipes;
723
724 return 0;
725 }
726
727 /* deallocate the parent array in the parent adapter */
728 void
nx_upipe_na_dealloc(struct nexus_adapter * na)729 nx_upipe_na_dealloc(struct nexus_adapter *na)
730 {
731 if (na->na_pipes) {
732 if (na->na_next_pipe > 0) {
733 SK_ERR("freeing not empty pipe array for %s "
734 "(%u dangling pipes)!", na->na_name,
735 na->na_next_pipe);
736 }
737 sk_free_type_array(struct nexus_upipe_adapter *,
738 na->na_max_pipes, na->na_pipes);
739 na->na_pipes = NULL;
740 na->na_max_pipes = 0;
741 na->na_next_pipe = 0;
742 }
743 }
744
745 /* find a pipe endpoint with the given id among the parent's pipes */
746 static struct nexus_upipe_adapter *
nx_upipe_find(struct nexus_adapter * parent,uint32_t pipe_id)747 nx_upipe_find(struct nexus_adapter *parent, uint32_t pipe_id)
748 {
749 uint32_t i;
750 struct nexus_upipe_adapter *na;
751
752 for (i = 0; i < parent->na_next_pipe; i++) {
753 na = parent->na_pipes[i];
754 if (na->pna_id == pipe_id) {
755 return na;
756 }
757 }
758 return NULL;
759 }
760
761 /* add a new pipe endpoint to the parent array */
762 static int
nx_upipe_na_add(struct nexus_adapter * parent,struct nexus_upipe_adapter * na)763 nx_upipe_na_add(struct nexus_adapter *parent, struct nexus_upipe_adapter *na)
764 {
765 if (parent->na_next_pipe >= parent->na_max_pipes) {
766 uint32_t npipes = parent->na_max_pipes ?
767 2 * parent->na_max_pipes : 2;
768 int error = nx_upipe_na_alloc(parent, npipes);
769 if (error) {
770 return error;
771 }
772 }
773
774 parent->na_pipes[parent->na_next_pipe] = na;
775 na->pna_parent_slot = parent->na_next_pipe;
776 parent->na_next_pipe++;
777 return 0;
778 }
779
780 /* remove the given pipe endpoint from the parent array */
781 static void
nx_upipe_na_remove(struct nexus_adapter * parent,struct nexus_upipe_adapter * na)782 nx_upipe_na_remove(struct nexus_adapter *parent, struct nexus_upipe_adapter *na)
783 {
784 uint32_t n;
785 n = --parent->na_next_pipe;
786 if (n != na->pna_parent_slot) {
787 struct nexus_upipe_adapter **p =
788 &parent->na_pipes[na->pna_parent_slot];
789 *p = parent->na_pipes[n];
790 (*p)->pna_parent_slot = na->pna_parent_slot;
791 }
792 parent->na_pipes[n] = NULL;
793 }
794
795 static int
nx_upipe_na_txsync(struct __kern_channel_ring * txkring,struct proc * p,uint32_t flags)796 nx_upipe_na_txsync(struct __kern_channel_ring *txkring, struct proc *p,
797 uint32_t flags)
798 {
799 struct __kern_channel_ring *rxkring = txkring->ckr_pipe;
800 volatile uint64_t *tx_tsync, *tx_tnote, *rx_tsync;
801 int sent = 0, ret = 0;
802
803 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
804 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u "
805 "flags 0x%x -> kr \"%s\" (0x%llx) krflags 0x%b ring %u",
806 sk_proc_name_address(p), sk_proc_pid(p), txkring->ckr_name,
807 SK_KVA(txkring), txkring->ckr_flags, CKRF_BITS,
808 txkring->ckr_ring_id, flags, rxkring->ckr_name, SK_KVA(rxkring),
809 rxkring->ckr_flags, CKRF_BITS, rxkring->ckr_ring_id);
810
811 /*
812 * Serialize write access to the transmit ring, since another
813 * thread coming down for rxsync might pick up pending slots.
814 */
815 ASSERT(txkring->ckr_owner == current_thread());
816
817 /*
818 * Record the time of sync and grab sync time of other side;
819 * use atomic store and load since we're not holding the
820 * lock used by the receive ring. This allows us to avoid
821 * the potentially costly membar_sync().
822 */
823 /* deconst */
824 tx_tsync = __DECONST(uint64_t *, &txkring->ckr_ring->ring_sync_time);
825 atomic_set_64(tx_tsync, txkring->ckr_sync_time);
826
827 /*
828 * Read from the peer's kring, not its user ring; the peer's channel
829 * may be defunct, in which case it's unsafe to access its user ring.
830 */
831 rx_tsync = __DECONST(uint64_t *, &rxkring->ckr_sync_time);
832 tx_tnote = __DECONST(uint64_t *, &txkring->ckr_ring->ring_notify_time);
833 *tx_tnote = atomic_add_64_ov(rx_tsync, 0);
834
835 if (__probable(txkring->ckr_rhead != txkring->ckr_khead)) {
836 sent = nx_upipe_na_txsync_locked(txkring, p, flags,
837 &ret, FALSE);
838 }
839
840 if (sent != 0) {
841 (void) rxkring->ckr_na_notify(rxkring, p, 0);
842 }
843
844 return ret;
845 }
846
847 int
nx_upipe_na_txsync_locked(struct __kern_channel_ring * txkring,struct proc * p,uint32_t flags,int * ret,boolean_t rx)848 nx_upipe_na_txsync_locked(struct __kern_channel_ring *txkring, struct proc *p,
849 uint32_t flags, int *ret, boolean_t rx)
850 {
851 #pragma unused(p, flags, rx)
852 struct __kern_channel_ring *rxkring = txkring->ckr_pipe;
853 const slot_idx_t lim_tx = txkring->ckr_lim;
854 const slot_idx_t lim_rx = rxkring->ckr_lim;
855 slot_idx_t j, k;
856 int n, m, b, sent = 0;
857 uint32_t byte_count = 0;
858 int limit; /* max # of slots to transfer */
859
860 *ret = 0;
861
862 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
863 "%s(%d) kr \"%s\", kh %3u kt %3u | "
864 "rh %3u rt %3u [pre%s]", sk_proc_name_address(p),
865 sk_proc_pid(p), txkring->ckr_name, txkring->ckr_khead,
866 txkring->ckr_ktail, txkring->ckr_rhead,
867 txkring->ckr_rtail, rx ? "*" : "");
868 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
869 "%s(%d) kr \"%s\", kh %3u kt %3u | "
870 "rh %3u rt %3u [pre%s]", sk_proc_name_address(p),
871 sk_proc_pid(p), rxkring->ckr_name, rxkring->ckr_khead,
872 rxkring->ckr_ktail, rxkring->ckr_rhead,
873 rxkring->ckr_rtail, rx ? "*" : "");
874
875 if (__improbable(KR_DROP(txkring) || KR_DROP(rxkring))) {
876 *ret = ENXIO;
877 goto done;
878 }
879
880 j = rxkring->ckr_ktail; /* RX */
881 k = txkring->ckr_khead; /* TX */
882
883 /* # of new tx slots */
884 n = txkring->ckr_rhead - txkring->ckr_khead;
885 if (n < 0) {
886 n += txkring->ckr_num_slots;
887 }
888 limit = n;
889
890 /* # of rx busy (unclaimed) slots */
891 b = j - rxkring->ckr_khead;
892 if (b < 0) {
893 b += rxkring->ckr_num_slots;
894 }
895
896 /* # of rx avail free slots (subtract busy from max) */
897 m = lim_rx - b;
898 if (m < limit) {
899 limit = m;
900 }
901
902 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
903 "%s(%d) kr \"%s\" -> new %u, kr \"%s\" "
904 "-> free %u", sk_proc_name_address(p), sk_proc_pid(p),
905 txkring->ckr_name, n, rxkring->ckr_name, m);
906
907 /* rxring is full, or nothing to send? */
908 if (__improbable((sent = limit) == 0)) {
909 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
910 "%s(%d) kr \"%s\" -> %s%s",
911 sk_proc_name_address(p), sk_proc_pid(p), (n > m) ?
912 rxkring->ckr_name : txkring->ckr_name, ((n > m) ?
913 "no room avail" : "no new slots"),
914 (rx ? " (lost race, ok)" : ""));
915 goto done;
916 }
917
918 ASSERT(limit > 0);
919 while (limit--) {
920 struct __kern_slot_desc *ksd_tx = KR_KSD(txkring, k);
921 struct __user_slot_desc *usd_tx = KR_USD(txkring, k);
922 struct __kern_slot_desc *ksd_rx = KR_KSD(rxkring, j);
923 struct __user_slot_desc *usd_rx = KR_USD(rxkring, j);
924 struct __kern_quantum *kqum;
925
926 kqum = ksd_tx->sd_qum;
927 /*
928 * Packets failing internalization should be dropped in
929 * TX sync prologue.
930 */
931 ASSERT((kqum->qum_qflags & (QUM_F_INTERNALIZED |
932 QUM_F_FINALIZED)) == (QUM_F_INTERNALIZED |
933 QUM_F_FINALIZED));
934
935 byte_count += kqum->qum_len;
936
937 /*
938 * Swap the slots.
939 *
940 * XXX: [email protected] -- this bypasses the slot attach/detach
941 * interface, and needs to be changed when upipe adopts the
942 * packet APIs. SD_SWAP() will perform a block copy of the
943 * swap, and will readjust the kernel slot descriptor's sd_user
944 * accordingly.
945 */
946 SD_SWAP(ksd_rx, usd_rx, ksd_tx, usd_tx);
947
948 j = SLOT_NEXT(j, lim_rx);
949 k = SLOT_NEXT(k, lim_tx);
950 }
951
952 kr_update_stats(rxkring, sent, byte_count);
953 if (__improbable(kr_stat_enable != 0)) {
954 txkring->ckr_stats = rxkring->ckr_stats;
955 }
956
957 /*
958 * Make sure the slots are updated before ckr_ktail reach global
959 * visibility, since we are not holding rx ring's kr_enter().
960 */
961 membar_sync();
962
963 rxkring->ckr_ktail = j;
964 txkring->ckr_khead = k;
965 txkring->ckr_ktail = SLOT_PREV(k, lim_tx);
966
967 done:
968 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
969 "%s(%d) kr \"%s\", kh %3u kt %3u | "
970 "rh %3u rt %3u [post%s]", sk_proc_name_address(p),
971 sk_proc_pid(p), txkring->ckr_name, txkring->ckr_khead,
972 txkring->ckr_ktail, txkring->ckr_rhead,
973 txkring->ckr_rtail, rx ? "*" : "");
974 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_TX,
975 "%s(%d) kr \"%s\", kh %3u kt %3u | "
976 "rh %3u rt %3u [post%s]", sk_proc_name_address(p),
977 sk_proc_pid(p), rxkring->ckr_name, rxkring->ckr_khead,
978 rxkring->ckr_ktail, rxkring->ckr_rhead,
979 rxkring->ckr_rtail, rx ? "*" : "");
980
981 return sent;
982 }
983
984 static int
nx_upipe_na_rxsync(struct __kern_channel_ring * rxkring,struct proc * p,uint32_t flags)985 nx_upipe_na_rxsync(struct __kern_channel_ring *rxkring, struct proc *p,
986 uint32_t flags)
987 {
988 #pragma unused(p)
989 struct __kern_channel_ring *txkring = rxkring->ckr_pipe;
990 volatile uint64_t *rx_tsync, *rx_tnote, *tx_tsync;
991 const slot_idx_t lim_rx = rxkring->ckr_lim;
992 int n; /* new slots from transmit side */
993 int m, b, ret = 0;
994 uint32_t r;
995
996 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_RX,
997 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u "
998 "flags 0x%x <- kr \"%s\" (0x%llx) krflags 0x%b ring %u",
999 sk_proc_name_address(p), sk_proc_pid(p), rxkring->ckr_name,
1000 SK_KVA(rxkring), rxkring->ckr_flags, CKRF_BITS,
1001 rxkring->ckr_ring_id, flags, txkring->ckr_name, SK_KVA(txkring),
1002 txkring->ckr_flags, CKRF_BITS, txkring->ckr_ring_id);
1003
1004 ASSERT(rxkring->ckr_owner == current_thread());
1005
1006 /* reclaim and get # of rx reclaimed slots */
1007 r = kr_reclaim(rxkring);
1008
1009 /* # of rx busy (unclaimed) slots */
1010 b = rxkring->ckr_ktail - rxkring->ckr_khead;
1011 if (b < 0) {
1012 b += rxkring->ckr_num_slots;
1013 }
1014
1015 /* # of rx avail free slots (subtract busy from max) */
1016 m = lim_rx - b;
1017
1018 /*
1019 * Check if there's any new slots on transmit ring; do this
1020 * first without acquiring that ring's ckr_qlock, and use
1021 * the memory barrier (paired with second one in txsync.)
1022 * If we missed the race we'd just pay the cost of acquiring
1023 * ckr_qlock and potentially returning from "internal txsync"
1024 * without anything to process, which is okay.
1025 */
1026 membar_sync();
1027 n = txkring->ckr_rhead - txkring->ckr_khead;
1028 if (n < 0) {
1029 n += txkring->ckr_num_slots;
1030 }
1031
1032 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_RX,
1033 "%s(%d) kr \"%s\" <- free %u, kr \"%s\" <- new %u",
1034 sk_proc_name_address(p), sk_proc_pid(p),
1035 rxkring->ckr_name, m, txkring->ckr_name, n);
1036
1037 /*
1038 * Record the time of sync and grab sync time of other side;
1039 * use atomic store and load since we're not holding the
1040 * lock used by the receive ring. This allows us to avoid
1041 * the potentially costly membar_sync().
1042 */
1043 /* deconst */
1044 rx_tsync = __DECONST(uint64_t *, &rxkring->ckr_ring->ring_sync_time);
1045 atomic_set_64(rx_tsync, rxkring->ckr_sync_time);
1046
1047 /*
1048 * Read from the peer's kring, not its user ring; the peer's channel
1049 * may be defunct, in which case it's unsafe to access its user ring.
1050 */
1051 tx_tsync = __DECONST(uint64_t *, &txkring->ckr_sync_time);
1052 rx_tnote = __DECONST(uint64_t *, &rxkring->ckr_ring->ring_notify_time);
1053 *rx_tnote = atomic_add_64_ov(tx_tsync, 0);
1054
1055 /*
1056 * If we have slots to pick up from the transmit side and and we
1057 * have space available, perform an equivalent of "internal txsync".
1058 *
1059 * Acquire write access to the transmit (peer) ring,
1060 * Serialize write access to it, since another thread
1061 * coming down for txsync might add new slots.
1062 * If we fail to get the kring lock, then don't worry because
1063 * there's already a transmit sync in progress to move packets.
1064 */
1065 if (__probable(n != 0 && m != 0 && (flags & NA_SYNCF_MONITOR) == 0)) {
1066 (void) kr_enter(txkring, TRUE);
1067 n = nx_upipe_na_txsync_locked(txkring, p, flags, &ret, TRUE);
1068 kr_exit(txkring);
1069 } else {
1070 n = 0;
1071 }
1072
1073 /*
1074 * If we have reclaimed some slots or transferred new slots
1075 * from the transmit side, notify the other end. Also notify
1076 * ourselves to pick up newly transferred ones, if any.
1077 */
1078 if (__probable(r != 0 || n != 0)) {
1079 SK_DF(SK_VERB_USER_PIPE | SK_VERB_SYNC | SK_VERB_RX,
1080 "%s(%d) kr \"%s\", kh %3u kt %3u | "
1081 "rh %3u rt %3u [rel %u new %u]",
1082 sk_proc_name_address(p), sk_proc_pid(p), rxkring->ckr_name,
1083 rxkring->ckr_khead, rxkring->ckr_ktail,
1084 rxkring->ckr_rhead, rxkring->ckr_rtail, r, n);
1085
1086 (void) txkring->ckr_na_notify(txkring, p, 0);
1087 }
1088
1089 return ret;
1090 }
1091
1092 static int
nx_upipe_na_rings_create(struct nexus_adapter * na,struct kern_channel * ch)1093 nx_upipe_na_rings_create(struct nexus_adapter *na, struct kern_channel *ch)
1094 {
1095 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
1096 struct nexus_adapter *ona = &pna->pna_peer->pna_up;
1097 int error = 0;
1098 enum txrx t;
1099 uint32_t i;
1100
1101 /*
1102 * Create krings and all the rings for this end;
1103 * we'll update ckr_save_ring pointers below.
1104 */
1105 error = na_rings_mem_setup(na, 0, FALSE, ch);
1106 if (error != 0) {
1107 goto err;
1108 }
1109
1110 /* update our hidden ring pointers */
1111 for_rx_tx(t) {
1112 for (i = 0; i < na_get_nrings(na, t); i++) {
1113 NAKR(na, t)[i].ckr_save_ring =
1114 NAKR(na, t)[i].ckr_ring;
1115 }
1116 }
1117
1118 /* now, create krings and rings of the other end */
1119 error = na_rings_mem_setup(ona, 0, FALSE, ch);
1120 if (error != 0) {
1121 na_rings_mem_teardown(na, ch, FALSE); /* this end */
1122 goto err;
1123 }
1124
1125 for_rx_tx(t) {
1126 for (i = 0; i < na_get_nrings(ona, t); i++) {
1127 NAKR(ona, t)[i].ckr_save_ring =
1128 NAKR(ona, t)[i].ckr_ring;
1129 }
1130 }
1131
1132 /* cross link the krings */
1133 for_rx_tx(t) {
1134 /* swap NR_TX <-> NR_RX (skip host ring) */
1135 enum txrx r = sk_txrx_swap(t);
1136 for (i = 0; i < na_get_nrings(na, t); i++) {
1137 NAKR(na, t)[i].ckr_pipe =
1138 NAKR(&pna->pna_peer->pna_up, r) + i;
1139 NAKR(&pna->pna_peer->pna_up, r)[i].ckr_pipe =
1140 NAKR(na, t) + i;
1141 }
1142 }
1143 err:
1144 return error;
1145 }
1146
1147 /*
1148 * Pipe endpoints are created and destroyed together, so that endopoints do not
1149 * have to check for the existence of their peer at each ?xsync.
1150 *
1151 * To play well with the existing nexus adapter infrastructure (refcounts etc.),
1152 * we adopt the following strategy:
1153 *
1154 * 1) The first endpoint that is created also creates the other endpoint and
1155 * grabs a reference to it.
1156 *
1157 * state A) user1 --> endpoint1 --> endpoint2
1158 *
1159 * 2) If, starting from state A, endpoint2 is then registered, endpoint1 gives
1160 * its reference to the user:
1161 *
1162 * state B) user1 --> endpoint1 endpoint2 <--- user2
1163 *
1164 * 3) Assume that, starting from state B endpoint2 is closed. In the unregister
1165 * callback endpoint2 notes that endpoint1 is still active and adds a reference
1166 * from endpoint1 to itself. When user2 then releases her own reference,
1167 * endpoint2 is not destroyed and we are back to state A. A symmetrical state
1168 * would be reached if endpoint1 were released instead.
1169 *
1170 * 4) If, starting from state A, endpoint1 is closed, the destructor notes that
1171 * it owns a reference to endpoint2 and releases it.
1172 *
1173 * Something similar goes on for the creation and destruction of the krings.
1174 */
1175
1176
1177 /*
1178 * nx_upipe_na_krings_create.
1179 *
1180 * There are two cases:
1181 *
1182 * 1) state is
1183 *
1184 * usr1 --> e1 --> e2
1185 *
1186 * and we are e1. We have to create both sets
1187 * of krings.
1188 *
1189 * 2) state is
1190 *
1191 * usr1 --> e1 --> e2
1192 *
1193 * and we are e2. e1 is certainly registered and our
1194 * krings already exist, but they may be hidden.
1195 */
1196 static int
nx_upipe_na_krings_create(struct nexus_adapter * na,struct kern_channel * ch)1197 nx_upipe_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
1198 {
1199 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
1200 int error = 0;
1201 enum txrx t;
1202 uint32_t i;
1203
1204 /*
1205 * Verify symmetrical ring counts; validated
1206 * at nexus provider registration time.
1207 */
1208 ASSERT(na_get_nrings(na, NR_TX) == na_get_nrings(na, NR_RX));
1209
1210 if (pna->pna_peer_ref) {
1211 /* case 1) above */
1212 SK_DF(SK_VERB_USER_PIPE,
1213 "0x%llx: case 1, create everything", SK_KVA(na));
1214 error = nx_upipe_na_rings_create(na, ch);
1215 } else {
1216 /* case 2) above */
1217 /* recover the hidden rings */
1218 SK_DF(SK_VERB_USER_PIPE,
1219 "0x%llx: case 2, hidden rings", SK_KVA(na));
1220 for_rx_tx(t) {
1221 for (i = 0; i < na_get_nrings(na, t); i++) {
1222 NAKR(na, t)[i].ckr_ring =
1223 NAKR(na, t)[i].ckr_save_ring;
1224 }
1225 }
1226 }
1227
1228 ASSERT(error == 0 || (na->na_tx_rings == NULL &&
1229 na->na_rx_rings == NULL && na->na_slot_ctxs == NULL));
1230 ASSERT(error == 0 || (pna->pna_peer->pna_up.na_tx_rings == NULL &&
1231 pna->pna_peer->pna_up.na_rx_rings == NULL &&
1232 pna->pna_peer->pna_up.na_slot_ctxs == NULL));
1233
1234 return error;
1235 }
1236
1237 /*
1238 * nx_upipe_na_activate.
1239 *
1240 * There are two cases on registration (onoff==1)
1241 *
1242 * 1.a) state is
1243 *
1244 * usr1 --> e1 --> e2
1245 *
1246 * and we are e1. Nothing special to do.
1247 *
1248 * 1.b) state is
1249 *
1250 * usr1 --> e1 --> e2 <-- usr2
1251 *
1252 * and we are e2. Drop the ref e1 is holding.
1253 *
1254 * There are two additional cases on unregister (onoff==0)
1255 *
1256 * 2.a) state is
1257 *
1258 * usr1 --> e1 --> e2
1259 *
1260 * and we are e1. Nothing special to do, e2 will
1261 * be cleaned up by the destructor of e1.
1262 *
1263 * 2.b) state is
1264 *
1265 * usr1 --> e1 e2 <-- usr2
1266 *
1267 * and we are either e1 or e2. Add a ref from the
1268 * other end and hide our rings.
1269 */
1270 static int
nx_upipe_na_activate(struct nexus_adapter * na,na_activate_mode_t mode)1271 nx_upipe_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
1272 {
1273 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
1274
1275 SK_LOCK_ASSERT_HELD();
1276
1277 SK_DF(SK_VERB_USER_PIPE, "na \"%s\" (0x%llx) %s", na->na_name,
1278 SK_KVA(na), na_activate_mode2str(mode));
1279
1280 switch (mode) {
1281 case NA_ACTIVATE_MODE_ON:
1282 atomic_bitset_32(&na->na_flags, NAF_ACTIVE);
1283 break;
1284
1285 case NA_ACTIVATE_MODE_DEFUNCT:
1286 break;
1287
1288 case NA_ACTIVATE_MODE_OFF:
1289 atomic_bitclear_32(&na->na_flags, NAF_ACTIVE);
1290 break;
1291
1292 default:
1293 VERIFY(0);
1294 /* NOTREACHED */
1295 __builtin_unreachable();
1296 }
1297
1298 if (pna->pna_peer_ref) {
1299 SK_DF(SK_VERB_USER_PIPE,
1300 "0x%llx: case 1.a or 2.a, nothing to do", SK_KVA(na));
1301 return 0;
1302 }
1303
1304 switch (mode) {
1305 case NA_ACTIVATE_MODE_ON:
1306 SK_DF(SK_VERB_USER_PIPE,
1307 "0x%llx: case 1.b, drop peer", SK_KVA(na));
1308 if (pna->pna_peer->pna_peer_ref) {
1309 pna->pna_peer->pna_peer_ref = FALSE;
1310 (void) na_release_locked(na);
1311 }
1312 break;
1313
1314 case NA_ACTIVATE_MODE_OFF:
1315 SK_DF(SK_VERB_USER_PIPE,
1316 "0x%llx: case 2.b, grab peer", SK_KVA(na));
1317 if (!pna->pna_peer->pna_peer_ref) {
1318 na_retain_locked(na);
1319 pna->pna_peer->pna_peer_ref = TRUE;
1320 }
1321 break;
1322
1323 default:
1324 break;
1325 }
1326
1327 return 0;
1328 }
1329
1330 /*
1331 * nx_upipe_na_krings_delete.
1332 *
1333 * There are two cases:
1334 *
1335 * 1) state is
1336 *
1337 * usr1 --> e1 --> e2
1338 *
1339 * and we are e1 (e2 is not bound, so krings_delete cannot be
1340 * called on it);
1341 *
1342 * 2) state is
1343 *
1344 * usr1 --> e1 e2 <-- usr2
1345 *
1346 * and we are either e1 or e2.
1347 *
1348 * In the former case we have to also delete the krings of e2;
1349 * in the latter case we do nothing (note that our krings
1350 * have already been hidden in the unregister callback).
1351 */
1352 static void
nx_upipe_na_krings_delete(struct nexus_adapter * na,struct kern_channel * ch,boolean_t defunct)1353 nx_upipe_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
1354 boolean_t defunct)
1355 {
1356 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
1357 struct nexus_adapter *ona; /* na of the other end */
1358 uint32_t i;
1359 enum txrx t;
1360
1361 SK_LOCK_ASSERT_HELD();
1362
1363 if (!pna->pna_peer_ref) {
1364 SK_DF(SK_VERB_USER_PIPE,
1365 "0x%llx: case 2, kept alive by peer", SK_KVA(na));
1366 /*
1367 * If adapter is defunct (note the explicit test against
1368 * NAF_DEFUNCT, and not the "defunct" parameter passed in
1369 * by the caller), then the peer's channel has gone defunct.
1370 * We get here because this channel was not defuncted, and
1371 * that this is the last active reference to the adapter.
1372 * At this point we tear everything down, since the caller
1373 * will proceed to destroying the memory regions.
1374 */
1375 if (na->na_flags & NAF_DEFUNCT) {
1376 na_rings_mem_teardown(na, ch, defunct);
1377 }
1378 return;
1379 }
1380
1381 /* case 1) above */
1382 SK_DF(SK_VERB_USER_PIPE,
1383 "0x%llx: case 1, deleting everyhing", SK_KVA(na));
1384
1385 ASSERT(na->na_channels == 0 || (na->na_flags & NAF_DEFUNCT));
1386
1387 /* restore the ring to be deleted on the peer */
1388 ona = &pna->pna_peer->pna_up;
1389 if (ona->na_tx_rings == NULL) {
1390 /*
1391 * Already deleted, we must be on an
1392 * cleanup-after-error path
1393 * Just delete this end
1394 */
1395 na_rings_mem_teardown(na, ch, defunct);
1396 return;
1397 }
1398
1399 /* delete the memory rings */
1400 na_rings_mem_teardown(na, ch, defunct);
1401
1402 if (!defunct) {
1403 for_rx_tx(t) {
1404 for (i = 0; i < na_get_nrings(ona, t); i++) {
1405 NAKR(ona, t)[i].ckr_ring =
1406 NAKR(ona, t)[i].ckr_save_ring;
1407 }
1408 }
1409 }
1410
1411 /* Delete the memory rings */
1412 na_rings_mem_teardown(ona, ch, defunct);
1413 }
1414
1415 static void
nx_upipe_na_dtor(struct nexus_adapter * na)1416 nx_upipe_na_dtor(struct nexus_adapter *na)
1417 {
1418 struct nexus_upipe_adapter *pna = (struct nexus_upipe_adapter *)na;
1419 struct nx_upipe *u = NX_UPIPE_PRIVATE(na->na_nx);
1420
1421 SK_LOCK_ASSERT_HELD();
1422
1423 SK_DF(SK_VERB_USER_PIPE, "0x%llx", SK_KVA(na));
1424 if (pna->pna_peer_ref) {
1425 SK_DF(SK_VERB_USER_PIPE,
1426 "0x%llx: clean up peer 0x%llx", SK_KVA(na),
1427 SK_KVA(&pna->pna_peer->pna_up));
1428 pna->pna_peer_ref = FALSE;
1429 (void) na_release_locked(&pna->pna_peer->pna_up);
1430 }
1431 if (pna->pna_role == CH_ENDPOINT_USER_PIPE_MASTER) {
1432 nx_upipe_na_remove(pna->pna_parent, pna);
1433 }
1434 (void) na_release_locked(pna->pna_parent);
1435 pna->pna_parent = NULL;
1436
1437 /* release reference to parent adapter held by nx_upipe_na_find() */
1438 ASSERT(u->nup_pna_users != 0);
1439 if (--u->nup_pna_users == 0) {
1440 ASSERT(u->nup_pna != NULL);
1441 SK_DF(SK_VERB_USER_PIPE, "release parent: \"%s\" (0x%llx)",
1442 u->nup_pna->na_name, SK_KVA(u->nup_pna));
1443 na_release_locked(u->nup_pna);
1444 u->nup_pna = NULL;
1445 }
1446 }
1447
1448 int
nx_upipe_na_find(struct kern_nexus * nx,struct kern_channel * ch,struct chreq * chr,struct nxbind * nxb,struct proc * p,struct nexus_adapter ** na,boolean_t create)1449 nx_upipe_na_find(struct kern_nexus *nx, struct kern_channel *ch,
1450 struct chreq *chr, struct nxbind *nxb, struct proc *p,
1451 struct nexus_adapter **na, boolean_t create)
1452 {
1453 #pragma unused(ch, p)
1454 struct nx_upipe *u = NX_UPIPE_PRIVATE(nx);
1455 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
1456 struct nexus_adapter *pna = NULL; /* parent adapter */
1457 boolean_t anon = NX_ANONYMOUS_PROV(nx);
1458 struct nexus_upipe_adapter *mna, *sna, *req;
1459 ch_endpoint_t ep = chr->cr_endpoint;
1460 uint32_t pipe_id;
1461 int error;
1462
1463 SK_LOCK_ASSERT_HELD();
1464 *na = NULL;
1465
1466 #if SK_LOG
1467 uuid_string_t uuidstr;
1468 SK_D("name \"%s\" spec_uuid \"%s\" port %d mode 0x%b pipe_id %u "
1469 "ring_id %d ring_set %u ep_type %u:%u create %u%s",
1470 chr->cr_name, sk_uuid_unparse(chr->cr_spec_uuid, uuidstr),
1471 (int)chr->cr_port, chr->cr_mode, CHMODE_BITS,
1472 chr->cr_pipe_id, (int)chr->cr_ring_id, chr->cr_ring_set,
1473 chr->cr_real_endpoint, chr->cr_endpoint, create,
1474 (ep != CH_ENDPOINT_USER_PIPE_MASTER &&
1475 ep != CH_ENDPOINT_USER_PIPE_SLAVE) ? " (skipped)" : "");
1476 #endif /* SK_LOG */
1477
1478 if (ep != CH_ENDPOINT_USER_PIPE_MASTER &&
1479 ep != CH_ENDPOINT_USER_PIPE_SLAVE) {
1480 return 0;
1481 }
1482
1483 /*
1484 * Check client credentials.
1485 */
1486 if (chr->cr_port == NEXUS_PORT_USER_PIPE_SERVER) {
1487 if (!anon && (u->nup_srv_nxb == NULL || nxb == NULL ||
1488 !nxb_is_equal(u->nup_srv_nxb, nxb))) {
1489 return EACCES;
1490 }
1491 } else {
1492 ASSERT(chr->cr_port == NEXUS_PORT_USER_PIPE_CLIENT);
1493 if (!anon && (u->nup_cli_nxb == NULL || nxb == NULL ||
1494 !nxb_is_equal(u->nup_cli_nxb, nxb))) {
1495 return EACCES;
1496 }
1497 }
1498
1499 /*
1500 * First, try to find a previously-created parent adapter
1501 * for this nexus; else, create one and store it in the
1502 * nexus. We'll release this at nexus destructor time.
1503 */
1504 if ((pna = u->nup_pna) != NULL) {
1505 na_retain_locked(pna); /* for us */
1506 SK_DF(SK_VERB_USER_PIPE, "found parent: \"%s\" (0x%llx)",
1507 pna->na_name, SK_KVA(pna));
1508 } else {
1509 /* callee will hold a reference for us upon success */
1510 error = na_pseudo_create(nx, chr, &pna);
1511 if (error != 0) {
1512 SK_ERR("parent create failed: %d", error);
1513 return error;
1514 }
1515 /* hold an extra reference for nx_upipe */
1516 u->nup_pna = pna;
1517 na_retain_locked(pna);
1518 SK_DF(SK_VERB_USER_PIPE, "created parent: \"%s\" (0x%llx)",
1519 pna->na_name, SK_KVA(pna));
1520 }
1521
1522 /* next, lookup the pipe id in the parent list */
1523 req = NULL;
1524 pipe_id = chr->cr_pipe_id;
1525 mna = nx_upipe_find(pna, pipe_id);
1526 if (mna != NULL) {
1527 if (mna->pna_role == ep) {
1528 SK_DF(SK_VERB_USER_PIPE,
1529 "found pipe_id %u directly at slot %u",
1530 pipe_id, mna->pna_parent_slot);
1531 req = mna;
1532 } else {
1533 SK_DF(SK_VERB_USER_PIPE,
1534 "found pipe_id %u indirectly at slot %u",
1535 pipe_id, mna->pna_parent_slot);
1536 req = mna->pna_peer;
1537 }
1538 /*
1539 * The pipe we have found already holds a ref to the parent,
1540 * so we need to drop the one we got from above.
1541 */
1542 (void) na_release_locked(pna);
1543 goto found;
1544 }
1545 SK_DF(SK_VERB_USER_PIPE,
1546 "pipe_id %u not found, create %u", pipe_id, create);
1547 if (!create) {
1548 error = ENODEV;
1549 goto put_out;
1550 }
1551 /*
1552 * We create both master and slave.
1553 * The endpoint we were asked for holds a reference to
1554 * the other one.
1555 */
1556 mna = na_upipe_alloc(Z_WAITOK);
1557
1558 ASSERT(mna->pna_up.na_type == NA_USER_PIPE);
1559 ASSERT(mna->pna_up.na_free == na_upipe_free);
1560
1561 (void) snprintf(mna->pna_up.na_name, sizeof(mna->pna_up.na_name),
1562 "%s{%u", pna->na_name, pipe_id);
1563 uuid_generate_random(mna->pna_up.na_uuid);
1564
1565 mna->pna_id = pipe_id;
1566 mna->pna_role = CH_ENDPOINT_USER_PIPE_MASTER;
1567 mna->pna_parent = pna;
1568 mna->pna_up.na_txsync = nx_upipe_na_txsync;
1569 mna->pna_up.na_rxsync = nx_upipe_na_rxsync;
1570 mna->pna_up.na_activate = nx_upipe_na_activate;
1571 mna->pna_up.na_dtor = nx_upipe_na_dtor;
1572 mna->pna_up.na_krings_create = nx_upipe_na_krings_create;
1573 mna->pna_up.na_krings_delete = nx_upipe_na_krings_delete;
1574 mna->pna_up.na_arena = pna->na_arena;
1575 skmem_arena_retain((&mna->pna_up)->na_arena);
1576 atomic_bitset_32(&mna->pna_up.na_flags, NAF_MEM_LOANED);
1577 *(nexus_meta_type_t *)(uintptr_t)&mna->pna_up.na_md_type =
1578 pna->na_md_type;
1579 *(nexus_meta_subtype_t *)(uintptr_t)&mna->pna_up.na_md_subtype =
1580 pna->na_md_subtype;
1581
1582 *(nexus_stats_type_t *)(uintptr_t)&mna->pna_up.na_stats_type =
1583 NEXUS_STATS_TYPE_INVALID;
1584 *(uint32_t *)(uintptr_t)&mna->pna_up.na_flowadv_max =
1585 nxp->nxp_flowadv_max;
1586 ASSERT(mna->pna_up.na_flowadv_max == 0 ||
1587 skmem_arena_nexus(mna->pna_up.na_arena)->arn_flowadv_obj != NULL);
1588
1589 /*
1590 * Parent adapter parameters must match the nexus provider's by the
1591 * time we get here, since na_find() above shouldn't return
1592 * one otherwise.
1593 */
1594 na_set_nrings(&mna->pna_up, NR_TX, nxp->nxp_tx_rings);
1595 na_set_nrings(&mna->pna_up, NR_RX, nxp->nxp_rx_rings);
1596 na_set_nslots(&mna->pna_up, NR_TX, nxp->nxp_tx_slots);
1597 na_set_nslots(&mna->pna_up, NR_RX, nxp->nxp_rx_slots);
1598 ASSERT(na_get_nrings(&mna->pna_up, NR_TX) == na_get_nrings(pna, NR_TX));
1599 ASSERT(na_get_nrings(&mna->pna_up, NR_RX) == na_get_nrings(pna, NR_RX));
1600 ASSERT(na_get_nslots(&mna->pna_up, NR_TX) == na_get_nslots(pna, NR_TX));
1601 ASSERT(na_get_nslots(&mna->pna_up, NR_RX) == na_get_nslots(pna, NR_RX));
1602
1603 na_attach_common(&mna->pna_up, nx, &nx_upipe_prov_s);
1604
1605 /* register the master with the parent */
1606 error = nx_upipe_na_add(pna, mna);
1607 if (error != 0) {
1608 goto free_mna;
1609 }
1610
1611 /* create the slave */
1612 sna = na_upipe_alloc(Z_WAITOK);
1613
1614 /* most fields are the same, copy from master and then fix */
1615 bcopy(mna, sna, sizeof(*sna));
1616 skmem_arena_retain((&sna->pna_up)->na_arena);
1617 atomic_bitset_32(&sna->pna_up.na_flags, NAF_MEM_LOANED);
1618
1619 ASSERT(sna->pna_up.na_type == NA_USER_PIPE);
1620 ASSERT(sna->pna_up.na_free == na_upipe_free);
1621
1622 (void) snprintf(sna->pna_up.na_name, sizeof(sna->pna_up.na_name),
1623 "%s}%d", pna->na_name, pipe_id);
1624 uuid_generate_random(sna->pna_up.na_uuid);
1625
1626 sna->pna_role = CH_ENDPOINT_USER_PIPE_SLAVE;
1627 na_attach_common(&sna->pna_up, nx, &nx_upipe_prov_s);
1628
1629 /* join the two endpoints */
1630 mna->pna_peer = sna;
1631 sna->pna_peer = mna;
1632
1633 /*
1634 * We already have a reference to the parent, but we
1635 * need another one for the other endpoint we created
1636 */
1637 na_retain_locked(pna);
1638
1639 if ((chr->cr_mode & CHMODE_DEFUNCT_OK) != 0) {
1640 atomic_bitset_32(&pna->na_flags, NAF_DEFUNCT_OK);
1641 }
1642
1643 if (ep == CH_ENDPOINT_USER_PIPE_MASTER) {
1644 req = mna;
1645 mna->pna_peer_ref = TRUE;
1646 na_retain_locked(&sna->pna_up);
1647 } else {
1648 req = sna;
1649 sna->pna_peer_ref = TRUE;
1650 na_retain_locked(&mna->pna_up);
1651 }
1652
1653 /* parent adapter now has two users (mna and sna) */
1654 u->nup_pna_users += 2;
1655
1656 #if SK_LOG
1657 SK_DF(SK_VERB_USER_PIPE, "created master 0x%llx and slave 0x%llx",
1658 SK_KVA(mna), SK_KVA(sna));
1659 SK_DF(SK_VERB_USER_PIPE, "mna: \"%s\"", mna->pna_up.na_name);
1660 SK_DF(SK_VERB_USER_PIPE, " UUID: %s",
1661 sk_uuid_unparse(mna->pna_up.na_uuid, uuidstr));
1662 SK_DF(SK_VERB_USER_PIPE, " nx: 0x%llx (\"%s\":\"%s\")",
1663 SK_KVA(mna->pna_up.na_nx), NX_DOM(mna->pna_up.na_nx)->nxdom_name,
1664 NX_DOM_PROV(mna->pna_up.na_nx)->nxdom_prov_name);
1665 SK_DF(SK_VERB_USER_PIPE, " flags: 0x%b",
1666 mna->pna_up.na_flags, NAF_BITS);
1667 SK_DF(SK_VERB_USER_PIPE, " flowadv_max: %u",
1668 mna->pna_up.na_flowadv_max);
1669 SK_DF(SK_VERB_USER_PIPE, " rings: tx %u rx %u",
1670 na_get_nrings(&mna->pna_up, NR_TX),
1671 na_get_nrings(&mna->pna_up, NR_RX));
1672 SK_DF(SK_VERB_USER_PIPE, " slots: tx %u rx %u",
1673 na_get_nslots(&mna->pna_up, NR_TX),
1674 na_get_nslots(&mna->pna_up, NR_RX));
1675 SK_DF(SK_VERB_USER_PIPE, " next_pipe: %u", mna->pna_up.na_next_pipe);
1676 SK_DF(SK_VERB_USER_PIPE, " max_pipes: %u", mna->pna_up.na_max_pipes);
1677 SK_DF(SK_VERB_USER_PIPE, " parent: \"%s\"",
1678 mna->pna_parent->na_name);
1679 SK_DF(SK_VERB_USER_PIPE, " id: %u", mna->pna_id);
1680 SK_DF(SK_VERB_USER_PIPE, " role: %u", mna->pna_role);
1681 SK_DF(SK_VERB_USER_PIPE, " peer_ref: %u", mna->pna_peer_ref);
1682 SK_DF(SK_VERB_USER_PIPE, " parent_slot: %u", mna->pna_parent_slot);
1683 SK_DF(SK_VERB_USER_PIPE, "sna: \"%s\"", sna->pna_up.na_name);
1684 SK_DF(SK_VERB_USER_PIPE, " UUID: %s",
1685 sk_uuid_unparse(sna->pna_up.na_uuid, uuidstr));
1686 SK_DF(SK_VERB_USER_PIPE, " nx: 0x%llx (\"%s\":\"%s\")",
1687 SK_KVA(sna->pna_up.na_nx), NX_DOM(sna->pna_up.na_nx)->nxdom_name,
1688 NX_DOM_PROV(sna->pna_up.na_nx)->nxdom_prov_name);
1689 SK_DF(SK_VERB_USER_PIPE, " flags: 0x%b",
1690 sna->pna_up.na_flags, NAF_BITS);
1691 SK_DF(SK_VERB_USER_PIPE, " flowadv_max: %u",
1692 sna->pna_up.na_flowadv_max);
1693 SK_DF(SK_VERB_USER_PIPE, " rings: tx %u rx %u",
1694 na_get_nrings(&sna->pna_up, NR_TX),
1695 na_get_nrings(&sna->pna_up, NR_RX));
1696 SK_DF(SK_VERB_USER_PIPE, " slots: tx %u rx %u",
1697 na_get_nslots(&sna->pna_up, NR_TX),
1698 na_get_nslots(&sna->pna_up, NR_RX));
1699 SK_DF(SK_VERB_USER_PIPE, " next_pipe: %u", sna->pna_up.na_next_pipe);
1700 SK_DF(SK_VERB_USER_PIPE, " max_pipes: %u", sna->pna_up.na_max_pipes);
1701 SK_DF(SK_VERB_USER_PIPE, " parent: \"%s\"",
1702 sna->pna_parent->na_name);
1703 SK_DF(SK_VERB_USER_PIPE, " id: %u", sna->pna_id);
1704 SK_DF(SK_VERB_USER_PIPE, " role: %u", sna->pna_role);
1705 SK_DF(SK_VERB_USER_PIPE, " peer_ref: %u", sna->pna_peer_ref);
1706 SK_DF(SK_VERB_USER_PIPE, " parent_slot: %u", sna->pna_parent_slot);
1707 #endif /* SK_LOG */
1708
1709 found:
1710
1711 SK_DF(SK_VERB_USER_PIPE, "pipe_id %u role %s at 0x%llx", pipe_id,
1712 (req->pna_role == CH_ENDPOINT_USER_PIPE_MASTER ?
1713 "master" : "slave"), SK_KVA(req));
1714 if ((chr->cr_mode & CHMODE_DEFUNCT_OK) == 0) {
1715 atomic_bitclear_32(&pna->na_flags, NAF_DEFUNCT_OK);
1716 }
1717 *na = &req->pna_up;
1718 na_retain_locked(*na);
1719
1720 /*
1721 * Keep the reference to the parent; it will be released
1722 * by the adapter's destructor.
1723 */
1724 return 0;
1725
1726 free_mna:
1727 if (mna->pna_up.na_arena != NULL) {
1728 skmem_arena_release((&mna->pna_up)->na_arena);
1729 mna->pna_up.na_arena = NULL;
1730 }
1731 NA_FREE(&mna->pna_up);
1732 put_out:
1733 (void) na_release_locked(pna);
1734 return error;
1735 }
1736
1737 static struct nx_upipe *
nx_upipe_alloc(zalloc_flags_t how)1738 nx_upipe_alloc(zalloc_flags_t how)
1739 {
1740 struct nx_upipe *u;
1741
1742 SK_LOCK_ASSERT_HELD();
1743
1744 u = zalloc_flags(nx_upipe_zone, how | Z_ZERO);
1745 if (u) {
1746 SK_DF(SK_VERB_MEM, "upipe 0x%llx ALLOC", SK_KVA(u));
1747 }
1748 return u;
1749 }
1750
1751 static void
nx_upipe_free(struct nx_upipe * u)1752 nx_upipe_free(struct nx_upipe *u)
1753 {
1754 ASSERT(u->nup_pna == NULL);
1755 ASSERT(u->nup_pna_users == 0);
1756 ASSERT(u->nup_cli_nxb == NULL);
1757 ASSERT(u->nup_srv_nxb == NULL);
1758
1759 SK_DF(SK_VERB_MEM, "upipe 0x%llx FREE", SK_KVA(u));
1760 zfree(nx_upipe_zone, u);
1761 }
1762