1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54 #include <skywalk/os_skywalk_private.h>
55 #include <skywalk/nexus/upipe/nx_user_pipe.h>
56 #include <skywalk/nexus/kpipe/nx_kernel_pipe.h>
57
58 /* XXX arbitrary */
59 #define NX_KPIPE_RINGSIZE 128 /* default ring size */
60 #define NX_KPIPE_MINSLOTS 2
61 #define NX_KPIPE_MAXSLOTS 4096
62 #define NX_KPIPE_MAXRINGS NX_MAX_NUM_RING_PAIR
63 #define NX_KPIPE_BUFSIZE (2 * 1024)
64 #define NX_KPIPE_MINBUFSIZE 64
65 #define NX_KPIPE_MAXBUFSIZE (16 * 1024)
66
67 static int nx_kpipe_na_txsync(struct __kern_channel_ring *, struct proc *,
68 uint32_t);
69 static int nx_kpipe_na_rxsync(struct __kern_channel_ring *, struct proc *,
70 uint32_t);
71 static int nx_kpipe_na_activate(struct nexus_adapter *, na_activate_mode_t);
72 static void nx_kpipe_na_dtor(struct nexus_adapter *);
73 static int nx_kpipe_na_krings_create(struct nexus_adapter *,
74 struct kern_channel *);
75 static void nx_kpipe_na_krings_delete(struct nexus_adapter *,
76 struct kern_channel *, boolean_t);
77
78 static void nx_kpipe_dom_init(struct nxdom *);
79 static void nx_kpipe_dom_terminate(struct nxdom *);
80 static void nx_kpipe_dom_fini(struct nxdom *);
81 static int nx_kpipe_dom_bind_port(struct kern_nexus *, nexus_port_t *,
82 struct nxbind *, void *);
83 static int nx_kpipe_dom_unbind_port(struct kern_nexus *, nexus_port_t);
84 static int nx_kpipe_dom_connect(struct kern_nexus_domain_provider *,
85 struct kern_nexus *, struct kern_channel *, struct chreq *,
86 struct kern_channel *, struct nxbind *, struct proc *);
87 static void nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *,
88 struct kern_nexus *, struct kern_channel *);
89 static void nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *,
90 struct kern_nexus *, struct kern_channel *, struct proc *);
91 static void nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *,
92 struct kern_nexus *, struct kern_channel *, boolean_t);
93
94 static int nx_kpipe_prov_init(struct kern_nexus_domain_provider *);
95 static int nx_kpipe_prov_params_adjust(
96 const struct kern_nexus_domain_provider *,
97 const struct nxprov_params *, struct nxprov_adjusted_params *);
98 static int nx_kpipe_prov_params(struct kern_nexus_domain_provider *,
99 const uint32_t, const struct nxprov_params *, struct nxprov_params *,
100 struct skmem_region_params[SKMEM_REGIONS], uint32_t);
101 static int nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *,
102 struct kern_nexus *, struct nexus_adapter *);
103 static void nx_kpipe_prov_fini(struct kern_nexus_domain_provider *);
104 static int nx_kpipe_prov_nx_ctor(struct kern_nexus *);
105 static void nx_kpipe_prov_nx_dtor(struct kern_nexus *);
106 static int nx_kpipe_prov_nx_mem_info(struct kern_nexus *,
107 struct kern_pbufpool **, struct kern_pbufpool **);
108
109 static struct nexus_kpipe_adapter *na_kpipe_alloc(zalloc_flags_t);
110 static void na_kpipe_free(struct nexus_adapter *);
111
112 struct nxdom nx_kpipe_dom_s = {
113 .nxdom_prov_head =
114 STAILQ_HEAD_INITIALIZER(nx_kpipe_dom_s.nxdom_prov_head),
115 .nxdom_type = NEXUS_TYPE_KERNEL_PIPE,
116 .nxdom_md_type = NEXUS_META_TYPE_QUANTUM,
117 .nxdom_md_subtype = NEXUS_META_SUBTYPE_PAYLOAD,
118 .nxdom_name = "kpipe",
119 .nxdom_ports = {
120 .nb_def = 1,
121 .nb_min = 1,
122 .nb_max = 1,
123 },
124 .nxdom_tx_rings = {
125 .nb_def = 1,
126 .nb_min = 1,
127 .nb_max = NX_KPIPE_MAXRINGS,
128 },
129 .nxdom_rx_rings = {
130 .nb_def = 1,
131 .nb_min = 1,
132 .nb_max = NX_KPIPE_MAXRINGS,
133 },
134 .nxdom_tx_slots = {
135 .nb_def = NX_KPIPE_RINGSIZE,
136 .nb_min = NX_KPIPE_MINSLOTS,
137 .nb_max = NX_KPIPE_MAXSLOTS,
138 },
139 .nxdom_rx_slots = {
140 .nb_def = NX_KPIPE_RINGSIZE,
141 .nb_min = NX_KPIPE_MINSLOTS,
142 .nb_max = NX_KPIPE_MAXSLOTS,
143 },
144 .nxdom_buf_size = {
145 .nb_def = NX_KPIPE_BUFSIZE,
146 .nb_min = NX_KPIPE_MINBUFSIZE,
147 .nb_max = NX_KPIPE_MAXBUFSIZE,
148 },
149 .nxdom_large_buf_size = {
150 .nb_def = 0,
151 .nb_min = 0,
152 .nb_max = 0,
153 },
154 .nxdom_meta_size = {
155 .nb_def = NX_METADATA_OBJ_MIN_SZ,
156 .nb_min = NX_METADATA_OBJ_MIN_SZ,
157 .nb_max = NX_METADATA_USR_MAX_SZ,
158 },
159 .nxdom_stats_size = {
160 .nb_def = 0,
161 .nb_min = 0,
162 .nb_max = NX_STATS_MAX_SZ,
163 },
164 .nxdom_pipes = {
165 .nb_def = 0,
166 .nb_min = 0,
167 .nb_max = NX_UPIPE_MAXPIPES,
168 },
169 .nxdom_flowadv_max = {
170 .nb_def = 0,
171 .nb_min = 0,
172 .nb_max = NX_FLOWADV_MAX,
173 },
174 .nxdom_nexusadv_size = {
175 .nb_def = 0,
176 .nb_min = 0,
177 .nb_max = NX_NEXUSADV_MAX_SZ,
178 },
179 .nxdom_capabilities = {
180 .nb_def = NXPCAP_USER_CHANNEL,
181 .nb_min = NXPCAP_USER_CHANNEL,
182 .nb_max = NXPCAP_USER_CHANNEL,
183 },
184 .nxdom_qmap = {
185 .nb_def = NEXUS_QMAP_TYPE_INVALID,
186 .nb_min = NEXUS_QMAP_TYPE_INVALID,
187 .nb_max = NEXUS_QMAP_TYPE_INVALID,
188 },
189 .nxdom_max_frags = {
190 .nb_def = NX_PBUF_FRAGS_DEFAULT,
191 .nb_min = NX_PBUF_FRAGS_MIN,
192 .nb_max = NX_PBUF_FRAGS_DEFAULT,
193 },
194 .nxdom_init = nx_kpipe_dom_init,
195 .nxdom_terminate = nx_kpipe_dom_terminate,
196 .nxdom_fini = nx_kpipe_dom_fini,
197 .nxdom_find_port = NULL,
198 .nxdom_port_is_reserved = NULL,
199 .nxdom_bind_port = nx_kpipe_dom_bind_port,
200 .nxdom_unbind_port = nx_kpipe_dom_unbind_port,
201 .nxdom_connect = nx_kpipe_dom_connect,
202 .nxdom_disconnect = nx_kpipe_dom_disconnect,
203 .nxdom_defunct = nx_kpipe_dom_defunct,
204 .nxdom_defunct_finalize = nx_kpipe_dom_defunct_finalize,
205 };
206
207 static struct kern_nexus_domain_provider nx_kpipe_prov_s = {
208 .nxdom_prov_name = NEXUS_PROVIDER_KERNEL_PIPE,
209 .nxdom_prov_flags = NXDOMPROVF_DEFAULT,
210 .nxdom_prov_cb = {
211 .dp_cb_init = nx_kpipe_prov_init,
212 .dp_cb_fini = nx_kpipe_prov_fini,
213 .dp_cb_params = nx_kpipe_prov_params,
214 .dp_cb_mem_new = nx_kpipe_prov_mem_new,
215 .dp_cb_config = NULL,
216 .dp_cb_nx_ctor = nx_kpipe_prov_nx_ctor,
217 .dp_cb_nx_dtor = nx_kpipe_prov_nx_dtor,
218 .dp_cb_nx_mem_info = nx_kpipe_prov_nx_mem_info,
219 .dp_cb_nx_mib_get = NULL,
220 .dp_cb_nx_stop = NULL,
221 },
222 };
223
224 static ZONE_DEFINE(na_kpipe_zone, SKMEM_ZONE_PREFIX ".na.kpipe",
225 sizeof(struct nexus_kpipe_adapter), ZC_ZFREE_CLEARMEM);
226
227 static void
nx_kpipe_dom_init(struct nxdom * nxdom)228 nx_kpipe_dom_init(struct nxdom *nxdom)
229 {
230 SK_LOCK_ASSERT_HELD();
231 ASSERT(!(nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED));
232
233 (void) nxdom_prov_add(nxdom, &nx_kpipe_prov_s);
234 }
235
236 static void
nx_kpipe_dom_terminate(struct nxdom * nxdom)237 nx_kpipe_dom_terminate(struct nxdom *nxdom)
238 {
239 struct kern_nexus_domain_provider *nxdom_prov, *tnxdp;
240
241 SK_LOCK_ASSERT_HELD();
242
243 STAILQ_FOREACH_SAFE(nxdom_prov, &nxdom->nxdom_prov_head,
244 nxdom_prov_link, tnxdp) {
245 (void) nxdom_prov_del(nxdom_prov);
246 }
247 }
248
249 static void
nx_kpipe_dom_fini(struct nxdom * nxdom)250 nx_kpipe_dom_fini(struct nxdom *nxdom)
251 {
252 #pragma unused(nxdom)
253 }
254
255 static int
nx_kpipe_dom_bind_port(struct kern_nexus * nx,nexus_port_t * nx_port,struct nxbind * nxb0,void * info)256 nx_kpipe_dom_bind_port(struct kern_nexus *nx, nexus_port_t *nx_port,
257 struct nxbind *nxb0, void *info)
258 {
259 #pragma unused(info)
260 struct nxbind *nxb = NULL;
261 int error = 0;
262
263 ASSERT(nx_port != NULL);
264 ASSERT(nxb0 != NULL);
265
266 switch (*nx_port) {
267 case NEXUS_PORT_KERNEL_PIPE_CLIENT:
268 if (nx->nx_arg != NULL) {
269 error = EEXIST;
270 break;
271 }
272
273 nxb = nxb_alloc(Z_WAITOK);
274 nxb_move(nxb0, nxb);
275 nx->nx_arg = nxb;
276
277 ASSERT(error == 0);
278 break;
279
280 default:
281 error = EDOM;
282 break;
283 }
284
285 return error;
286 }
287
288 static int
nx_kpipe_dom_unbind_port(struct kern_nexus * nx,nexus_port_t nx_port)289 nx_kpipe_dom_unbind_port(struct kern_nexus *nx, nexus_port_t nx_port)
290 {
291 struct nxbind *nxb = NULL;
292 int error = 0;
293
294 ASSERT(nx_port != NEXUS_PORT_ANY);
295
296 switch (nx_port) {
297 case NEXUS_PORT_KERNEL_PIPE_CLIENT:
298 if ((nxb = nx->nx_arg) == NULL) {
299 error = ENOENT;
300 break;
301 }
302 nx->nx_arg = NULL;
303 nxb_free(nxb);
304 ASSERT(error == 0);
305 break;
306
307 default:
308 error = EDOM;
309 break;
310 }
311
312 return error;
313 }
314
315 static int
nx_kpipe_dom_connect(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,struct chreq * chr,struct kern_channel * ch0,struct nxbind * nxb,struct proc * p)316 nx_kpipe_dom_connect(struct kern_nexus_domain_provider *nxdom_prov,
317 struct kern_nexus *nx, struct kern_channel *ch, struct chreq *chr,
318 struct kern_channel *ch0, struct nxbind *nxb, struct proc *p)
319 {
320 #pragma unused(nxdom_prov)
321 nexus_port_t port = chr->cr_port;
322 int err = 0;
323
324 SK_DF(SK_VERB_KERNEL_PIPE, "port %d mode 0x%b",
325 (int)port, chr->cr_mode, CHMODE_BITS);
326
327 SK_LOCK_ASSERT_HELD();
328
329 ASSERT(nx->nx_prov->nxprov_params->nxp_type ==
330 nxdom_prov->nxdom_prov_dom->nxdom_type &&
331 nx->nx_prov->nxprov_params->nxp_type == NEXUS_TYPE_KERNEL_PIPE);
332
333 if (port != NEXUS_PORT_KERNEL_PIPE_CLIENT) {
334 err = EINVAL;
335 goto done;
336 }
337
338 /*
339 * XXX: user packet pool is not supported for kernel pipe for now.
340 */
341 if (chr->cr_mode & CHMODE_USER_PACKET_POOL) {
342 SK_ERR("User Packet pool mode not supported for kpipe");
343 err = ENOTSUP;
344 goto done;
345 }
346
347 if (chr->cr_mode & CHMODE_EVENT_RING) {
348 SK_ERR("event ring is not supported for kpipe");
349 err = ENOTSUP;
350 goto done;
351 }
352
353 if (chr->cr_mode & CHMODE_LOW_LATENCY) {
354 SK_ERR("low latency is not supported for kpipe");
355 err = ENOTSUP;
356 goto done;
357 }
358
359 chr->cr_ring_set = RING_SET_DEFAULT;
360 chr->cr_real_endpoint = chr->cr_endpoint = CH_ENDPOINT_KERNEL_PIPE;
361 (void) snprintf(chr->cr_name, sizeof(chr->cr_name), "kpipe:%llu:%.*s",
362 nx->nx_id, (int)nx->nx_prov->nxprov_params->nxp_namelen,
363 nx->nx_prov->nxprov_params->nxp_name);
364
365 err = na_connect(nx, ch, chr, ch0, nxb, p);
366 if (err == 0) {
367 /*
368 * Mark the kernel slot descriptor region as busy; this
369 * prevents it from being torn-down at channel defunct
370 * time, as the (external) nexus owner may be calling
371 * KPIs that require accessing the slots.
372 */
373 skmem_arena_nexus_sd_set_noidle(
374 skmem_arena_nexus(ch->ch_na->na_arena), 1);
375 }
376
377 done:
378 return err;
379 }
380
381 static void
nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch)382 nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *nxdom_prov,
383 struct kern_nexus *nx, struct kern_channel *ch)
384 {
385 #pragma unused(nxdom_prov, nx)
386 SK_LOCK_ASSERT_HELD();
387
388 SK_D("channel 0x%llx -!- nexus 0x%llx (%s:\"%s\":%u:%d)", SK_KVA(ch),
389 SK_KVA(nx), nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
390 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
391
392 /*
393 * Release busy assertion held earlier in nx_kpipe_dom_connect();
394 * this allows for the final arena teardown to succeed.
395 */
396 skmem_arena_nexus_sd_set_noidle(
397 skmem_arena_nexus(ch->ch_na->na_arena), -1);
398
399 na_disconnect(nx, ch);
400 }
401
402 static void
nx_kpipe_dom_defunct(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,struct proc * p)403 nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *nxdom_prov,
404 struct kern_nexus *nx, struct kern_channel *ch, struct proc *p)
405 {
406 #pragma unused(nxdom_prov, nx)
407 struct nexus_adapter *na = ch->ch_na;
408 ring_id_t qfirst;
409 ring_id_t qlast;
410 enum txrx t;
411 uint32_t i;
412
413 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
414 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
415 ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE);
416
417 /*
418 * Interface drivers like utun & IPsec access the kpipe rings
419 * outside of a kpipe channel sync context. They hold rights
420 * to the ring through kr_enter().
421 */
422 for_rx_tx(t) {
423 qfirst = ch->ch_first[t];
424 qlast = ch->ch_last[t];
425
426 for (i = qfirst; i < qlast; i++) {
427 (void) kr_enter(&NAKR(na, t)[i], TRUE);
428 }
429 }
430
431 na_ch_rings_defunct(ch, p);
432
433 for_rx_tx(t) {
434 qfirst = ch->ch_first[t];
435 qlast = ch->ch_last[t];
436
437 for (i = qfirst; i < qlast; i++) {
438 kr_exit(&NAKR(na, t)[i]);
439 }
440 }
441 }
442
443 static void
nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct kern_channel * ch,boolean_t locked)444 nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *nxdom_prov,
445 struct kern_nexus *nx, struct kern_channel *ch, boolean_t locked)
446 {
447 #pragma unused(nxdom_prov)
448 if (!locked) {
449 SK_LOCK_ASSERT_NOTHELD();
450 SK_LOCK();
451 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
452 } else {
453 SK_LOCK_ASSERT_HELD();
454 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
455 }
456
457 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
458 ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE);
459
460 na_defunct(nx, ch, ch->ch_na, locked);
461
462 SK_D("%s(%d): ch 0x%llx -/- nx 0x%llx (%s:\"%s\":%u:%d)",
463 ch->ch_name, ch->ch_pid, SK_KVA(ch), SK_KVA(nx),
464 nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
465 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
466
467 if (!locked) {
468 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
469 SK_UNLOCK();
470 } else {
471 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
472 SK_LOCK_ASSERT_HELD();
473 }
474 }
475
476 static int
nx_kpipe_prov_init(struct kern_nexus_domain_provider * nxdom_prov)477 nx_kpipe_prov_init(struct kern_nexus_domain_provider *nxdom_prov)
478 {
479 #pragma unused(nxdom_prov)
480 SK_D("initializing %s", nxdom_prov->nxdom_prov_name);
481 return 0;
482 }
483
484 static int
nx_kpipe_prov_params_adjust(const struct kern_nexus_domain_provider * nxdom_prov,const struct nxprov_params * nxp,struct nxprov_adjusted_params * adj)485 nx_kpipe_prov_params_adjust(const struct kern_nexus_domain_provider *nxdom_prov,
486 const struct nxprov_params *nxp, struct nxprov_adjusted_params *adj)
487 {
488 #pragma unused(nxdom_prov, nxp, adj)
489 return 0;
490 }
491
492 static int
nx_kpipe_prov_params(struct kern_nexus_domain_provider * nxdom_prov,const uint32_t req,const struct nxprov_params * nxp0,struct nxprov_params * nxp,struct skmem_region_params srp[SKMEM_REGIONS],uint32_t pp_region_config_flags)493 nx_kpipe_prov_params(struct kern_nexus_domain_provider *nxdom_prov,
494 const uint32_t req, const struct nxprov_params *nxp0,
495 struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS],
496 uint32_t pp_region_config_flags)
497 {
498 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
499
500 return nxprov_params_adjust(nxdom_prov, req, nxp0, nxp, srp,
501 nxdom, nxdom, nxdom, pp_region_config_flags,
502 nx_kpipe_prov_params_adjust);
503 }
504
505 static int
nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider * nxdom_prov,struct kern_nexus * nx,struct nexus_adapter * na)506 nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *nxdom_prov,
507 struct kern_nexus *nx, struct nexus_adapter *na)
508 {
509 #pragma unused(nxdom_prov)
510 int err = 0;
511
512 SK_DF(SK_VERB_KERNEL_PIPE,
513 "nx 0x%llx (\"%s\":\"%s\") na \"%s\" (0x%llx)", SK_KVA(nx),
514 NX_DOM(nx)->nxdom_name, nxdom_prov->nxdom_prov_name, na->na_name,
515 SK_KVA(na));
516
517 ASSERT(na->na_arena == NULL);
518 ASSERT(NX_USER_CHANNEL_PROV(nx));
519 /*
520 * Store pp in the nexus to handle kern_nexus_get_pbufpool() calls.
521 */
522 na->na_arena = skmem_arena_create_for_nexus(na,
523 NX_PROV(nx)->nxprov_region_params, &nx->nx_tx_pp,
524 &nx->nx_rx_pp, 0, NULL, &err);
525 ASSERT(na->na_arena != NULL || err != 0);
526 ASSERT(nx->nx_tx_pp == NULL || (nx->nx_tx_pp->pp_md_type ==
527 NX_DOM(nx)->nxdom_md_type && nx->nx_tx_pp->pp_md_subtype ==
528 NX_DOM(nx)->nxdom_md_subtype));
529 ASSERT(nx->nx_rx_pp == NULL || (nx->nx_rx_pp->pp_md_type ==
530 NX_DOM(nx)->nxdom_md_type && nx->nx_rx_pp->pp_md_subtype ==
531 NX_DOM(nx)->nxdom_md_subtype));
532
533 return err;
534 }
535
536 static void
nx_kpipe_prov_fini(struct kern_nexus_domain_provider * nxdom_prov)537 nx_kpipe_prov_fini(struct kern_nexus_domain_provider *nxdom_prov)
538 {
539 #pragma unused(nxdom_prov)
540 SK_D("destroying %s", nxdom_prov->nxdom_prov_name);
541 }
542
543 static int
nx_kpipe_prov_nx_ctor(struct kern_nexus * nx)544 nx_kpipe_prov_nx_ctor(struct kern_nexus *nx)
545 {
546 #pragma unused(nx)
547 SK_LOCK_ASSERT_HELD();
548 ASSERT(nx->nx_arg == NULL);
549 return 0;
550 }
551
552 static void
nx_kpipe_prov_nx_dtor(struct kern_nexus * nx)553 nx_kpipe_prov_nx_dtor(struct kern_nexus *nx)
554 {
555 struct nxbind *nxb;
556
557 SK_LOCK_ASSERT_HELD();
558
559 if ((nxb = nx->nx_arg) != NULL) {
560 nxb_free(nxb);
561 nx->nx_arg = NULL;
562 }
563 }
564
565 static int
nx_kpipe_prov_nx_mem_info(struct kern_nexus * nx,struct kern_pbufpool ** tpp,struct kern_pbufpool ** rpp)566 nx_kpipe_prov_nx_mem_info(struct kern_nexus *nx, struct kern_pbufpool **tpp,
567 struct kern_pbufpool **rpp)
568 {
569 ASSERT(nx->nx_tx_pp != NULL);
570 ASSERT(nx->nx_rx_pp != NULL);
571
572 if (tpp != NULL) {
573 *tpp = nx->nx_tx_pp;
574 }
575 if (rpp != NULL) {
576 *rpp = nx->nx_rx_pp;
577 }
578
579 return 0;
580 }
581
582 static struct nexus_kpipe_adapter *
na_kpipe_alloc(zalloc_flags_t how)583 na_kpipe_alloc(zalloc_flags_t how)
584 {
585 struct nexus_kpipe_adapter *kna;
586
587 _CASSERT(offsetof(struct nexus_kpipe_adapter, kna_up) == 0);
588
589 kna = zalloc_flags(na_kpipe_zone, how | Z_ZERO);
590 if (kna) {
591 kna->kna_up.na_type = NA_KERNEL_PIPE;
592 kna->kna_up.na_free = na_kpipe_free;
593 }
594 return kna;
595 }
596
597 static void
na_kpipe_free(struct nexus_adapter * na)598 na_kpipe_free(struct nexus_adapter *na)
599 {
600 struct nexus_kpipe_adapter *kna = (struct nexus_kpipe_adapter *)na;
601
602 ASSERT(kna->kna_up.na_refcount == 0);
603 SK_DF(SK_VERB_MEM, "kna 0x%llx FREE", SK_KVA(kna));
604 bzero(kna, sizeof(*kna));
605 zfree(na_kpipe_zone, kna);
606 }
607
608 static int
nx_kpipe_na_txsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)609 nx_kpipe_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
610 uint32_t flags)
611 {
612 #pragma unused(p)
613 SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_TX,
614 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x",
615 sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name,
616 SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id,
617 flags);
618
619 return nx_sync_tx(kring, (flags & NA_SYNCF_FORCE_RECLAIM));
620 }
621
622 static int
nx_kpipe_na_rxsync(struct __kern_channel_ring * kring,struct proc * p,uint32_t flags)623 nx_kpipe_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
624 uint32_t flags)
625 {
626 #pragma unused(p)
627 SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_RX,
628 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x",
629 sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name,
630 SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id,
631 flags);
632
633 ASSERT(kring->ckr_rhead <= kring->ckr_lim);
634
635 return nx_sync_rx(kring, (flags & NA_SYNCF_FORCE_READ));
636 }
637
638 static int
nx_kpipe_na_activate(struct nexus_adapter * na,na_activate_mode_t mode)639 nx_kpipe_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
640 {
641 ASSERT(na->na_type == NA_KERNEL_PIPE);
642
643 SK_DF(SK_VERB_KERNEL_PIPE, "na \"%s\" (0x%llx) %s", na->na_name,
644 SK_KVA(na), na_activate_mode2str(mode));
645
646 switch (mode) {
647 case NA_ACTIVATE_MODE_ON:
648 atomic_bitset_32(&na->na_flags, NAF_ACTIVE);
649 break;
650
651 case NA_ACTIVATE_MODE_DEFUNCT:
652 break;
653
654 case NA_ACTIVATE_MODE_OFF:
655 atomic_bitclear_32(&na->na_flags, NAF_ACTIVE);
656 break;
657
658 default:
659 VERIFY(0);
660 /* NOTREACHED */
661 __builtin_unreachable();
662 }
663
664 return 0;
665 }
666
667 static void
nx_kpipe_na_dtor(struct nexus_adapter * na)668 nx_kpipe_na_dtor(struct nexus_adapter *na)
669 {
670 #pragma unused(na)
671 ASSERT(na->na_type == NA_KERNEL_PIPE);
672 }
673
674 static int
nx_kpipe_na_krings_create(struct nexus_adapter * na,struct kern_channel * ch)675 nx_kpipe_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
676 {
677 ASSERT(na->na_type == NA_KERNEL_PIPE);
678 /*
679 * The assumption here is that all kernel pipe instances
680 * are handled by IOSkywalkFamily, and thus we allocate
681 * the context area for it to store its object references.
682 */
683 return na_rings_mem_setup(na, 0, TRUE, ch);
684 }
685
686 static void
nx_kpipe_na_krings_delete(struct nexus_adapter * na,struct kern_channel * ch,boolean_t defunct)687 nx_kpipe_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
688 boolean_t defunct)
689 {
690 ASSERT(na->na_type == NA_KERNEL_PIPE);
691
692 na_rings_mem_teardown(na, ch, defunct);
693 }
694
695 int
nx_kpipe_na_find(struct kern_nexus * nx,struct kern_channel * ch,struct chreq * chr,struct nxbind * nxb,struct proc * p,struct nexus_adapter ** ret,boolean_t create)696 nx_kpipe_na_find(struct kern_nexus *nx, struct kern_channel *ch,
697 struct chreq *chr, struct nxbind *nxb, struct proc *p,
698 struct nexus_adapter **ret, boolean_t create)
699 {
700 #pragma unused(ch, p)
701 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
702 struct nexus_kpipe_adapter *kna;
703 ch_endpoint_t ep = chr->cr_endpoint;
704 struct nexus_adapter *na = NULL;
705 int error = 0;
706
707 SK_LOCK_ASSERT_HELD();
708 *ret = NULL;
709
710 #if SK_LOG
711 uuid_string_t uuidstr;
712 SK_D("name \"%s\" spec_uuid \"%s\" port %d mode 0x%b pipe_id %u "
713 "ring_id %d ring_set %u ep_type %u:%u create %u%s",
714 chr->cr_name, sk_uuid_unparse(chr->cr_spec_uuid, uuidstr),
715 (int)chr->cr_port, chr->cr_mode, CHMODE_BITS,
716 chr->cr_pipe_id, (int)chr->cr_ring_id, chr->cr_ring_set,
717 chr->cr_real_endpoint, chr->cr_endpoint, create,
718 (ep != CH_ENDPOINT_KERNEL_PIPE) ? " (skipped)" : "");
719 #endif /* SK_LOG */
720
721 if (ep != CH_ENDPOINT_KERNEL_PIPE) {
722 return 0;
723 }
724
725 if (!create) {
726 return ENODEV;
727 }
728
729 /*
730 * Check client credentials.
731 */
732 if (!NX_ANONYMOUS_PROV(nx) && (nx->nx_arg == NULL || nxb == NULL ||
733 !nxb_is_equal(nx->nx_arg, nxb))) {
734 return EACCES;
735 }
736
737 kna = na_kpipe_alloc(Z_WAITOK);
738
739 na = &kna->kna_up;
740 ASSERT(na->na_type == NA_KERNEL_PIPE);
741 ASSERT(na->na_free == na_kpipe_free);
742
743 (void) snprintf(na->na_name, sizeof(na->na_name),
744 "%s{%u", chr->cr_name, NEXUS_PORT_KERNEL_PIPE_CLIENT);
745 uuid_generate_random(na->na_uuid);
746
747 na->na_txsync = nx_kpipe_na_txsync;
748 na->na_rxsync = nx_kpipe_na_rxsync;
749 na->na_activate = nx_kpipe_na_activate;
750 na->na_dtor = nx_kpipe_na_dtor;
751 na->na_krings_create = nx_kpipe_na_krings_create;
752 na->na_krings_delete = nx_kpipe_na_krings_delete;
753 na_set_nrings(na, NR_TX, nxp->nxp_tx_rings);
754 na_set_nrings(na, NR_RX, nxp->nxp_rx_rings);
755 na_set_nslots(na, NR_TX, nxp->nxp_tx_slots);
756 na_set_nslots(na, NR_RX, nxp->nxp_rx_slots);
757 /*
758 * Verify upper bounds; the parameters must have already been
759 * validated by nxdom_prov_params() by the time we get here.
760 */
761 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
762 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
763 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
764 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
765
766 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
767 NEXUS_STATS_TYPE_INVALID;
768
769 na_attach_common(na, nx, &nx_kpipe_prov_s);
770
771 if ((error = NX_DOM_PROV(nx)->nxdom_prov_mem_new(NX_DOM_PROV(nx),
772 nx, na)) != 0) {
773 ASSERT(na->na_arena == NULL);
774 goto err;
775 }
776 ASSERT(na->na_arena != NULL);
777
778 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
779 ASSERT(na->na_flowadv_max == 0 ||
780 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
781
782 #if SK_LOG
783 SK_DF(SK_VERB_KERNEL_PIPE, "created kpipe adapter 0x%llx", SK_KVA(kna));
784 SK_DF(SK_VERB_KERNEL_PIPE, "na_name: \"%s\"", na->na_name);
785 SK_DF(SK_VERB_KERNEL_PIPE, " UUID: %s",
786 sk_uuid_unparse(na->na_uuid, uuidstr));
787 SK_DF(SK_VERB_KERNEL_PIPE, " nx: 0x%llx (\"%s\":\"%s\")",
788 SK_KVA(na->na_nx), NX_DOM(na->na_nx)->nxdom_name,
789 NX_DOM_PROV(na->na_nx)->nxdom_prov_name);
790 SK_DF(SK_VERB_KERNEL_PIPE, " flags: 0x%b",
791 na->na_flags, NAF_BITS);
792 SK_DF(SK_VERB_KERNEL_PIPE, " flowadv_max: %u", na->na_flowadv_max);
793 SK_DF(SK_VERB_KERNEL_PIPE, " rings: tx %u rx %u",
794 na_get_nrings(na, NR_TX),
795 na_get_nrings(na, NR_RX));
796 SK_DF(SK_VERB_KERNEL_PIPE, " slots: tx %u rx %u",
797 na_get_nslots(na, NR_TX),
798 na_get_nslots(na, NR_RX));
799 #if CONFIG_NEXUS_USER_PIPE
800 SK_DF(SK_VERB_KERNEL_PIPE, " next_pipe: %u", na->na_next_pipe);
801 SK_DF(SK_VERB_KERNEL_PIPE, " max_pipes: %u", na->na_max_pipes);
802 #endif /* CONFIG_NEXUS_USER_PIPE */
803 #endif /* SK_LOG */
804
805 *ret = na;
806 na_retain_locked(*ret);
807
808 return 0;
809
810 err:
811 ASSERT(na != NULL);
812 if (na->na_arena != NULL) {
813 skmem_arena_release(na->na_arena);
814 na->na_arena = NULL;
815 }
816 NA_FREE(na);
817
818 return error;
819 }
820
821 #if (DEVELOPMENT || DEBUG)
822 SYSCTL_NODE(_kern_skywalk, OID_AUTO, kpipe, CTLFLAG_RW | CTLFLAG_LOCKED,
823 0, "Skywalk kpipe tuning");
824 #endif
825