1 /*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/param.h>
30 #include <sys/sdt.h>
31 #include <sys/systm.h>
32 #include <sys/filedesc.h>
33 #include <sys/proc_internal.h>
34 #include <sys/file_internal.h>
35 #include <sys/vnode_internal.h>
36 #include <sys/sysproto.h>
37 #include <security/audit/audit.h>
38 #include <skywalk/os_skywalk_private.h>
39
40 #include <kern/uipc_domain.h>
41
42 static int chop_select(struct fileproc *, int, void *, vfs_context_t);
43 static int chop_close(struct fileglob *, vfs_context_t);
44 static int chop_kqfilter(struct fileproc *, struct knote *, struct kevent_qos_s *);
45
46 static const struct fileops channel_ops = {
47 .fo_type = DTYPE_CHANNEL,
48 .fo_read = fo_no_read,
49 .fo_write = fo_no_write,
50 .fo_ioctl = fo_no_ioctl,
51 .fo_select = chop_select,
52 .fo_close = chop_close,
53 .fo_drain = fo_no_drain,
54 .fo_kqfilter = chop_kqfilter,
55 };
56
57 #if (DEVELOPMENT || DEBUG)
58 static uint32_t ch_force_defunct = 0;
59 SYSCTL_UINT(_kern_skywalk_channel, OID_AUTO, force_defunct,
60 CTLFLAG_RW | CTLFLAG_LOCKED, &ch_force_defunct, 0, "");
61 #endif /* !DEVELOPMENT && !DEBUG */
62
63 /*
64 * -fbounds-safety: Utility macro for the ci_key field of struct ch_init, which
65 * is of type user_addr_t (non-pointer type), which can't be marked as __sized_by.
66 */
67 #define USER_ADDR_TO_PTR(chan_init) \
68 __unsafe_forge_bidi_indexable(void *, (chan_init).ci_key, (chan_init).ci_key_len)
69
70 #if SK_LOG
71 const char *__null_terminated MODE_TAG_TXSYNC = "txsync";
72 const char *__null_terminated MODE_TAG_RXSYNC = "rxsync";
73 #endif /* SK_LOG */
74
75 static int
chop_select(struct fileproc * fp,int which,void * wql,vfs_context_t ctx)76 chop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
77 {
78 int revents = 0, events = 0;
79 struct kern_channel *ch;
80
81 ch = (struct kern_channel *)fp_get_data(fp);
82 if (ch == NULL) {
83 return 0;
84 }
85
86 switch (which) {
87 case FREAD:
88 events = POLLIN;
89 break;
90
91 case FWRITE:
92 events = POLLOUT;
93 break;
94
95 case 0:
96 events = POLLERR;
97 break;
98
99 default:
100 return 1;
101 }
102
103 /* caller will lock channel */
104 revents = ch_select(ch, events, wql, vfs_context_proc(ctx));
105 return (events & revents) ? 1 : 0;
106 }
107
108 static int
chop_close(struct fileglob * fg,vfs_context_t ctx)109 chop_close(struct fileglob *fg, vfs_context_t ctx)
110 {
111 #pragma unused(ctx)
112 struct kern_channel *ch;
113
114 ch = (struct kern_channel *)fg_get_data(fg);
115 fg_set_data(fg, NULL);
116 if (ch != NULL) {
117 ch_dtor(ch);
118 }
119
120 return 0;
121 }
122
123 /*
124 * This is called when a thread invokes kevent() to record
125 * a change in the configuration of the kqueue().
126 */
127 static int
chop_kqfilter(struct fileproc * fp,struct knote * kn,struct kevent_qos_s * kev)128 chop_kqfilter(struct fileproc *fp, struct knote *kn, struct kevent_qos_s *kev)
129 {
130 struct kern_channel *ch = (struct kern_channel *)fp_get_data(fp);
131
132 /* caller will lock channel */
133 return ch_kqfilter(ch, kn, kev);
134 }
135
136 int
__channel_open(struct proc * p,struct __channel_open_args * uap,int * retval)137 __channel_open(struct proc *p, struct __channel_open_args *uap, int *retval)
138 {
139 struct fileproc *__single fp = NULL;
140 struct kern_channel *ch = NULL;
141 struct ch_init init;
142 int fd = -1, err = 0;
143 void *key = NULL;
144 uint32_t keylen;
145 guardid_t guard;
146
147 if (__improbable(uap->init == USER_ADDR_NULL ||
148 uap->init_len < sizeof(init))) {
149 SK_PERR(p, "EINVAL: init %p, init_len %u", SK_KVA(uap->init),
150 uap->init_len);
151 err = EINVAL;
152 goto done;
153 }
154
155 err = copyin(uap->init, (caddr_t)&init, sizeof(init));
156 if (__improbable(err != 0)) {
157 SK_PERR(p, "copyin err %u: init %p", err, SK_KVA(uap->init));
158 goto done;
159 }
160
161 if (__improbable(init.ci_version != CHANNEL_INIT_CURRENT_VERSION)) {
162 SK_PERR(p, "ENOTSUP: init.ci_version %u != %u", init.ci_version,
163 CHANNEL_INIT_CURRENT_VERSION);
164 err = ENOTSUP;
165 goto done;
166 } else if (__improbable(uuid_is_null(init.ci_nx_uuid))) {
167 SK_PERR(p, "EINVAL: uuid_is_null");
168 err = EINVAL;
169 goto done;
170 } else if (__improbable((init.ci_key_len != 0 &&
171 init.ci_key == USER_ADDR_NULL) ||
172 (init.ci_key_len == 0 && init.ci_key != USER_ADDR_NULL))) {
173 SK_PERR(p, "EINVAL: ci_key_len %i, ci_key %p",
174 init.ci_key_len, SK_KVA(init.ci_key));
175 err = EINVAL;
176 goto done;
177 }
178
179 if ((init.ci_ch_mode & CHMODE_EVENT_RING) != 0) {
180 if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) == 0) {
181 SK_PERR(p, "EINVAL: PACKET_POOL is required for EVENT ring");
182 err = EINVAL;
183 goto done;
184 }
185 }
186
187 #if (DEVELOPMENT || DEBUG)
188 if (__improbable(ch_force_defunct)) {
189 init.ci_ch_mode |= CHMODE_DEFUNCT_OK;
190 }
191 #endif /* !DEVELOPMENT && !DEBUG */
192
193 /* generate guard ID based on nexus instance UUID */
194 sk_gen_guard_id(TRUE, init.ci_nx_uuid, &guard);
195
196 err = falloc_guarded(p, &fp, &fd, vfs_context_current(), &guard,
197 GUARD_CLOSE | GUARD_DUP | GUARD_SOCKET_IPC | GUARD_FILEPORT | GUARD_WRITE);
198 if (__improbable(err != 0)) {
199 SK_PERR(p, "falloc_guarded: %u", err);
200 goto done;
201 }
202
203 keylen = init.ci_key_len;
204 if (keylen != 0) {
205 if (__improbable(keylen > NEXUS_MAX_KEY_LEN)) {
206 SK_PERR(p, "EINVAL: ci_key_len %u", keylen);
207 err = EINVAL;
208 goto done;
209 }
210
211 key = sk_alloc_data(keylen, Z_WAITOK, skmem_tag_ch_key);
212 if (__improbable(key == NULL)) {
213 SK_PERR(p, "ENOMEM: ci_key_len %u", keylen);
214 err = ENOMEM;
215 goto done;
216 }
217
218 err = copyin(init.ci_key, (caddr_t)key, keylen);
219 if (__improbable(err != 0)) {
220 SK_PERR(p, "copyin err %u: ci_key %p, ci_key_len %u",
221 err, SK_KVA(init.ci_key), keylen);
222 goto done;
223 }
224 }
225
226 /* let ch_open() take over this key upon success */
227 init.ci_key = (user_addr_t)key;
228 key = NULL;
229
230 if (__improbable((ch = ch_open(&init, p, fd, &err)) == NULL)) {
231 /* in case not processed */
232 key = USER_ADDR_TO_PTR(init);
233 ASSERT(err != 0);
234 SK_PERR(p, "ch_open nx_port %d err %u",
235 (int)init.ci_nx_port, err);
236 goto done;
237 }
238 /* in case not processed */
239 key = USER_ADDR_TO_PTR(init);
240
241 /* update userland with respect to guard ID, etc. */
242 init.ci_guard = guard;
243 init.ci_key = USER_ADDR_NULL;
244 err = copyout(&init, uap->init, sizeof(init));
245 if (__improbable(err != 0)) {
246 SK_PERR(p, "copyout err %u: init %p", err,
247 SK_KVA(uap->init));
248 goto done;
249 }
250
251 fp->fp_flags |= FP_CLOEXEC | FP_CLOFORK;
252 fp->fp_glob->fg_flag &= ~(FREAD | FWRITE);
253 fp->fp_glob->fg_ops = &channel_ops;
254 fp_set_data(fp, ch); /* ref from ch_open */
255
256 proc_fdlock(p);
257 procfdtbl_releasefd(p, fd, NULL);
258 fp_drop(p, fd, fp, 1);
259 proc_fdunlock(p);
260
261 *retval = fd;
262
263 SK_D("%s(%d) nx_port %d fd %d %s", sk_proc_name(p),
264 sk_proc_pid(p), (int)init.ci_nx_port, fd, ch->ch_na->na_name);
265
266 done:
267 if (key != NULL) {
268 sk_free_data(key, keylen);
269 key = NULL;
270 }
271 if (__improbable(err != 0)) {
272 if (ch != NULL) {
273 ch_dtor(ch);
274 ch = NULL;
275 }
276 if (fp != NULL) {
277 fp_free(p, fd, fp);
278 fp = NULL;
279 }
280 }
281
282 return err;
283 }
284
285 int
__channel_get_info(struct proc * p,struct __channel_get_info_args * uap,int * retval)286 __channel_get_info(struct proc *p, struct __channel_get_info_args *uap,
287 int *retval)
288 {
289 #pragma unused(retval)
290 struct fileproc *__single fp;
291 struct kern_channel *ch = NULL;
292 int err = 0;
293
294 AUDIT_ARG(fd, uap->c);
295
296 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
297 if (__improbable(err != 0)) {
298 SK_PERR(p, "fp_get_ftype err %u", err);
299 return err;
300 }
301 ch = (struct kern_channel *__single)fp_get_data(fp);
302
303 if (__improbable(uap->cinfo == USER_ADDR_NULL ||
304 uap->cinfolen < sizeof(struct ch_info))) {
305 SK_PERR(p, "EINVAL: cinfo %p, cinfolen %u",
306 SK_KVA(uap->cinfo), uap->cinfolen);
307 err = EINVAL;
308 goto done;
309 }
310
311 lck_mtx_lock(&ch->ch_lock);
312 err = copyout(ch->ch_info, uap->cinfo, sizeof(struct ch_info));
313 lck_mtx_unlock(&ch->ch_lock);
314 if (__improbable(err != 0)) {
315 SK_PERR(p, "copyout err %u: cinfo %p", err,
316 SK_KVA(uap->cinfo));
317 goto done;
318 }
319
320 done:
321 fp_drop(p, uap->c, fp, 0);
322
323 return err;
324 }
325
326 #if SK_LOG
327 /* Hoisted out of line to reduce kernel stack footprint */
328 SK_LOG_ATTRIBUTE
329 static void
channel_sync_log1(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct kern_channel * ch,const struct __kern_channel_ring * kring,ring_id_t i)330 channel_sync_log1(uint64_t verb, const char *sync, struct proc *p,
331 const struct nexus_adapter *na, const struct kern_channel *ch,
332 const struct __kern_channel_ring *kring, ring_id_t i)
333 {
334 verb |= SK_VERB_SYNC;
335 SK_DF(verb, "%s(%d) pre: %s ring %u na \"%s\" (%p) ch %p "
336 "th %p h %u kh %u", sk_proc_name(p), sk_proc_pid(p),
337 sync, i, na->na_name, SK_KVA(na), SK_KVA(ch),
338 SK_KVA(current_thread()), kring->ckr_ring->ring_head,
339 kring->ckr_khead);
340 }
341
342 SK_LOG_ATTRIBUTE
343 static void
channel_sync_log2(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct __kern_channel_ring * kring,ring_id_t i)344 channel_sync_log2(uint64_t verb, const char *sync, struct proc *p,
345 const struct nexus_adapter *na, const struct __kern_channel_ring *kring,
346 ring_id_t i)
347 {
348 verb |= SK_VERB_SYNC;
349 SK_DF(verb, "%s(%d) post: %s ring %u na \"%s\" h %u kh %u",
350 sk_proc_name(p), sk_proc_pid(p), sync, i, na->na_name,
351 kring->ckr_ring->ring_head, kring->ckr_khead);
352 }
353 #endif /* SK_LOG */
354
355 int
__channel_sync(struct proc * p,struct __channel_sync_args * uap,int * retval)356 __channel_sync(struct proc *p, struct __channel_sync_args *uap, int *retval)
357 {
358 #pragma unused(retval)
359 struct fileproc *__single fp;
360 sk_protect_t protect = NULL;
361 struct nexus_adapter *na;
362 struct __kern_channel_ring *krings, *kring;
363 struct kern_channel *ch;
364 sync_mode_t mode;
365 ring_id_t i, qfirst, qlast;
366 sync_flags_t flags, upp_sync_flags = 0;
367 enum txrx t;
368 int err;
369 int s;
370
371 net_update_uptime();
372
373 AUDIT_ARG(fd, uap->c);
374
375 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
376 if (__improbable(err != 0)) {
377 SK_PERR(p, "fp_get_ftype err %u", err);
378 return err;
379 }
380 ch = (struct kern_channel *__single)fp_get_data(fp);
381
382 lck_mtx_lock(&ch->ch_lock);
383 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
384
385 mode = uap->mode;
386 flags = uap->flags;
387 if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX &&
388 mode != CHANNEL_SYNC_UPP)) {
389 SK_PERR(p, "EINVAL: mode %u", mode);
390 err = EINVAL;
391 goto done;
392 }
393
394 if (__improbable((ch->ch_flags & CHANF_USER_PACKET_POOL) == 0 &&
395 (flags & (CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE |
396 CHANNEL_SYNCF_ALLOC_BUF)) != 0)) {
397 SK_PERR(p, "EINVAL: !CHANF_USER_PACKET_POOL with "
398 "SYNCF_ALLOC/FREE");
399 err = EINVAL;
400 goto done;
401 }
402
403 if (__improbable(ch->ch_flags & CHANF_DEFUNCT)) {
404 SK_PERR(p, "channel is defunct");
405 err = ENXIO;
406 goto done;
407 }
408
409 /* clear CHANF_DEFUNCT_SKIP if it was set during defunct last time */
410 if (__improbable(ch->ch_flags & CHANF_DEFUNCT_SKIP)) {
411 os_atomic_andnot(&ch->ch_flags, CHANF_DEFUNCT_SKIP, relaxed);
412 }
413
414 na = ch->ch_na; /* we have a reference */
415 ASSERT(na != NULL);
416 ASSERT(NA_IS_ACTIVE(na));
417
418 if (__improbable(na_reject_channel(ch, na))) {
419 SK_PERR(p, "channel is non-permissive");
420 err = ENXIO;
421 goto done;
422 }
423
424 /* mark thread with sync-in-progress flag */
425 protect = sk_sync_protect();
426
427 /* update our work timestamp */
428 na->na_work_ts = net_uptime();
429
430 /* and make this channel eligible for draining again */
431 if (na->na_flags & NAF_DRAINING) {
432 os_atomic_andnot(&na->na_flags, NAF_DRAINING, relaxed);
433 }
434
435 if (mode == CHANNEL_SYNC_UPP) {
436 upp_sync_flags = NA_SYNCF_FORCE_UPP_SYNC;
437 if (flags & CHANNEL_SYNCF_PURGE) {
438 upp_sync_flags |= NA_SYNCF_UPP_PURGE;
439 }
440 goto packet_pool_sync;
441 }
442
443 t = (mode == CHANNEL_SYNC_TX ? NR_TX : NR_RX);
444 krings = NAKR(na, t);
445 qfirst = ch->ch_first[t];
446 qlast = ch->ch_last[t];
447
448 for (i = qfirst; i < qlast; i++) {
449 kring = krings + i;
450 s = kr_enter(kring, TRUE);
451 ASSERT(s == 0);
452 #if SK_LOG
453 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
454 channel_sync_log1((mode == CHANNEL_SYNC_TX) ?
455 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
456 MODE_TAG_TXSYNC : MODE_TAG_RXSYNC, p, na, ch, kring, i);
457 }
458 #endif /* SK_LOG */
459
460 switch (mode) {
461 case CHANNEL_SYNC_TX: {
462 int error = 0;
463
464 DTRACE_SKYWALK2(pretxprologue, struct kern_channel *,
465 ch, struct __kern_channel_ring *, kring);
466
467 if (kr_txsync_prologue(ch, kring, p) >=
468 kring->ckr_num_slots) {
469 kr_log_bad_ring(kring);
470 error = EFAULT;
471 if (!err) {
472 SK_PERR(p, "EFAULT: "
473 "kr_txsync_prologue()");
474 err = EFAULT;
475 }
476 } else if (kring->ckr_na_sync(kring, p,
477 NA_SYNCF_FORCE_RECLAIM) == 0) {
478 kr_txsync_finalize(ch, kring, p);
479 } else {
480 error = EIO;
481 if (!err) {
482 SK_PERR(p, "EIO: TX "
483 "kring->ckr_na_sync()");
484 err = EIO;
485 }
486 }
487
488 DTRACE_SKYWALK3(posttxfinalize, struct kern_channel *,
489 ch, struct __kern_channel_ring *, kring, int,
490 error);
491 }
492 break;
493
494 case CHANNEL_SYNC_RX: {
495 int error = 0;
496
497 DTRACE_SKYWALK2(prerxprologue, struct kern_channel *,
498 ch, struct __kern_channel_ring *, kring);
499
500 if (kr_rxsync_prologue(ch, kring, p) >=
501 kring->ckr_num_slots) {
502 kr_log_bad_ring(kring);
503 error = EFAULT;
504 if (!err) {
505 SK_PERR(p, "EFAULT: "
506 "kr_rxsync_prologue()");
507 err = EFAULT;
508 }
509 } else if (kring->ckr_na_sync(kring, p,
510 NA_SYNCF_FORCE_READ) == 0) {
511 kr_rxsync_finalize(ch, kring, p);
512 } else {
513 error = EIO;
514 if (!err) {
515 SK_PERR(p, "EIO: " "RX "
516 "kring->ckr_na_sync()");
517 err = EIO;
518 }
519 }
520
521 DTRACE_SKYWALK3(postrxfinalize, struct kern_channel *,
522 ch, struct __kern_channel_ring *, kring, int,
523 error);
524 }
525 break;
526
527 default:
528 VERIFY(0);
529 /* NOTREACHED */
530 __builtin_unreachable();
531 }
532
533 #if SK_LOG
534 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
535 channel_sync_log2((mode == CHANNEL_SYNC_TX) ?
536 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
537 MODE_TAG_TXSYNC : MODE_TAG_RXSYNC, p, na, kring, i);
538 }
539 #endif /* SK_LOG */
540
541 kr_exit(kring);
542 }
543
544 packet_pool_sync:
545 if (flags & (CHANNEL_SYNCF_ALLOC |
546 CHANNEL_SYNCF_LARGE_ALLOC | CHANNEL_SYNCF_ALLOC_BUF)) {
547 enum txrx type;
548
549 if (flags & CHANNEL_SYNCF_LARGE_ALLOC) {
550 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC));
551 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC_BUF));
552 type = NR_LBA;
553 qfirst = ch->ch_first[type];
554 qlast = ch->ch_last[type];
555 } else {
556 type = NR_A;
557 qfirst = ch->ch_first[type];
558 qlast = ch->ch_last[type];
559 if (!(flags & CHANNEL_SYNCF_ALLOC)) {
560 qfirst++;
561 } else if ((qlast - qfirst) > 1 &&
562 !(flags & CHANNEL_SYNCF_ALLOC_BUF)) {
563 qlast--;
564 }
565 }
566 ASSERT(qfirst != qlast);
567 krings = NAKR(na, type);
568
569 for (i = qfirst; i < qlast; i++) {
570 kring = krings + i;
571 s = kr_enter(kring, TRUE);
572 ASSERT(s == 0);
573 #if SK_LOG
574 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
575 channel_sync_log1(0, "alloc-sync", p, na, ch,
576 kring, i);
577 }
578 #endif /* SK_LOG */
579 if (__improbable(kr_alloc_sync_prologue(kring, p) >=
580 kring->ckr_num_slots)) {
581 kr_log_bad_ring(kring);
582 if (!err) {
583 SK_PERR(p,
584 "EFAULT: kr_alloc_sync_prologue()");
585 err = EFAULT;
586 }
587 } else if (kring->ckr_na_sync(kring, p,
588 upp_sync_flags) == 0) {
589 kr_alloc_sync_finalize(kring, p);
590 } else {
591 if (!err) {
592 SK_PERR(p,
593 "EIO: ALLOC: ring->ckr_na_sync()");
594 err = EIO;
595 }
596 skywalk_kill_process(p,
597 SKYWALK_KILL_REASON_SYNC_FAILED |
598 SKYWALK_KILL_REASON_ALLOC_SYNC);
599 }
600 #if SK_LOG
601 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
602 channel_sync_log2(0, "alloc-sync", p, na,
603 kring, i);
604 }
605 #endif /* SK_LOG */
606 kr_exit(kring);
607 }
608 } /* CHANNEL_SYNCF_ALLOC */
609
610 if (flags & CHANNEL_SYNCF_FREE) {
611 qfirst = ch->ch_first[NR_F];
612 qlast = ch->ch_last[NR_F];
613 ASSERT(qfirst != qlast);
614 krings = NAKR(na, NR_F);
615
616 for (i = qfirst; i < qlast; i++) {
617 kring = krings + i;
618 s = kr_enter(kring, TRUE);
619 ASSERT(s == 0);
620 #if SK_LOG
621 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
622 channel_sync_log1(0, "free-sync", p, na, ch,
623 kring, i);
624 }
625 #endif /* SK_LOG */
626 if (__improbable(kr_free_sync_prologue(kring, p) >=
627 kring->ckr_num_slots)) {
628 kr_log_bad_ring(kring);
629 if (!err) {
630 SK_PERR(p,
631 "EFAULT: kr_free_sync_prologue()");
632 err = EFAULT;
633 }
634 } else if (kring->ckr_na_sync(kring, p,
635 upp_sync_flags) == 0) {
636 kr_free_sync_finalize(kring, p);
637 } else {
638 if (!err) {
639 SK_PERR(p,
640 "EIO: FREE: ring->ckr_na_sync()");
641 err = EIO;
642 }
643 skywalk_kill_process(p,
644 SKYWALK_KILL_REASON_SYNC_FAILED |
645 SKYWALK_KILL_REASON_FREE_SYNC);
646 }
647 #if SK_LOG
648 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
649 channel_sync_log2(0, "free-sync", p, na,
650 kring, i);
651 }
652 #endif /* SK_LOG */
653 kr_exit(kring);
654 }
655 } /* CHANNEL_SYNCF_FREE */
656
657 /* unmark thread with sync-in-progress flag */
658 sk_sync_unprotect(protect);
659
660 done:
661 lck_mtx_unlock(&ch->ch_lock);
662 fp_drop(p, uap->c, fp, 0);
663
664 VERIFY(!sk_is_sync_protected());
665
666 return err;
667 }
668
669 int
__channel_get_opt(struct proc * p,struct __channel_get_opt_args * uap,int * retval)670 __channel_get_opt(struct proc *p, struct __channel_get_opt_args *uap,
671 int *retval)
672 {
673 #pragma unused(retval)
674 struct fileproc *__single fp;
675 struct kern_channel *ch = NULL;
676 struct sockopt sopt;
677 uint32_t optlen;
678 int err = 0;
679
680 AUDIT_ARG(fd, uap->c);
681
682 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
683 if (err != 0) {
684 SK_PERR(p, "fp_get_ftype err %u", err);
685 return err;
686 }
687 ch = (struct kern_channel *__single)fp_get_data(fp);
688
689 if (uap->aoptlen == USER_ADDR_NULL) {
690 SK_PERR(p, "EINVAL: uap->aoptlen == USER_ADDR_NULL");
691 err = EINVAL;
692 goto done;
693 }
694
695 if (uap->aoptval != USER_ADDR_NULL) {
696 err = copyin(uap->aoptlen, &optlen, sizeof(optlen));
697 if (err != 0) {
698 SK_PERR(p, "copyin err %u: aoptlen %p", err,
699 SK_KVA(uap->aoptlen));
700 goto done;
701 }
702 } else {
703 optlen = 0;
704 }
705
706 bzero(&sopt, sizeof(sopt));
707 sopt.sopt_dir = SOPT_GET;
708 sopt.sopt_name = uap->opt;
709 sopt.sopt_val = uap->aoptval;
710 sopt.sopt_valsize = optlen;
711 sopt.sopt_p = p;
712
713 lck_mtx_lock(&ch->ch_lock);
714 err = ch_get_opt(ch, &sopt);
715 lck_mtx_unlock(&ch->ch_lock);
716 if (err == 0) {
717 optlen = (uint32_t)sopt.sopt_valsize;
718 err = copyout(&optlen, uap->aoptlen, sizeof(optlen));
719 #if SK_LOG
720 if (err != 0) {
721 SK_PERR(p, "copyout err %u: aoptlen %p", err,
722 SK_KVA(uap->aoptlen));
723 }
724 #endif
725 }
726
727 done:
728 fp_drop(p, uap->c, fp, 0);
729
730 return err;
731 }
732
733 int
__channel_set_opt(struct proc * p,struct __channel_set_opt_args * uap,int * retval)734 __channel_set_opt(struct proc *p, struct __channel_set_opt_args *uap,
735 int *retval)
736 {
737 #pragma unused(retval)
738 struct fileproc *__single fp;
739 struct kern_channel *ch = NULL;
740 struct sockopt sopt;
741 int err = 0;
742
743 AUDIT_ARG(fd, uap->c);
744
745 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
746 if (err != 0) {
747 SK_PERR(p, "fp_get_ftype err %u", err);
748 return err;
749 }
750 ch = (struct kern_channel *__single)fp_get_data(fp);
751
752 bzero(&sopt, sizeof(sopt));
753 sopt.sopt_dir = SOPT_SET;
754 sopt.sopt_name = uap->opt;
755 sopt.sopt_val = uap->aoptval;
756 sopt.sopt_valsize = uap->optlen;
757 sopt.sopt_p = p;
758
759 lck_mtx_lock(&ch->ch_lock);
760 if (__improbable(ch->ch_flags & (CHANF_CLOSING | CHANF_DEFUNCT))) {
761 SK_PERR(p, "channel is closing/defunct");
762 err = ENXIO;
763 } else if (__improbable(ch->ch_na == NULL ||
764 !NA_IS_ACTIVE(ch->ch_na) || na_reject_channel(ch, ch->ch_na))) {
765 SK_PERR(p, "channel is non-permissive");
766 err = ENXIO;
767 } else {
768 err = ch_set_opt(ch, &sopt);
769 }
770 lck_mtx_unlock(&ch->ch_lock);
771
772 fp_drop(p, uap->c, fp, 0);
773
774 #if SK_LOG
775 if (err != 0) {
776 SK_PERR(p, "ch_set_opt() err %u", err);
777 }
778 #endif
779
780 return err;
781 }
782