1 /*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/param.h>
30 #include <sys/sdt.h>
31 #include <sys/systm.h>
32 #include <sys/filedesc.h>
33 #include <sys/proc_internal.h>
34 #include <sys/file_internal.h>
35 #include <sys/vnode_internal.h>
36 #include <sys/sysproto.h>
37 #include <security/audit/audit.h>
38 #include <skywalk/os_skywalk_private.h>
39
40 static int chop_select(struct fileproc *, int, void *, vfs_context_t);
41 static int chop_close(struct fileglob *, vfs_context_t);
42 static int chop_kqfilter(struct fileproc *, struct knote *, struct kevent_qos_s *);
43
44 static const struct fileops channel_ops = {
45 .fo_type = DTYPE_CHANNEL,
46 .fo_read = fo_no_read,
47 .fo_write = fo_no_write,
48 .fo_ioctl = fo_no_ioctl,
49 .fo_select = chop_select,
50 .fo_close = chop_close,
51 .fo_drain = fo_no_drain,
52 .fo_kqfilter = chop_kqfilter,
53 };
54
55 #if (DEVELOPMENT || DEBUG)
56 static uint32_t ch_force_defunct = 0;
57 SYSCTL_UINT(_kern_skywalk_channel, OID_AUTO, force_defunct,
58 CTLFLAG_RW | CTLFLAG_LOCKED, &ch_force_defunct, 0, "");
59 #endif /* !DEVELOPMENT && !DEBUG */
60
61 static int
chop_select(struct fileproc * fp,int which,void * wql,vfs_context_t ctx)62 chop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
63 {
64 int revents = 0, events = 0;
65 struct kern_channel *ch;
66
67 ch = (struct kern_channel *)fp_get_data(fp);
68 if (ch == NULL) {
69 return 0;
70 }
71
72 switch (which) {
73 case FREAD:
74 events = POLLIN;
75 break;
76
77 case FWRITE:
78 events = POLLOUT;
79 break;
80
81 case 0:
82 events = POLLERR;
83 break;
84
85 default:
86 return 1;
87 }
88
89 /* caller will lock channel */
90 revents = ch_select(ch, events, wql, vfs_context_proc(ctx));
91 return (events & revents) ? 1 : 0;
92 }
93
94 static int
chop_close(struct fileglob * fg,vfs_context_t ctx)95 chop_close(struct fileglob *fg, vfs_context_t ctx)
96 {
97 #pragma unused(ctx)
98 struct kern_channel *ch;
99
100 ch = (struct kern_channel *)fg_get_data(fg);
101 fg_set_data(fg, NULL);
102 if (ch != NULL) {
103 ch_dtor(ch);
104 }
105
106 return 0;
107 }
108
109 /*
110 * This is called when a thread invokes kevent() to record
111 * a change in the configuration of the kqueue().
112 */
113 static int
chop_kqfilter(struct fileproc * fp,struct knote * kn,struct kevent_qos_s * kev)114 chop_kqfilter(struct fileproc *fp, struct knote *kn, struct kevent_qos_s *kev)
115 {
116 struct kern_channel *ch = (struct kern_channel *)fp_get_data(fp);
117
118 /* caller will lock channel */
119 return ch_kqfilter(ch, kn, kev);
120 }
121
122 int
__channel_open(struct proc * p,struct __channel_open_args * uap,int * retval)123 __channel_open(struct proc *p, struct __channel_open_args *uap, int *retval)
124 {
125 struct fileproc *fp = NULL;
126 struct kern_channel *ch = NULL;
127 struct ch_init init;
128 int fd = -1, err = 0;
129 void *key = NULL;
130 uint32_t keylen;
131 guardid_t guard;
132
133 if (__improbable(uap->init == USER_ADDR_NULL ||
134 uap->init_len < sizeof(init))) {
135 SK_DSC(p, "EINVAL: init 0x%llx, init_len %u", SK_KVA(uap->init),
136 uap->init_len);
137 err = EINVAL;
138 goto done;
139 }
140
141 err = copyin(uap->init, (caddr_t)&init, sizeof(init));
142 if (__improbable(err != 0)) {
143 SK_DSC(p, "copyin err %u: init 0x%llx", err, SK_KVA(uap->init));
144 goto done;
145 }
146
147 if (__improbable(init.ci_version != CHANNEL_INIT_CURRENT_VERSION)) {
148 SK_DSC(p, "ENOTSUP: init.ci_version %u != %u", init.ci_version,
149 CHANNEL_INIT_CURRENT_VERSION);
150 err = ENOTSUP;
151 goto done;
152 } else if (__improbable(uuid_is_null(init.ci_nx_uuid))) {
153 SK_DSC(p, "EINVAL: uuid_is_null");
154 err = EINVAL;
155 goto done;
156 } else if (__improbable((init.ci_key_len != 0 &&
157 init.ci_key == USER_ADDR_NULL) ||
158 (init.ci_key_len == 0 && init.ci_key != USER_ADDR_NULL))) {
159 SK_DSC(p, "EINVAL: ci_key_len %i, ci_key 0x%llx",
160 init.ci_key_len, SK_KVA(init.ci_key));
161 err = EINVAL;
162 goto done;
163 }
164
165 if ((init.ci_ch_mode & CHMODE_MONITOR) != 0) {
166 if (__improbable((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) != 0)) {
167 SK_DSC(p, "EINVAL: PACKET_POOL not supported for MONITOR mode");
168 err = EINVAL;
169 goto done;
170 }
171 if (__improbable((init.ci_ch_mode & CHMODE_EVENT_RING) != 0)) {
172 SK_DSC(p, "EINVAL: EVENT ring not supported for MONITOR mode");
173 err = EINVAL;
174 goto done;
175 }
176 if (__improbable((init.ci_ch_mode & CHMODE_LOW_LATENCY) != 0)) {
177 SK_DSC(p, "EINVAL: low latency not supported for MONITOR mode");
178 err = EINVAL;
179 goto done;
180 }
181 }
182
183 if ((init.ci_ch_mode & CHMODE_EVENT_RING) != 0) {
184 if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) == 0) {
185 SK_DSC(p, "EINVAL: PACKET_POOL is required for EVENT ring");
186 err = EINVAL;
187 goto done;
188 }
189 }
190
191 #if (DEVELOPMENT || DEBUG)
192 if (__improbable(ch_force_defunct)) {
193 init.ci_ch_mode |= CHMODE_DEFUNCT_OK;
194 }
195 #endif /* !DEVELOPMENT && !DEBUG */
196
197 /* generate guard ID based on nexus instance UUID */
198 sk_gen_guard_id(TRUE, init.ci_nx_uuid, &guard);
199
200 err = falloc_guarded(p, &fp, &fd, vfs_context_current(), &guard,
201 GUARD_CLOSE | GUARD_DUP | GUARD_SOCKET_IPC | GUARD_FILEPORT | GUARD_WRITE);
202 if (__improbable(err != 0)) {
203 SK_DSC(p, "falloc_guarded: %u", err);
204 goto done;
205 }
206
207 keylen = init.ci_key_len;
208 if (keylen != 0) {
209 if (__improbable(keylen > NEXUS_MAX_KEY_LEN)) {
210 SK_DSC(p, "EINVAL: ci_key_len %u", keylen);
211 err = EINVAL;
212 goto done;
213 }
214
215 key = sk_alloc_data(keylen, Z_WAITOK, skmem_tag_ch_key);
216 if (__improbable(key == NULL)) {
217 SK_DSC(p, "ENOMEM: ci_key_len %u", keylen);
218 err = ENOMEM;
219 goto done;
220 }
221
222 err = copyin(init.ci_key, (caddr_t)key, keylen);
223 if (__improbable(err != 0)) {
224 SK_DSC(p, "copyin err %u: ci_key 0x%llx, ci_key_len %u",
225 err, SK_KVA(init.ci_key), keylen);
226 goto done;
227 }
228 }
229
230 /* let ch_open() take over this key upon success */
231 init.ci_key = (user_addr_t)key;
232 key = NULL;
233
234 if (__improbable((ch = ch_open(&init, p, fd, &err)) == NULL)) {
235 /* in case not processed */
236 key = (void *)init.ci_key;
237 ASSERT(err != 0);
238 SK_DSC(p, "ch_open nx_port %d err %u",
239 (int)init.ci_nx_port, err);
240 goto done;
241 }
242 /* in case not processed */
243 key = (void *)init.ci_key;
244
245 /* update userland with respect to guard ID, etc. */
246 init.ci_guard = guard;
247 init.ci_key = USER_ADDR_NULL;
248 err = copyout(&init, uap->init, sizeof(init));
249 if (__improbable(err != 0)) {
250 SK_DSC(p, "copyout err %u: init 0x%llx", err,
251 SK_KVA(uap->init));
252 goto done;
253 }
254
255 fp->fp_flags |= FP_CLOEXEC | FP_CLOFORK;
256 fp->fp_glob->fg_flag &= ~(FREAD | FWRITE);
257 fp->fp_glob->fg_ops = &channel_ops;
258 fp_set_data(fp, ch); /* ref from ch_open */
259
260 proc_fdlock(p);
261 procfdtbl_releasefd(p, fd, NULL);
262 fp_drop(p, fd, fp, 1);
263 proc_fdunlock(p);
264
265 *retval = fd;
266
267 SK_D("%s(%d) nx_port %d fd %d guard 0x%llx",
268 sk_proc_name_address(p), sk_proc_pid(p), (int)init.ci_nx_port,
269 fd, guard);
270
271 done:
272 if (key != NULL) {
273 sk_free_data(key, keylen);
274 key = NULL;
275 }
276 if (__improbable(err != 0)) {
277 if (ch != NULL) {
278 ch_dtor(ch);
279 ch = NULL;
280 }
281 if (fp != NULL) {
282 fp_free(p, fd, fp);
283 fp = NULL;
284 }
285 }
286
287 return err;
288 }
289
290 int
__channel_get_info(struct proc * p,struct __channel_get_info_args * uap,int * retval)291 __channel_get_info(struct proc *p, struct __channel_get_info_args *uap,
292 int *retval)
293 {
294 #pragma unused(retval)
295 struct fileproc *fp;
296 struct kern_channel *ch = NULL;
297 int err = 0;
298
299 AUDIT_ARG(fd, uap->c);
300
301 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
302 if (__improbable(err != 0)) {
303 SK_DSC(p, "fp_get_ftype err %u", err);
304 return err;
305 }
306 ch = fp_get_data(fp);
307
308 if (__improbable(uap->cinfo == USER_ADDR_NULL ||
309 uap->cinfolen < sizeof(struct ch_info))) {
310 SK_DSC(p, "EINVAL: cinfo 0x%llx, cinfolen %u",
311 SK_KVA(uap->cinfo), uap->cinfolen);
312 err = EINVAL;
313 goto done;
314 }
315
316 lck_mtx_lock(&ch->ch_lock);
317 err = copyout(ch->ch_info, uap->cinfo, sizeof(struct ch_info));
318 lck_mtx_unlock(&ch->ch_lock);
319 if (__improbable(err != 0)) {
320 SK_DSC(p, "copyout err %u: cinfo 0x%llx", err,
321 SK_KVA(uap->cinfo));
322 goto done;
323 }
324
325 done:
326 fp_drop(p, uap->c, fp, 0);
327
328 return err;
329 }
330
331 #if SK_LOG
332 /* Hoisted out of line to reduce kernel stack footprint */
333 SK_LOG_ATTRIBUTE
334 static void
channel_sync_log1(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct kern_channel * ch,const struct __kern_channel_ring * kring,ring_id_t i)335 channel_sync_log1(uint64_t verb, const char *sync, struct proc *p,
336 const struct nexus_adapter *na, const struct kern_channel *ch,
337 const struct __kern_channel_ring *kring, ring_id_t i)
338 {
339 verb |= SK_VERB_SYNC;
340 SK_DF(verb, "%s(%d) pre: %s ring %u na \"%s\" (0x%llx) ch 0x%llx "
341 "th 0x%llx h %u kh %u", sk_proc_name_address(p), sk_proc_pid(p),
342 sync, i, na->na_name, SK_KVA(na), SK_KVA(ch),
343 SK_KVA(current_thread()), kring->ckr_ring->ring_head,
344 kring->ckr_khead);
345 }
346
347 SK_LOG_ATTRIBUTE
348 static void
channel_sync_log2(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct __kern_channel_ring * kring,ring_id_t i)349 channel_sync_log2(uint64_t verb, const char *sync, struct proc *p,
350 const struct nexus_adapter *na, const struct __kern_channel_ring *kring,
351 ring_id_t i)
352 {
353 verb |= SK_VERB_SYNC;
354 SK_DF(verb, "%s(%d) post: %s ring %u na \"%s\" h %u kh %u",
355 sk_proc_name_address(p), sk_proc_pid(p), sync, i, na->na_name,
356 kring->ckr_ring->ring_head, kring->ckr_khead);
357 }
358 #endif /* SK_LOG */
359
360 int
__channel_sync(struct proc * p,struct __channel_sync_args * uap,int * retval)361 __channel_sync(struct proc *p, struct __channel_sync_args *uap, int *retval)
362 {
363 #pragma unused(retval)
364 struct fileproc *fp;
365 sk_protect_t protect = NULL;
366 struct nexus_adapter *na;
367 struct __kern_channel_ring *krings, *kring;
368 struct kern_channel *ch;
369 sync_mode_t mode;
370 ring_id_t i, qfirst, qlast;
371 sync_flags_t flags, upp_sync_flags = 0;
372 enum txrx t;
373 int err;
374 int s;
375
376 net_update_uptime();
377
378 AUDIT_ARG(fd, uap->c);
379
380 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
381 if (__improbable(err != 0)) {
382 SK_DSC(p, "fp_get_ftype err %u", err);
383 return err;
384 }
385 ch = fp_get_data(fp);
386
387 lck_mtx_lock(&ch->ch_lock);
388 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
389
390 mode = uap->mode;
391 flags = uap->flags;
392 if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX &&
393 mode != CHANNEL_SYNC_UPP)) {
394 SK_DSC(p, "EINVAL: mode %u", mode);
395 err = EINVAL;
396 goto done;
397 }
398
399 if (__improbable((ch->ch_flags & CHANF_USER_PACKET_POOL) == 0 &&
400 (flags & (CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE |
401 CHANNEL_SYNCF_ALLOC_BUF)) != 0)) {
402 SK_DSC(p, "EINVAL: !CHANF_USER_PACKET_POOL with "
403 "SYNCF_ALLOC/FREE");
404 err = EINVAL;
405 goto done;
406 }
407
408 if (__improbable(ch->ch_flags & CHANF_DEFUNCT)) {
409 SK_DSC(p, "channel is defunct");
410 err = ENXIO;
411 goto done;
412 }
413
414 /* clear CHANF_DEFUNCT_SKIP if it was set during defunct last time */
415 if (__improbable(ch->ch_flags & CHANF_DEFUNCT_SKIP)) {
416 os_atomic_andnot(&ch->ch_flags, CHANF_DEFUNCT_SKIP, relaxed);
417 }
418
419 na = ch->ch_na; /* we have a reference */
420 ASSERT(na != NULL);
421 ASSERT(NA_IS_ACTIVE(na));
422
423 if (__improbable(na_reject_channel(ch, na))) {
424 SK_DSC(p, "channel is non-permissive");
425 err = ENXIO;
426 goto done;
427 }
428
429 /* mark thread with sync-in-progress flag */
430 protect = sk_sync_protect();
431
432 /* update our work timestamp */
433 na->na_work_ts = _net_uptime;
434
435 /* and make this channel eligible for draining again */
436 if (na->na_flags & NAF_DRAINING) {
437 os_atomic_andnot(&na->na_flags, NAF_DRAINING, relaxed);
438 }
439
440 if (mode == CHANNEL_SYNC_UPP) {
441 upp_sync_flags = NA_SYNCF_FORCE_UPP_SYNC;
442 if (flags & CHANNEL_SYNCF_PURGE) {
443 upp_sync_flags |= NA_SYNCF_UPP_PURGE;
444 }
445 goto packet_pool_sync;
446 }
447
448 t = (mode == CHANNEL_SYNC_TX ? NR_TX : NR_RX);
449 krings = NAKR(na, t);
450 qfirst = ch->ch_first[t];
451 qlast = ch->ch_last[t];
452
453 for (i = qfirst; i < qlast; i++) {
454 kring = krings + i;
455 s = kr_enter(kring, TRUE);
456 ASSERT(s == 0);
457 #if SK_LOG
458 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
459 channel_sync_log1((mode == CHANNEL_SYNC_TX) ?
460 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
461 "txsync" : "rxsync", p, na, ch, kring, i);
462 }
463 #endif /* SK_LOG */
464
465 switch (mode) {
466 case CHANNEL_SYNC_TX: {
467 int error = 0;
468
469 DTRACE_SKYWALK2(pretxprologue, struct kern_channel *,
470 ch, struct __kern_channel_ring *, kring);
471
472 if (kr_txsync_prologue(ch, kring, p) >=
473 kring->ckr_num_slots) {
474 kr_log_bad_ring(kring);
475 error = EFAULT;
476 if (!err) {
477 SK_DSC(p, "EFAULT: "
478 "kr_txsync_prologue()");
479 err = EFAULT;
480 }
481 } else if (kring->ckr_na_sync(kring, p,
482 NA_SYNCF_FORCE_RECLAIM) == 0) {
483 kr_txsync_finalize(ch, kring, p);
484 } else {
485 error = EIO;
486 if (!err) {
487 SK_DSC(p, "EIO: TX "
488 "kring->ckr_na_sync()");
489 err = EIO;
490 }
491 }
492
493 DTRACE_SKYWALK3(posttxfinalize, struct kern_channel *,
494 ch, struct __kern_channel_ring *, kring, int,
495 error);
496 }
497 break;
498
499 case CHANNEL_SYNC_RX: {
500 int error = 0;
501
502 DTRACE_SKYWALK2(prerxprologue, struct kern_channel *,
503 ch, struct __kern_channel_ring *, kring);
504
505 if (kr_rxsync_prologue(ch, kring, p) >=
506 kring->ckr_num_slots) {
507 kr_log_bad_ring(kring);
508 error = EFAULT;
509 if (!err) {
510 SK_DSC(p, "EFAULT: "
511 "kr_rxsync_prologue()");
512 err = EFAULT;
513 }
514 } else if (kring->ckr_na_sync(kring, p,
515 NA_SYNCF_FORCE_READ) == 0) {
516 kr_rxsync_finalize(ch, kring, p);
517 } else {
518 error = EIO;
519 if (!err) {
520 SK_DSC(p, "EIO: " "RX "
521 "kring->ckr_na_sync()");
522 err = EIO;
523 }
524 }
525
526 DTRACE_SKYWALK3(postrxfinalize, struct kern_channel *,
527 ch, struct __kern_channel_ring *, kring, int,
528 error);
529 }
530 break;
531
532 default:
533 VERIFY(0);
534 /* NOTREACHED */
535 __builtin_unreachable();
536 }
537
538 #if SK_LOG
539 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
540 channel_sync_log2((mode == CHANNEL_SYNC_TX) ?
541 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
542 "txsync" : "rxsync", p, na, kring, i);
543 }
544 #endif /* SK_LOG */
545
546 kr_exit(kring);
547 }
548
549 packet_pool_sync:
550 if (flags & (CHANNEL_SYNCF_ALLOC |
551 CHANNEL_SYNCF_LARGE_ALLOC | CHANNEL_SYNCF_ALLOC_BUF)) {
552 enum txrx type;
553
554 if (flags & CHANNEL_SYNCF_LARGE_ALLOC) {
555 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC));
556 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC_BUF));
557 type = NR_LBA;
558 qfirst = ch->ch_first[type];
559 qlast = ch->ch_last[type];
560 } else {
561 type = NR_A;
562 qfirst = ch->ch_first[type];
563 qlast = ch->ch_last[type];
564 if (!(flags & CHANNEL_SYNCF_ALLOC)) {
565 qfirst++;
566 } else if ((qlast - qfirst) > 1 &&
567 !(flags & CHANNEL_SYNCF_ALLOC_BUF)) {
568 qlast--;
569 }
570 }
571 ASSERT(qfirst != qlast);
572 krings = NAKR(na, type);
573
574 for (i = qfirst; i < qlast; i++) {
575 kring = krings + i;
576 s = kr_enter(kring, TRUE);
577 ASSERT(s == 0);
578 #if SK_LOG
579 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
580 channel_sync_log1(0, "alloc-sync", p, na, ch,
581 kring, i);
582 }
583 #endif /* SK_LOG */
584 if (__improbable(kr_alloc_sync_prologue(kring, p) >=
585 kring->ckr_num_slots)) {
586 kr_log_bad_ring(kring);
587 if (!err) {
588 SK_DSC(p,
589 "EFAULT: kr_alloc_sync_prologue()");
590 err = EFAULT;
591 }
592 } else if (kring->ckr_na_sync(kring, p,
593 upp_sync_flags) == 0) {
594 kr_alloc_sync_finalize(kring, p);
595 } else {
596 if (!err) {
597 SK_DSC(p,
598 "EIO: ALLOC: ring->ckr_na_sync()");
599 err = EIO;
600 }
601 skywalk_kill_process(p,
602 SKYWALK_KILL_REASON_SYNC_FAILED |
603 SKYWALK_KILL_REASON_ALLOC_SYNC);
604 }
605 #if SK_LOG
606 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
607 channel_sync_log2(0, "alloc-sync", p, na,
608 kring, i);
609 }
610 #endif /* SK_LOG */
611 kr_exit(kring);
612 }
613 } /* CHANNEL_SYNCF_ALLOC */
614
615 if (flags & CHANNEL_SYNCF_FREE) {
616 qfirst = ch->ch_first[NR_F];
617 qlast = ch->ch_last[NR_F];
618 ASSERT(qfirst != qlast);
619 krings = NAKR(na, NR_F);
620
621 for (i = qfirst; i < qlast; i++) {
622 kring = krings + i;
623 s = kr_enter(kring, TRUE);
624 ASSERT(s == 0);
625 #if SK_LOG
626 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
627 channel_sync_log1(0, "free-sync", p, na, ch,
628 kring, i);
629 }
630 #endif /* SK_LOG */
631 if (__improbable(kr_free_sync_prologue(kring, p) >=
632 kring->ckr_num_slots)) {
633 kr_log_bad_ring(kring);
634 if (!err) {
635 SK_DSC(p,
636 "EFAULT: kr_free_sync_prologue()");
637 err = EFAULT;
638 }
639 } else if (kring->ckr_na_sync(kring, p,
640 upp_sync_flags) == 0) {
641 kr_free_sync_finalize(kring, p);
642 } else {
643 if (!err) {
644 SK_DSC(p,
645 "EIO: FREE: ring->ckr_na_sync()");
646 err = EIO;
647 }
648 skywalk_kill_process(p,
649 SKYWALK_KILL_REASON_SYNC_FAILED |
650 SKYWALK_KILL_REASON_FREE_SYNC);
651 }
652 #if SK_LOG
653 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
654 channel_sync_log2(0, "free-sync", p, na,
655 kring, i);
656 }
657 #endif /* SK_LOG */
658 kr_exit(kring);
659 }
660 } /* CHANNEL_SYNCF_FREE */
661
662 /* unmark thread with sync-in-progress flag */
663 sk_sync_unprotect(protect);
664
665 done:
666 lck_mtx_unlock(&ch->ch_lock);
667 fp_drop(p, uap->c, fp, 0);
668
669 VERIFY(!sk_is_sync_protected());
670
671 return err;
672 }
673
674 int
__channel_get_opt(struct proc * p,struct __channel_get_opt_args * uap,int * retval)675 __channel_get_opt(struct proc *p, struct __channel_get_opt_args *uap,
676 int *retval)
677 {
678 #pragma unused(retval)
679 struct fileproc *fp;
680 struct kern_channel *ch = NULL;
681 struct sockopt sopt;
682 uint32_t optlen;
683 int err = 0;
684
685 AUDIT_ARG(fd, uap->c);
686
687 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
688 if (err != 0) {
689 SK_DSC(p, "fp_get_ftype err %u", err);
690 return err;
691 }
692 ch = fp_get_data(fp);
693
694 if (uap->aoptlen == USER_ADDR_NULL) {
695 SK_DSC(p, "EINVAL: uap->aoptlen == USER_ADDR_NULL");
696 err = EINVAL;
697 goto done;
698 }
699
700 if (uap->aoptval != USER_ADDR_NULL) {
701 err = copyin(uap->aoptlen, &optlen, sizeof(optlen));
702 if (err != 0) {
703 SK_DSC(p, "copyin err %u: aoptlen 0x%llx", err,
704 SK_KVA(uap->aoptlen));
705 goto done;
706 }
707 } else {
708 optlen = 0;
709 }
710
711 bzero(&sopt, sizeof(sopt));
712 sopt.sopt_dir = SOPT_GET;
713 sopt.sopt_name = uap->opt;
714 sopt.sopt_val = uap->aoptval;
715 sopt.sopt_valsize = optlen;
716 sopt.sopt_p = p;
717
718 lck_mtx_lock(&ch->ch_lock);
719 err = ch_get_opt(ch, &sopt);
720 lck_mtx_unlock(&ch->ch_lock);
721 if (err == 0) {
722 optlen = (uint32_t)sopt.sopt_valsize;
723 err = copyout(&optlen, uap->aoptlen, sizeof(optlen));
724 #if SK_LOG
725 if (err != 0) {
726 SK_DSC(p, "copyout err %u: aoptlen 0x%llx", err,
727 SK_KVA(uap->aoptlen));
728 }
729 #endif
730 }
731
732 done:
733 fp_drop(p, uap->c, fp, 0);
734
735 return err;
736 }
737
738 int
__channel_set_opt(struct proc * p,struct __channel_set_opt_args * uap,int * retval)739 __channel_set_opt(struct proc *p, struct __channel_set_opt_args *uap,
740 int *retval)
741 {
742 #pragma unused(retval)
743 struct fileproc *fp;
744 struct kern_channel *ch = NULL;
745 struct sockopt sopt;
746 int err = 0;
747
748 AUDIT_ARG(fd, uap->c);
749
750 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
751 if (err != 0) {
752 SK_DSC(p, "fp_get_ftype err %u", err);
753 return err;
754 }
755 ch = fp_get_data(fp);
756
757 bzero(&sopt, sizeof(sopt));
758 sopt.sopt_dir = SOPT_SET;
759 sopt.sopt_name = uap->opt;
760 sopt.sopt_val = uap->aoptval;
761 sopt.sopt_valsize = uap->optlen;
762 sopt.sopt_p = p;
763
764 lck_mtx_lock(&ch->ch_lock);
765 if (__improbable(ch->ch_flags & (CHANF_CLOSING | CHANF_DEFUNCT))) {
766 SK_DSC(p, "channel is closing/defunct");
767 err = ENXIO;
768 } else if (__improbable(ch->ch_na == NULL ||
769 !NA_IS_ACTIVE(ch->ch_na) || na_reject_channel(ch, ch->ch_na))) {
770 SK_DSC(p, "channel is non-permissive");
771 err = ENXIO;
772 } else {
773 err = ch_set_opt(ch, &sopt);
774 }
775 lck_mtx_unlock(&ch->ch_lock);
776
777 fp_drop(p, uap->c, fp, 0);
778
779 #if SK_LOG
780 if (err != 0) {
781 SK_DSC(p, "ch_set_opt() err %u", err);
782 }
783 #endif
784
785 return err;
786 }
787