1 /*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/param.h>
30 #include <sys/sdt.h>
31 #include <sys/systm.h>
32 #include <sys/filedesc.h>
33 #include <sys/proc_internal.h>
34 #include <sys/file_internal.h>
35 #include <sys/vnode_internal.h>
36 #include <sys/sysproto.h>
37 #include <security/audit/audit.h>
38 #include <skywalk/os_skywalk_private.h>
39
40 static int chop_select(struct fileproc *, int, void *, vfs_context_t);
41 static int chop_close(struct fileglob *, vfs_context_t);
42 static int chop_kqfilter(struct fileproc *, struct knote *, struct kevent_qos_s *);
43
44 static const struct fileops channel_ops = {
45 .fo_type = DTYPE_CHANNEL,
46 .fo_read = fo_no_read,
47 .fo_write = fo_no_write,
48 .fo_ioctl = fo_no_ioctl,
49 .fo_select = chop_select,
50 .fo_close = chop_close,
51 .fo_drain = fo_no_drain,
52 .fo_kqfilter = chop_kqfilter,
53 };
54
55 #if (DEVELOPMENT || DEBUG)
56 static uint32_t ch_force_defunct = 0;
57 SYSCTL_UINT(_kern_skywalk_channel, OID_AUTO, force_defunct,
58 CTLFLAG_RW | CTLFLAG_LOCKED, &ch_force_defunct, 0, "");
59 #endif /* !DEVELOPMENT && !DEBUG */
60
61 /*
62 * -fbounds-safety: Utility macro for the ci_key field of struct ch_init, which
63 * is of type user_addr_t (non-pointer type), which can't be marked as __sized_by.
64 */
65 #define USER_ADDR_TO_PTR(chan_init) \
66 __unsafe_forge_bidi_indexable(void *, (chan_init).ci_key, (chan_init).ci_key_len)
67
68 #if SK_LOG
69 const char *__null_terminated MODE_TAG_TXSYNC = "txsync";
70 const char *__null_terminated MODE_TAG_RXSYNC = "rxsync";
71 #endif /* SK_LOG */
72
73 static int
chop_select(struct fileproc * fp,int which,void * wql,vfs_context_t ctx)74 chop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
75 {
76 int revents = 0, events = 0;
77 struct kern_channel *ch;
78
79 ch = (struct kern_channel *)fp_get_data(fp);
80 if (ch == NULL) {
81 return 0;
82 }
83
84 switch (which) {
85 case FREAD:
86 events = POLLIN;
87 break;
88
89 case FWRITE:
90 events = POLLOUT;
91 break;
92
93 case 0:
94 events = POLLERR;
95 break;
96
97 default:
98 return 1;
99 }
100
101 /* caller will lock channel */
102 revents = ch_select(ch, events, wql, vfs_context_proc(ctx));
103 return (events & revents) ? 1 : 0;
104 }
105
106 static int
chop_close(struct fileglob * fg,vfs_context_t ctx)107 chop_close(struct fileglob *fg, vfs_context_t ctx)
108 {
109 #pragma unused(ctx)
110 struct kern_channel *ch;
111
112 ch = (struct kern_channel *)fg_get_data(fg);
113 fg_set_data(fg, NULL);
114 if (ch != NULL) {
115 ch_dtor(ch);
116 }
117
118 return 0;
119 }
120
121 /*
122 * This is called when a thread invokes kevent() to record
123 * a change in the configuration of the kqueue().
124 */
125 static int
chop_kqfilter(struct fileproc * fp,struct knote * kn,struct kevent_qos_s * kev)126 chop_kqfilter(struct fileproc *fp, struct knote *kn, struct kevent_qos_s *kev)
127 {
128 struct kern_channel *ch = (struct kern_channel *)fp_get_data(fp);
129
130 /* caller will lock channel */
131 return ch_kqfilter(ch, kn, kev);
132 }
133
134 int
__channel_open(struct proc * p,struct __channel_open_args * uap,int * retval)135 __channel_open(struct proc *p, struct __channel_open_args *uap, int *retval)
136 {
137 struct fileproc *__single fp = NULL;
138 struct kern_channel *ch = NULL;
139 struct ch_init init;
140 int fd = -1, err = 0;
141 void *key = NULL;
142 uint32_t keylen;
143 guardid_t guard;
144
145 if (__improbable(uap->init == USER_ADDR_NULL ||
146 uap->init_len < sizeof(init))) {
147 SK_DSC(p, "EINVAL: init 0x%llx, init_len %u", SK_KVA(uap->init),
148 uap->init_len);
149 err = EINVAL;
150 goto done;
151 }
152
153 err = copyin(uap->init, (caddr_t)&init, sizeof(init));
154 if (__improbable(err != 0)) {
155 SK_DSC(p, "copyin err %u: init 0x%llx", err, SK_KVA(uap->init));
156 goto done;
157 }
158
159 if (__improbable(init.ci_version != CHANNEL_INIT_CURRENT_VERSION)) {
160 SK_DSC(p, "ENOTSUP: init.ci_version %u != %u", init.ci_version,
161 CHANNEL_INIT_CURRENT_VERSION);
162 err = ENOTSUP;
163 goto done;
164 } else if (__improbable(uuid_is_null(init.ci_nx_uuid))) {
165 SK_DSC(p, "EINVAL: uuid_is_null");
166 err = EINVAL;
167 goto done;
168 } else if (__improbable((init.ci_key_len != 0 &&
169 init.ci_key == USER_ADDR_NULL) ||
170 (init.ci_key_len == 0 && init.ci_key != USER_ADDR_NULL))) {
171 SK_DSC(p, "EINVAL: ci_key_len %i, ci_key 0x%llx",
172 init.ci_key_len, SK_KVA(init.ci_key));
173 err = EINVAL;
174 goto done;
175 }
176
177 if ((init.ci_ch_mode & CHMODE_MONITOR) != 0) {
178 if (__improbable((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) != 0)) {
179 SK_DSC(p, "EINVAL: PACKET_POOL not supported for MONITOR mode");
180 err = EINVAL;
181 goto done;
182 }
183 if (__improbable((init.ci_ch_mode & CHMODE_EVENT_RING) != 0)) {
184 SK_DSC(p, "EINVAL: EVENT ring not supported for MONITOR mode");
185 err = EINVAL;
186 goto done;
187 }
188 if (__improbable((init.ci_ch_mode & CHMODE_LOW_LATENCY) != 0)) {
189 SK_DSC(p, "EINVAL: low latency not supported for MONITOR mode");
190 err = EINVAL;
191 goto done;
192 }
193 }
194
195 if ((init.ci_ch_mode & CHMODE_EVENT_RING) != 0) {
196 if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) == 0) {
197 SK_DSC(p, "EINVAL: PACKET_POOL is required for EVENT ring");
198 err = EINVAL;
199 goto done;
200 }
201 }
202
203 #if (DEVELOPMENT || DEBUG)
204 if (__improbable(ch_force_defunct)) {
205 init.ci_ch_mode |= CHMODE_DEFUNCT_OK;
206 }
207 #endif /* !DEVELOPMENT && !DEBUG */
208
209 /* generate guard ID based on nexus instance UUID */
210 sk_gen_guard_id(TRUE, init.ci_nx_uuid, &guard);
211
212 err = falloc_guarded(p, &fp, &fd, vfs_context_current(), &guard,
213 GUARD_CLOSE | GUARD_DUP | GUARD_SOCKET_IPC | GUARD_FILEPORT | GUARD_WRITE);
214 if (__improbable(err != 0)) {
215 SK_DSC(p, "falloc_guarded: %u", err);
216 goto done;
217 }
218
219 keylen = init.ci_key_len;
220 if (keylen != 0) {
221 if (__improbable(keylen > NEXUS_MAX_KEY_LEN)) {
222 SK_DSC(p, "EINVAL: ci_key_len %u", keylen);
223 err = EINVAL;
224 goto done;
225 }
226
227 key = sk_alloc_data(keylen, Z_WAITOK, skmem_tag_ch_key);
228 if (__improbable(key == NULL)) {
229 SK_DSC(p, "ENOMEM: ci_key_len %u", keylen);
230 err = ENOMEM;
231 goto done;
232 }
233
234 err = copyin(init.ci_key, (caddr_t)key, keylen);
235 if (__improbable(err != 0)) {
236 SK_DSC(p, "copyin err %u: ci_key 0x%llx, ci_key_len %u",
237 err, SK_KVA(init.ci_key), keylen);
238 goto done;
239 }
240 }
241
242 /* let ch_open() take over this key upon success */
243 init.ci_key = (user_addr_t)key;
244 key = NULL;
245
246 if (__improbable((ch = ch_open(&init, p, fd, &err)) == NULL)) {
247 /* in case not processed */
248 key = USER_ADDR_TO_PTR(init);
249 ASSERT(err != 0);
250 SK_DSC(p, "ch_open nx_port %d err %u",
251 (int)init.ci_nx_port, err);
252 goto done;
253 }
254 /* in case not processed */
255 key = USER_ADDR_TO_PTR(init);
256
257 /* update userland with respect to guard ID, etc. */
258 init.ci_guard = guard;
259 init.ci_key = USER_ADDR_NULL;
260 err = copyout(&init, uap->init, sizeof(init));
261 if (__improbable(err != 0)) {
262 SK_DSC(p, "copyout err %u: init 0x%llx", err,
263 SK_KVA(uap->init));
264 goto done;
265 }
266
267 fp->fp_flags |= FP_CLOEXEC | FP_CLOFORK;
268 fp->fp_glob->fg_flag &= ~(FREAD | FWRITE);
269 fp->fp_glob->fg_ops = &channel_ops;
270 fp_set_data(fp, ch); /* ref from ch_open */
271
272 proc_fdlock(p);
273 procfdtbl_releasefd(p, fd, NULL);
274 fp_drop(p, fd, fp, 1);
275 proc_fdunlock(p);
276
277 *retval = fd;
278
279 SK_D("%s(%d) nx_port %d fd %d guard 0x%llx",
280 sk_proc_name_address(p), sk_proc_pid(p), (int)init.ci_nx_port,
281 fd, guard);
282
283 done:
284 if (key != NULL) {
285 sk_free_data(key, keylen);
286 key = NULL;
287 }
288 if (__improbable(err != 0)) {
289 if (ch != NULL) {
290 ch_dtor(ch);
291 ch = NULL;
292 }
293 if (fp != NULL) {
294 fp_free(p, fd, fp);
295 fp = NULL;
296 }
297 }
298
299 return err;
300 }
301
302 int
__channel_get_info(struct proc * p,struct __channel_get_info_args * uap,int * retval)303 __channel_get_info(struct proc *p, struct __channel_get_info_args *uap,
304 int *retval)
305 {
306 #pragma unused(retval)
307 struct fileproc *__single fp;
308 struct kern_channel *ch = NULL;
309 int err = 0;
310
311 AUDIT_ARG(fd, uap->c);
312
313 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
314 if (__improbable(err != 0)) {
315 SK_DSC(p, "fp_get_ftype err %u", err);
316 return err;
317 }
318 ch = (struct kern_channel *__single)fp_get_data(fp);
319
320 if (__improbable(uap->cinfo == USER_ADDR_NULL ||
321 uap->cinfolen < sizeof(struct ch_info))) {
322 SK_DSC(p, "EINVAL: cinfo 0x%llx, cinfolen %u",
323 SK_KVA(uap->cinfo), uap->cinfolen);
324 err = EINVAL;
325 goto done;
326 }
327
328 lck_mtx_lock(&ch->ch_lock);
329 err = copyout(ch->ch_info, uap->cinfo, sizeof(struct ch_info));
330 lck_mtx_unlock(&ch->ch_lock);
331 if (__improbable(err != 0)) {
332 SK_DSC(p, "copyout err %u: cinfo 0x%llx", err,
333 SK_KVA(uap->cinfo));
334 goto done;
335 }
336
337 done:
338 fp_drop(p, uap->c, fp, 0);
339
340 return err;
341 }
342
343 #if SK_LOG
344 /* Hoisted out of line to reduce kernel stack footprint */
345 SK_LOG_ATTRIBUTE
346 static void
channel_sync_log1(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct kern_channel * ch,const struct __kern_channel_ring * kring,ring_id_t i)347 channel_sync_log1(uint64_t verb, const char *sync, struct proc *p,
348 const struct nexus_adapter *na, const struct kern_channel *ch,
349 const struct __kern_channel_ring *kring, ring_id_t i)
350 {
351 verb |= SK_VERB_SYNC;
352 SK_DF(verb, "%s(%d) pre: %s ring %u na \"%s\" (0x%llx) ch 0x%llx "
353 "th 0x%llx h %u kh %u", sk_proc_name_address(p), sk_proc_pid(p),
354 sync, i, na->na_name, SK_KVA(na), SK_KVA(ch),
355 SK_KVA(current_thread()), kring->ckr_ring->ring_head,
356 kring->ckr_khead);
357 }
358
359 SK_LOG_ATTRIBUTE
360 static void
channel_sync_log2(uint64_t verb,const char * sync,struct proc * p,const struct nexus_adapter * na,const struct __kern_channel_ring * kring,ring_id_t i)361 channel_sync_log2(uint64_t verb, const char *sync, struct proc *p,
362 const struct nexus_adapter *na, const struct __kern_channel_ring *kring,
363 ring_id_t i)
364 {
365 verb |= SK_VERB_SYNC;
366 SK_DF(verb, "%s(%d) post: %s ring %u na \"%s\" h %u kh %u",
367 sk_proc_name_address(p), sk_proc_pid(p), sync, i, na->na_name,
368 kring->ckr_ring->ring_head, kring->ckr_khead);
369 }
370 #endif /* SK_LOG */
371
372 int
__channel_sync(struct proc * p,struct __channel_sync_args * uap,int * retval)373 __channel_sync(struct proc *p, struct __channel_sync_args *uap, int *retval)
374 {
375 #pragma unused(retval)
376 struct fileproc *__single fp;
377 sk_protect_t protect = NULL;
378 struct nexus_adapter *na;
379 struct __kern_channel_ring *krings, *kring;
380 struct kern_channel *ch;
381 sync_mode_t mode;
382 ring_id_t i, qfirst, qlast;
383 sync_flags_t flags, upp_sync_flags = 0;
384 enum txrx t;
385 int err;
386 int s;
387
388 net_update_uptime();
389
390 AUDIT_ARG(fd, uap->c);
391
392 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
393 if (__improbable(err != 0)) {
394 SK_DSC(p, "fp_get_ftype err %u", err);
395 return err;
396 }
397 ch = (struct kern_channel *__single)fp_get_data(fp);
398
399 lck_mtx_lock(&ch->ch_lock);
400 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
401
402 mode = uap->mode;
403 flags = uap->flags;
404 if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX &&
405 mode != CHANNEL_SYNC_UPP)) {
406 SK_DSC(p, "EINVAL: mode %u", mode);
407 err = EINVAL;
408 goto done;
409 }
410
411 if (__improbable((ch->ch_flags & CHANF_USER_PACKET_POOL) == 0 &&
412 (flags & (CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE |
413 CHANNEL_SYNCF_ALLOC_BUF)) != 0)) {
414 SK_DSC(p, "EINVAL: !CHANF_USER_PACKET_POOL with "
415 "SYNCF_ALLOC/FREE");
416 err = EINVAL;
417 goto done;
418 }
419
420 if (__improbable(ch->ch_flags & CHANF_DEFUNCT)) {
421 SK_DSC(p, "channel is defunct");
422 err = ENXIO;
423 goto done;
424 }
425
426 /* clear CHANF_DEFUNCT_SKIP if it was set during defunct last time */
427 if (__improbable(ch->ch_flags & CHANF_DEFUNCT_SKIP)) {
428 os_atomic_andnot(&ch->ch_flags, CHANF_DEFUNCT_SKIP, relaxed);
429 }
430
431 na = ch->ch_na; /* we have a reference */
432 ASSERT(na != NULL);
433 ASSERT(NA_IS_ACTIVE(na));
434
435 if (__improbable(na_reject_channel(ch, na))) {
436 SK_DSC(p, "channel is non-permissive");
437 err = ENXIO;
438 goto done;
439 }
440
441 /* mark thread with sync-in-progress flag */
442 protect = sk_sync_protect();
443
444 /* update our work timestamp */
445 na->na_work_ts = _net_uptime;
446
447 /* and make this channel eligible for draining again */
448 if (na->na_flags & NAF_DRAINING) {
449 os_atomic_andnot(&na->na_flags, NAF_DRAINING, relaxed);
450 }
451
452 if (mode == CHANNEL_SYNC_UPP) {
453 upp_sync_flags = NA_SYNCF_FORCE_UPP_SYNC;
454 if (flags & CHANNEL_SYNCF_PURGE) {
455 upp_sync_flags |= NA_SYNCF_UPP_PURGE;
456 }
457 goto packet_pool_sync;
458 }
459
460 t = (mode == CHANNEL_SYNC_TX ? NR_TX : NR_RX);
461 krings = NAKR(na, t);
462 qfirst = ch->ch_first[t];
463 qlast = ch->ch_last[t];
464
465 for (i = qfirst; i < qlast; i++) {
466 kring = krings + i;
467 s = kr_enter(kring, TRUE);
468 ASSERT(s == 0);
469 #if SK_LOG
470 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
471 channel_sync_log1((mode == CHANNEL_SYNC_TX) ?
472 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
473 MODE_TAG_TXSYNC : MODE_TAG_RXSYNC, p, na, ch, kring, i);
474 }
475 #endif /* SK_LOG */
476
477 switch (mode) {
478 case CHANNEL_SYNC_TX: {
479 int error = 0;
480
481 DTRACE_SKYWALK2(pretxprologue, struct kern_channel *,
482 ch, struct __kern_channel_ring *, kring);
483
484 if (kr_txsync_prologue(ch, kring, p) >=
485 kring->ckr_num_slots) {
486 kr_log_bad_ring(kring);
487 error = EFAULT;
488 if (!err) {
489 SK_DSC(p, "EFAULT: "
490 "kr_txsync_prologue()");
491 err = EFAULT;
492 }
493 } else if (kring->ckr_na_sync(kring, p,
494 NA_SYNCF_FORCE_RECLAIM) == 0) {
495 kr_txsync_finalize(ch, kring, p);
496 } else {
497 error = EIO;
498 if (!err) {
499 SK_DSC(p, "EIO: TX "
500 "kring->ckr_na_sync()");
501 err = EIO;
502 }
503 }
504
505 DTRACE_SKYWALK3(posttxfinalize, struct kern_channel *,
506 ch, struct __kern_channel_ring *, kring, int,
507 error);
508 }
509 break;
510
511 case CHANNEL_SYNC_RX: {
512 int error = 0;
513
514 DTRACE_SKYWALK2(prerxprologue, struct kern_channel *,
515 ch, struct __kern_channel_ring *, kring);
516
517 if (kr_rxsync_prologue(ch, kring, p) >=
518 kring->ckr_num_slots) {
519 kr_log_bad_ring(kring);
520 error = EFAULT;
521 if (!err) {
522 SK_DSC(p, "EFAULT: "
523 "kr_rxsync_prologue()");
524 err = EFAULT;
525 }
526 } else if (kring->ckr_na_sync(kring, p,
527 NA_SYNCF_FORCE_READ) == 0) {
528 kr_rxsync_finalize(ch, kring, p);
529 } else {
530 error = EIO;
531 if (!err) {
532 SK_DSC(p, "EIO: " "RX "
533 "kring->ckr_na_sync()");
534 err = EIO;
535 }
536 }
537
538 DTRACE_SKYWALK3(postrxfinalize, struct kern_channel *,
539 ch, struct __kern_channel_ring *, kring, int,
540 error);
541 }
542 break;
543
544 default:
545 VERIFY(0);
546 /* NOTREACHED */
547 __builtin_unreachable();
548 }
549
550 #if SK_LOG
551 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
552 channel_sync_log2((mode == CHANNEL_SYNC_TX) ?
553 SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ?
554 MODE_TAG_TXSYNC : MODE_TAG_RXSYNC, p, na, kring, i);
555 }
556 #endif /* SK_LOG */
557
558 kr_exit(kring);
559 }
560
561 packet_pool_sync:
562 if (flags & (CHANNEL_SYNCF_ALLOC |
563 CHANNEL_SYNCF_LARGE_ALLOC | CHANNEL_SYNCF_ALLOC_BUF)) {
564 enum txrx type;
565
566 if (flags & CHANNEL_SYNCF_LARGE_ALLOC) {
567 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC));
568 ASSERT(!(flags & CHANNEL_SYNCF_ALLOC_BUF));
569 type = NR_LBA;
570 qfirst = ch->ch_first[type];
571 qlast = ch->ch_last[type];
572 } else {
573 type = NR_A;
574 qfirst = ch->ch_first[type];
575 qlast = ch->ch_last[type];
576 if (!(flags & CHANNEL_SYNCF_ALLOC)) {
577 qfirst++;
578 } else if ((qlast - qfirst) > 1 &&
579 !(flags & CHANNEL_SYNCF_ALLOC_BUF)) {
580 qlast--;
581 }
582 }
583 ASSERT(qfirst != qlast);
584 krings = NAKR(na, type);
585
586 for (i = qfirst; i < qlast; i++) {
587 kring = krings + i;
588 s = kr_enter(kring, TRUE);
589 ASSERT(s == 0);
590 #if SK_LOG
591 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
592 channel_sync_log1(0, "alloc-sync", p, na, ch,
593 kring, i);
594 }
595 #endif /* SK_LOG */
596 if (__improbable(kr_alloc_sync_prologue(kring, p) >=
597 kring->ckr_num_slots)) {
598 kr_log_bad_ring(kring);
599 if (!err) {
600 SK_DSC(p,
601 "EFAULT: kr_alloc_sync_prologue()");
602 err = EFAULT;
603 }
604 } else if (kring->ckr_na_sync(kring, p,
605 upp_sync_flags) == 0) {
606 kr_alloc_sync_finalize(kring, p);
607 } else {
608 if (!err) {
609 SK_DSC(p,
610 "EIO: ALLOC: ring->ckr_na_sync()");
611 err = EIO;
612 }
613 skywalk_kill_process(p,
614 SKYWALK_KILL_REASON_SYNC_FAILED |
615 SKYWALK_KILL_REASON_ALLOC_SYNC);
616 }
617 #if SK_LOG
618 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
619 channel_sync_log2(0, "alloc-sync", p, na,
620 kring, i);
621 }
622 #endif /* SK_LOG */
623 kr_exit(kring);
624 }
625 } /* CHANNEL_SYNCF_ALLOC */
626
627 if (flags & CHANNEL_SYNCF_FREE) {
628 qfirst = ch->ch_first[NR_F];
629 qlast = ch->ch_last[NR_F];
630 ASSERT(qfirst != qlast);
631 krings = NAKR(na, NR_F);
632
633 for (i = qfirst; i < qlast; i++) {
634 kring = krings + i;
635 s = kr_enter(kring, TRUE);
636 ASSERT(s == 0);
637 #if SK_LOG
638 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
639 channel_sync_log1(0, "free-sync", p, na, ch,
640 kring, i);
641 }
642 #endif /* SK_LOG */
643 if (__improbable(kr_free_sync_prologue(kring, p) >=
644 kring->ckr_num_slots)) {
645 kr_log_bad_ring(kring);
646 if (!err) {
647 SK_DSC(p,
648 "EFAULT: kr_free_sync_prologue()");
649 err = EFAULT;
650 }
651 } else if (kring->ckr_na_sync(kring, p,
652 upp_sync_flags) == 0) {
653 kr_free_sync_finalize(kring, p);
654 } else {
655 if (!err) {
656 SK_DSC(p,
657 "EIO: FREE: ring->ckr_na_sync()");
658 err = EIO;
659 }
660 skywalk_kill_process(p,
661 SKYWALK_KILL_REASON_SYNC_FAILED |
662 SKYWALK_KILL_REASON_FREE_SYNC);
663 }
664 #if SK_LOG
665 if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) {
666 channel_sync_log2(0, "free-sync", p, na,
667 kring, i);
668 }
669 #endif /* SK_LOG */
670 kr_exit(kring);
671 }
672 } /* CHANNEL_SYNCF_FREE */
673
674 /* unmark thread with sync-in-progress flag */
675 sk_sync_unprotect(protect);
676
677 done:
678 lck_mtx_unlock(&ch->ch_lock);
679 fp_drop(p, uap->c, fp, 0);
680
681 VERIFY(!sk_is_sync_protected());
682
683 return err;
684 }
685
686 int
__channel_get_opt(struct proc * p,struct __channel_get_opt_args * uap,int * retval)687 __channel_get_opt(struct proc *p, struct __channel_get_opt_args *uap,
688 int *retval)
689 {
690 #pragma unused(retval)
691 struct fileproc *__single fp;
692 struct kern_channel *ch = NULL;
693 struct sockopt sopt;
694 uint32_t optlen;
695 int err = 0;
696
697 AUDIT_ARG(fd, uap->c);
698
699 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
700 if (err != 0) {
701 SK_DSC(p, "fp_get_ftype err %u", err);
702 return err;
703 }
704 ch = (struct kern_channel *__single)fp_get_data(fp);
705
706 if (uap->aoptlen == USER_ADDR_NULL) {
707 SK_DSC(p, "EINVAL: uap->aoptlen == USER_ADDR_NULL");
708 err = EINVAL;
709 goto done;
710 }
711
712 if (uap->aoptval != USER_ADDR_NULL) {
713 err = copyin(uap->aoptlen, &optlen, sizeof(optlen));
714 if (err != 0) {
715 SK_DSC(p, "copyin err %u: aoptlen 0x%llx", err,
716 SK_KVA(uap->aoptlen));
717 goto done;
718 }
719 } else {
720 optlen = 0;
721 }
722
723 bzero(&sopt, sizeof(sopt));
724 sopt.sopt_dir = SOPT_GET;
725 sopt.sopt_name = uap->opt;
726 sopt.sopt_val = uap->aoptval;
727 sopt.sopt_valsize = optlen;
728 sopt.sopt_p = p;
729
730 lck_mtx_lock(&ch->ch_lock);
731 err = ch_get_opt(ch, &sopt);
732 lck_mtx_unlock(&ch->ch_lock);
733 if (err == 0) {
734 optlen = (uint32_t)sopt.sopt_valsize;
735 err = copyout(&optlen, uap->aoptlen, sizeof(optlen));
736 #if SK_LOG
737 if (err != 0) {
738 SK_DSC(p, "copyout err %u: aoptlen 0x%llx", err,
739 SK_KVA(uap->aoptlen));
740 }
741 #endif
742 }
743
744 done:
745 fp_drop(p, uap->c, fp, 0);
746
747 return err;
748 }
749
750 int
__channel_set_opt(struct proc * p,struct __channel_set_opt_args * uap,int * retval)751 __channel_set_opt(struct proc *p, struct __channel_set_opt_args *uap,
752 int *retval)
753 {
754 #pragma unused(retval)
755 struct fileproc *__single fp;
756 struct kern_channel *ch = NULL;
757 struct sockopt sopt;
758 int err = 0;
759
760 AUDIT_ARG(fd, uap->c);
761
762 err = fp_get_ftype(p, uap->c, DTYPE_CHANNEL, ENODEV, &fp);
763 if (err != 0) {
764 SK_DSC(p, "fp_get_ftype err %u", err);
765 return err;
766 }
767 ch = (struct kern_channel *__single)fp_get_data(fp);
768
769 bzero(&sopt, sizeof(sopt));
770 sopt.sopt_dir = SOPT_SET;
771 sopt.sopt_name = uap->opt;
772 sopt.sopt_val = uap->aoptval;
773 sopt.sopt_valsize = uap->optlen;
774 sopt.sopt_p = p;
775
776 lck_mtx_lock(&ch->ch_lock);
777 if (__improbable(ch->ch_flags & (CHANF_CLOSING | CHANF_DEFUNCT))) {
778 SK_DSC(p, "channel is closing/defunct");
779 err = ENXIO;
780 } else if (__improbable(ch->ch_na == NULL ||
781 !NA_IS_ACTIVE(ch->ch_na) || na_reject_channel(ch, ch->ch_na))) {
782 SK_DSC(p, "channel is non-permissive");
783 err = ENXIO;
784 } else {
785 err = ch_set_opt(ch, &sopt);
786 }
787 lck_mtx_unlock(&ch->ch_lock);
788
789 fp_drop(p, uap->c, fp, 0);
790
791 #if SK_LOG
792 if (err != 0) {
793 SK_DSC(p, "ch_set_opt() err %u", err);
794 }
795 #endif
796
797 return err;
798 }
799