1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1982, 1986, 1989, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95
67 */
68 /*
69 * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
70 * support for mandatory and extensible security protections. This notice
71 * is included in support of clause 2.2 (b) of the Apple Public License,
72 * Version 2.0.
73 */
74
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/filedesc.h>
78 #include <sys/ioctl.h>
79 #include <sys/file_internal.h>
80 #include <sys/proc_internal.h>
81 #include <sys/socketvar.h>
82 #include <sys/uio_internal.h>
83 #include <sys/kernel.h>
84 #include <sys/guarded.h>
85 #include <sys/stat.h>
86 #include <sys/malloc.h>
87 #include <sys/sysproto.h>
88
89 #include <sys/mount_internal.h>
90 #include <sys/protosw.h>
91 #include <sys/ev.h>
92 #include <sys/user.h>
93 #include <sys/kdebug.h>
94 #include <sys/poll.h>
95 #include <sys/event.h>
96 #include <sys/eventvar.h>
97 #include <sys/proc.h>
98 #include <sys/kauth.h>
99
100 #include <machine/smp.h>
101 #include <mach/mach_types.h>
102 #include <kern/kern_types.h>
103 #include <kern/assert.h>
104 #include <kern/kalloc.h>
105 #include <kern/thread.h>
106 #include <kern/clock.h>
107 #include <kern/ledger.h>
108 #include <kern/monotonic.h>
109 #include <kern/task.h>
110 #include <kern/telemetry.h>
111 #include <kern/waitq.h>
112 #include <kern/sched_hygiene.h>
113 #include <kern/sched_prim.h>
114 #include <kern/mpsc_queue.h>
115 #include <kern/debug.h>
116
117 #include <sys/mbuf.h>
118 #include <sys/domain.h>
119 #include <sys/socket.h>
120 #include <sys/socketvar.h>
121 #include <sys/errno.h>
122 #include <sys/syscall.h>
123 #include <sys/pipe.h>
124
125 #include <security/audit/audit.h>
126
127 #include <net/if.h>
128 #include <net/route.h>
129
130 #include <netinet/in.h>
131 #include <netinet/in_systm.h>
132 #include <netinet/ip.h>
133 #include <netinet/in_pcb.h>
134 #include <netinet/ip_var.h>
135 #include <netinet/ip6.h>
136 #include <netinet/tcp.h>
137 #include <netinet/tcp_fsm.h>
138 #include <netinet/tcp_seq.h>
139 #include <netinet/tcp_timer.h>
140 #include <netinet/tcp_var.h>
141 #include <netinet/tcpip.h>
142 #include <netinet/tcp_debug.h>
143 /* for wait queue based select */
144 #include <kern/waitq.h>
145 #include <sys/vnode_internal.h>
146 /* for remote time api*/
147 #include <kern/remote_time.h>
148 #include <os/log.h>
149 #include <sys/log_data.h>
150
151 #include <machine/monotonic.h>
152
153 #if CONFIG_MACF
154 #include <security/mac_framework.h>
155 #endif
156
157 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
158 #include <mach_debug/mach_debug_types.h>
159 #endif
160
161 /* for entitlement check */
162 #include <IOKit/IOBSD.h>
163
164 /* XXX should be in a header file somewhere */
165 extern kern_return_t IOBSDGetPlatformUUID(__darwin_uuid_t uuid, mach_timespec_t timeoutp);
166
167 int do_uiowrite(struct proc *p, struct fileproc *fp, uio_t uio, int flags, user_ssize_t *retval);
168 __private_extern__ int dofileread(vfs_context_t ctx, struct fileproc *fp,
169 user_addr_t bufp, user_size_t nbyte,
170 off_t offset, int flags, user_ssize_t *retval);
171 __private_extern__ int dofilewrite(vfs_context_t ctx, struct fileproc *fp,
172 user_addr_t bufp, user_size_t nbyte,
173 off_t offset, int flags, user_ssize_t *retval);
174 static int preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_vnode);
175
176 /* needed by guarded_writev, etc. */
177 int write_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
178 off_t offset, int flags, guardid_t *puguard, user_ssize_t *retval);
179 int writev_uio(struct proc *p, int fd, user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
180 guardid_t *puguard, user_ssize_t *retval);
181
182 #define f_flag fp_glob->fg_flag
183 #define f_type fp_glob->fg_ops->fo_type
184 #define f_cred fp_glob->fg_cred
185 #define f_ops fp_glob->fg_ops
186
187 /*
188 * Validate if the file can be used for random access (pread, pwrite, etc).
189 *
190 * Conditions:
191 * proc_fdlock is held
192 *
193 * Returns: 0 Success
194 * ESPIPE
195 * ENXIO
196 */
197 static int
valid_for_random_access(struct fileproc * fp)198 valid_for_random_access(struct fileproc *fp)
199 {
200 if (__improbable(fp->f_type != DTYPE_VNODE)) {
201 return ESPIPE;
202 }
203
204 vnode_t vp = (struct vnode *)fp_get_data(fp);
205 if (__improbable(vnode_isfifo(vp))) {
206 return ESPIPE;
207 }
208
209 if (__improbable(vp->v_flag & VISTTY)) {
210 return ENXIO;
211 }
212
213 return 0;
214 }
215
216 /*
217 * Returns: 0 Success
218 * EBADF
219 * ESPIPE
220 * ENXIO
221 * fp_lookup:EBADF
222 * valid_for_random_access:ESPIPE
223 * valid_for_random_access:ENXIO
224 */
225 static int
preparefileread(struct proc * p,struct fileproc ** fp_ret,int fd,int check_for_pread)226 preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_pread)
227 {
228 int error;
229 struct fileproc *fp;
230
231 AUDIT_ARG(fd, fd);
232
233 proc_fdlock_spin(p);
234
235 error = fp_lookup(p, fd, &fp, 1);
236
237 if (error) {
238 proc_fdunlock(p);
239 return error;
240 }
241 if ((fp->f_flag & FREAD) == 0) {
242 error = EBADF;
243 goto out;
244 }
245 if (check_for_pread) {
246 if ((error = valid_for_random_access(fp))) {
247 goto out;
248 }
249 }
250
251 *fp_ret = fp;
252
253 proc_fdunlock(p);
254 return 0;
255
256 out:
257 fp_drop(p, fd, fp, 1);
258 proc_fdunlock(p);
259 return error;
260 }
261
262 static int
fp_readv(vfs_context_t ctx,struct fileproc * fp,uio_t uio,int flags,user_ssize_t * retval)263 fp_readv(vfs_context_t ctx, struct fileproc *fp, uio_t uio, int flags,
264 user_ssize_t *retval)
265 {
266 int error;
267 user_ssize_t count;
268
269 if ((error = uio_calculateresid_user(uio))) {
270 *retval = 0;
271 return error;
272 }
273
274 count = uio_resid(uio);
275 error = fo_read(fp, uio, flags, ctx);
276
277 switch (error) {
278 case ERESTART:
279 case EINTR:
280 case EWOULDBLOCK:
281 if (uio_resid(uio) != count) {
282 error = 0;
283 }
284 break;
285
286 default:
287 break;
288 }
289
290 *retval = count - uio_resid(uio);
291 return error;
292 }
293
294 /*
295 * Returns: 0 Success
296 * EINVAL
297 * fo_read:???
298 */
299 __private_extern__ int
dofileread(vfs_context_t ctx,struct fileproc * fp,user_addr_t bufp,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)300 dofileread(vfs_context_t ctx, struct fileproc *fp,
301 user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
302 user_ssize_t *retval)
303 {
304 UIO_STACKBUF(uio_buf, 1);
305 uio_t uio;
306 int spacetype;
307
308 if (nbyte > INT_MAX) {
309 *retval = 0;
310 return EINVAL;
311 }
312
313 spacetype = vfs_context_is64bit(ctx) ? UIO_USERSPACE64 : UIO_USERSPACE32;
314 uio = uio_createwithbuffer(1, offset, spacetype, UIO_READ, &uio_buf[0],
315 sizeof(uio_buf));
316
317 if (uio_addiov(uio, bufp, nbyte) != 0) {
318 *retval = 0;
319 return EINVAL;
320 }
321
322 return fp_readv(ctx, fp, uio, flags, retval);
323 }
324
325 static int
readv_internal(struct proc * p,int fd,uio_t uio,int flags,user_ssize_t * retval)326 readv_internal(struct proc *p, int fd, uio_t uio, int flags,
327 user_ssize_t *retval)
328 {
329 struct fileproc *fp = NULL;
330 struct vfs_context context;
331 int error;
332
333 if ((error = preparefileread(p, &fp, fd, flags & FOF_OFFSET))) {
334 *retval = 0;
335 return error;
336 }
337
338 context = *(vfs_context_current());
339 context.vc_ucred = fp->fp_glob->fg_cred;
340
341 error = fp_readv(&context, fp, uio, flags, retval);
342
343 fp_drop(p, fd, fp, 0);
344 return error;
345 }
346
347 static int
read_internal(struct proc * p,int fd,user_addr_t buf,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)348 read_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
349 off_t offset, int flags, user_ssize_t *retval)
350 {
351 UIO_STACKBUF(uio_buf, 1);
352 uio_t uio;
353 int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
354
355 if (nbyte > INT_MAX) {
356 *retval = 0;
357 return EINVAL;
358 }
359
360 uio = uio_createwithbuffer(1, offset, spacetype, UIO_READ,
361 &uio_buf[0], sizeof(uio_buf));
362
363 if (uio_addiov(uio, buf, nbyte) != 0) {
364 *retval = 0;
365 return EINVAL;
366 }
367
368 return readv_internal(p, fd, uio, flags, retval);
369 }
370
371 int
read_nocancel(struct proc * p,struct read_nocancel_args * uap,user_ssize_t * retval)372 read_nocancel(struct proc *p, struct read_nocancel_args *uap, user_ssize_t *retval)
373 {
374 return read_internal(p, uap->fd, uap->cbuf, uap->nbyte, (off_t)-1, 0,
375 retval);
376 }
377
378 /*
379 * Read system call.
380 *
381 * Returns: 0 Success
382 * preparefileread:EBADF
383 * preparefileread:ESPIPE
384 * preparefileread:ENXIO
385 * preparefileread:EBADF
386 * dofileread:???
387 */
388 int
read(struct proc * p,struct read_args * uap,user_ssize_t * retval)389 read(struct proc *p, struct read_args *uap, user_ssize_t *retval)
390 {
391 __pthread_testcancel(1);
392 return read_nocancel(p, (struct read_nocancel_args *)uap, retval);
393 }
394
395 int
pread_nocancel(struct proc * p,struct pread_nocancel_args * uap,user_ssize_t * retval)396 pread_nocancel(struct proc *p, struct pread_nocancel_args *uap, user_ssize_t *retval)
397 {
398 KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_pread) | DBG_FUNC_NONE),
399 uap->fd, uap->nbyte, (unsigned int)((uap->offset >> 32)), (unsigned int)(uap->offset), 0);
400
401 return read_internal(p, uap->fd, uap->buf, uap->nbyte, uap->offset,
402 FOF_OFFSET, retval);
403 }
404
405 /*
406 * Pread system call
407 *
408 * Returns: 0 Success
409 * preparefileread:EBADF
410 * preparefileread:ESPIPE
411 * preparefileread:ENXIO
412 * preparefileread:EBADF
413 * dofileread:???
414 */
415 int
pread(struct proc * p,struct pread_args * uap,user_ssize_t * retval)416 pread(struct proc *p, struct pread_args *uap, user_ssize_t *retval)
417 {
418 __pthread_testcancel(1);
419 return pread_nocancel(p, (struct pread_nocancel_args *)uap, retval);
420 }
421
422 /*
423 * Vector read.
424 *
425 * Returns: 0 Success
426 * EINVAL
427 * ENOMEM
428 * preparefileread:EBADF
429 * preparefileread:ESPIPE
430 * preparefileread:ENXIO
431 * preparefileread:EBADF
432 * copyin:EFAULT
433 * rd_uio:???
434 */
435 static int
readv_uio(struct proc * p,int fd,user_addr_t user_iovp,int iovcnt,off_t offset,int flags,user_ssize_t * retval)436 readv_uio(struct proc *p, int fd,
437 user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
438 user_ssize_t *retval)
439 {
440 uio_t uio = NULL;
441 int error;
442 struct user_iovec *iovp;
443
444 if (iovcnt <= 0 || iovcnt > UIO_MAXIOV) {
445 error = EINVAL;
446 goto out;
447 }
448
449 uio = uio_create(iovcnt, offset,
450 (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
451 UIO_READ);
452
453 iovp = uio_iovsaddr_user(uio);
454 if (iovp == NULL) {
455 error = ENOMEM;
456 goto out;
457 }
458
459 error = copyin_user_iovec_array(user_iovp,
460 IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32,
461 iovcnt, iovp);
462
463 if (error) {
464 goto out;
465 }
466
467 error = readv_internal(p, fd, uio, flags, retval);
468
469 out:
470 if (uio != NULL) {
471 uio_free(uio);
472 }
473
474 return error;
475 }
476
477 int
readv_nocancel(struct proc * p,struct readv_nocancel_args * uap,user_ssize_t * retval)478 readv_nocancel(struct proc *p, struct readv_nocancel_args *uap, user_ssize_t *retval)
479 {
480 return readv_uio(p, uap->fd, uap->iovp, uap->iovcnt, 0, 0, retval);
481 }
482
483 /*
484 * Scatter read system call.
485 */
486 int
readv(struct proc * p,struct readv_args * uap,user_ssize_t * retval)487 readv(struct proc *p, struct readv_args *uap, user_ssize_t *retval)
488 {
489 __pthread_testcancel(1);
490 return readv_nocancel(p, (struct readv_nocancel_args *)uap, retval);
491 }
492
493 int
sys_preadv_nocancel(struct proc * p,struct preadv_nocancel_args * uap,user_ssize_t * retval)494 sys_preadv_nocancel(struct proc *p, struct preadv_nocancel_args *uap, user_ssize_t *retval)
495 {
496 return readv_uio(p, uap->fd, uap->iovp, uap->iovcnt, uap->offset,
497 FOF_OFFSET, retval);
498 }
499
500 /*
501 * Preadv system call
502 */
503 int
sys_preadv(struct proc * p,struct preadv_args * uap,user_ssize_t * retval)504 sys_preadv(struct proc *p, struct preadv_args *uap, user_ssize_t *retval)
505 {
506 __pthread_testcancel(1);
507 return sys_preadv_nocancel(p, (struct preadv_nocancel_args *)uap, retval);
508 }
509
510 /*
511 * Returns: 0 Success
512 * EBADF
513 * ESPIPE
514 * ENXIO
515 * fp_lookup:EBADF
516 * fp_guard_exception:???
517 * valid_for_random_access:ESPIPE
518 * valid_for_random_access:ENXIO
519 */
520 static int
preparefilewrite(struct proc * p,struct fileproc ** fp_ret,int fd,int check_for_pwrite,guardid_t * puguard)521 preparefilewrite(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_pwrite,
522 guardid_t *puguard)
523 {
524 int error;
525 struct fileproc *fp;
526
527 AUDIT_ARG(fd, fd);
528
529 proc_fdlock_spin(p);
530
531 if (puguard) {
532 error = fp_lookup_guarded(p, fd, *puguard, &fp, 1);
533 if (error) {
534 proc_fdunlock(p);
535 return error;
536 }
537
538 if ((fp->f_flag & FWRITE) == 0) {
539 error = EBADF;
540 goto out;
541 }
542 } else {
543 error = fp_lookup(p, fd, &fp, 1);
544 if (error) {
545 proc_fdunlock(p);
546 return error;
547 }
548
549 /* Allow EBADF first. */
550 if ((fp->f_flag & FWRITE) == 0) {
551 error = EBADF;
552 goto out;
553 }
554
555 if (fp_isguarded(fp, GUARD_WRITE)) {
556 error = fp_guard_exception(p, fd, fp, kGUARD_EXC_WRITE);
557 goto out;
558 }
559 }
560
561 if (check_for_pwrite) {
562 if ((error = valid_for_random_access(fp))) {
563 goto out;
564 }
565 }
566
567 *fp_ret = fp;
568
569 proc_fdunlock(p);
570 return 0;
571
572 out:
573 fp_drop(p, fd, fp, 1);
574 proc_fdunlock(p);
575 return error;
576 }
577
578 static int
fp_writev(vfs_context_t ctx,struct fileproc * fp,uio_t uio,int flags,user_ssize_t * retval)579 fp_writev(vfs_context_t ctx, struct fileproc *fp, uio_t uio, int flags,
580 user_ssize_t *retval)
581 {
582 int error;
583 user_ssize_t count;
584
585 if ((error = uio_calculateresid_user(uio))) {
586 *retval = 0;
587 return error;
588 }
589
590 count = uio_resid(uio);
591 error = fo_write(fp, uio, flags, ctx);
592
593 switch (error) {
594 case ERESTART:
595 case EINTR:
596 case EWOULDBLOCK:
597 if (uio_resid(uio) != count) {
598 error = 0;
599 }
600 break;
601
602 case EPIPE:
603 if (fp->f_type != DTYPE_SOCKET &&
604 (fp->fp_glob->fg_lflags & FG_NOSIGPIPE) == 0) {
605 /* XXX Raise the signal on the thread? */
606 psignal(vfs_context_proc(ctx), SIGPIPE);
607 }
608 break;
609
610 default:
611 break;
612 }
613
614 if ((*retval = count - uio_resid(uio))) {
615 os_atomic_or(&fp->fp_glob->fg_flag, FWASWRITTEN, relaxed);
616 }
617
618 return error;
619 }
620
621 /*
622 * Returns: 0 Success
623 * EINVAL
624 * <fo_write>:EPIPE
625 * <fo_write>:??? [indirect through struct fileops]
626 */
627 __private_extern__ int
dofilewrite(vfs_context_t ctx,struct fileproc * fp,user_addr_t bufp,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)628 dofilewrite(vfs_context_t ctx, struct fileproc *fp,
629 user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
630 user_ssize_t *retval)
631 {
632 UIO_STACKBUF(uio_buf, 1);
633 uio_t uio;
634 int spacetype;
635
636 if (nbyte > INT_MAX) {
637 *retval = 0;
638 return EINVAL;
639 }
640
641 spacetype = vfs_context_is64bit(ctx) ? UIO_USERSPACE64 : UIO_USERSPACE32;
642 uio = uio_createwithbuffer(1, offset, spacetype, UIO_WRITE, &uio_buf[0],
643 sizeof(uio_buf));
644
645 if (uio_addiov(uio, bufp, nbyte) != 0) {
646 *retval = 0;
647 return EINVAL;
648 }
649
650 return fp_writev(ctx, fp, uio, flags, retval);
651 }
652
653 static int
writev_internal(struct proc * p,int fd,uio_t uio,int flags,guardid_t * puguard,user_ssize_t * retval)654 writev_internal(struct proc *p, int fd, uio_t uio, int flags,
655 guardid_t *puguard, user_ssize_t *retval)
656 {
657 struct fileproc *fp = NULL;
658 struct vfs_context context;
659 int error;
660
661 if ((error = preparefilewrite(p, &fp, fd, flags & FOF_OFFSET, puguard))) {
662 *retval = 0;
663 return error;
664 }
665
666 context = *(vfs_context_current());
667 context.vc_ucred = fp->fp_glob->fg_cred;
668
669 error = fp_writev(&context, fp, uio, flags, retval);
670
671 fp_drop(p, fd, fp, 0);
672 return error;
673 }
674
675 int
write_internal(struct proc * p,int fd,user_addr_t buf,user_size_t nbyte,off_t offset,int flags,guardid_t * puguard,user_ssize_t * retval)676 write_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
677 off_t offset, int flags, guardid_t *puguard, user_ssize_t *retval)
678 {
679 UIO_STACKBUF(uio_buf, 1);
680 uio_t uio;
681 int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
682
683 if (nbyte > INT_MAX) {
684 *retval = 0;
685 return EINVAL;
686 }
687
688 uio = uio_createwithbuffer(1, offset, spacetype, UIO_WRITE,
689 &uio_buf[0], sizeof(uio_buf));
690
691 if (uio_addiov(uio, buf, nbyte) != 0) {
692 *retval = 0;
693 return EINVAL;
694 }
695
696 return writev_internal(p, fd, uio, flags, puguard, retval);
697 }
698
699 int
write_nocancel(struct proc * p,struct write_nocancel_args * uap,user_ssize_t * retval)700 write_nocancel(struct proc *p, struct write_nocancel_args *uap, user_ssize_t *retval)
701 {
702 return write_internal(p, uap->fd, uap->cbuf, uap->nbyte, (off_t)-1, 0,
703 NULL, retval);
704 }
705
706 /*
707 * Write system call
708 *
709 * Returns: 0 Success
710 * EBADF
711 * fp_lookup:EBADF
712 * dofilewrite:???
713 */
714 int
write(struct proc * p,struct write_args * uap,user_ssize_t * retval)715 write(struct proc *p, struct write_args *uap, user_ssize_t *retval)
716 {
717 __pthread_testcancel(1);
718 return write_nocancel(p, (struct write_nocancel_args *)uap, retval);
719 }
720
721 int
pwrite_nocancel(struct proc * p,struct pwrite_nocancel_args * uap,user_ssize_t * retval)722 pwrite_nocancel(struct proc *p, struct pwrite_nocancel_args *uap, user_ssize_t *retval)
723 {
724 KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_pwrite) | DBG_FUNC_NONE),
725 uap->fd, uap->nbyte, (unsigned int)((uap->offset >> 32)), (unsigned int)(uap->offset), 0);
726
727 /* XXX: Should be < 0 instead? (See man page + pwritev) */
728 if (uap->offset == (off_t)-1) {
729 return EINVAL;
730 }
731
732 return write_internal(p, uap->fd, uap->buf, uap->nbyte, uap->offset,
733 FOF_OFFSET, NULL, retval);
734 }
735
736 /*
737 * pwrite system call
738 *
739 * Returns: 0 Success
740 * EBADF
741 * ESPIPE
742 * ENXIO
743 * EINVAL
744 * fp_lookup:EBADF
745 * dofilewrite:???
746 */
747 int
pwrite(struct proc * p,struct pwrite_args * uap,user_ssize_t * retval)748 pwrite(struct proc *p, struct pwrite_args *uap, user_ssize_t *retval)
749 {
750 __pthread_testcancel(1);
751 return pwrite_nocancel(p, (struct pwrite_nocancel_args *)uap, retval);
752 }
753
754 int
writev_uio(struct proc * p,int fd,user_addr_t user_iovp,int iovcnt,off_t offset,int flags,guardid_t * puguard,user_ssize_t * retval)755 writev_uio(struct proc *p, int fd,
756 user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
757 guardid_t *puguard, user_ssize_t *retval)
758 {
759 uio_t uio = NULL;
760 int error;
761 struct user_iovec *iovp;
762
763 if (iovcnt <= 0 || iovcnt > UIO_MAXIOV || offset < 0) {
764 error = EINVAL;
765 goto out;
766 }
767
768 uio = uio_create(iovcnt, offset,
769 (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
770 UIO_WRITE);
771
772 iovp = uio_iovsaddr_user(uio);
773 if (iovp == NULL) {
774 error = ENOMEM;
775 goto out;
776 }
777
778 error = copyin_user_iovec_array(user_iovp,
779 IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32,
780 iovcnt, iovp);
781
782 if (error) {
783 goto out;
784 }
785
786 error = writev_internal(p, fd, uio, flags, puguard, retval);
787
788 out:
789 if (uio != NULL) {
790 uio_free(uio);
791 }
792
793 return error;
794 }
795
796 int
writev_nocancel(struct proc * p,struct writev_nocancel_args * uap,user_ssize_t * retval)797 writev_nocancel(struct proc *p, struct writev_nocancel_args *uap, user_ssize_t *retval)
798 {
799 return writev_uio(p, uap->fd, uap->iovp, uap->iovcnt, 0, 0, NULL, retval);
800 }
801
802 /*
803 * Gather write system call
804 */
805 int
writev(struct proc * p,struct writev_args * uap,user_ssize_t * retval)806 writev(struct proc *p, struct writev_args *uap, user_ssize_t *retval)
807 {
808 __pthread_testcancel(1);
809 return writev_nocancel(p, (struct writev_nocancel_args *)uap, retval);
810 }
811
812 int
sys_pwritev_nocancel(struct proc * p,struct pwritev_nocancel_args * uap,user_ssize_t * retval)813 sys_pwritev_nocancel(struct proc *p, struct pwritev_nocancel_args *uap, user_ssize_t *retval)
814 {
815 return writev_uio(p, uap->fd, uap->iovp, uap->iovcnt, uap->offset,
816 FOF_OFFSET, NULL, retval);
817 }
818
819 /*
820 * Pwritev system call
821 */
822 int
sys_pwritev(struct proc * p,struct pwritev_args * uap,user_ssize_t * retval)823 sys_pwritev(struct proc *p, struct pwritev_args *uap, user_ssize_t *retval)
824 {
825 __pthread_testcancel(1);
826 return sys_pwritev_nocancel(p, (struct pwritev_nocancel_args *)uap, retval);
827 }
828
829 /*
830 * Ioctl system call
831 *
832 * Returns: 0 Success
833 * EBADF
834 * ENOTTY
835 * ENOMEM
836 * ESRCH
837 * copyin:EFAULT
838 * copyoutEFAULT
839 * fp_lookup:EBADF Bad file descriptor
840 * fo_ioctl:???
841 */
842 int
ioctl(struct proc * p,struct ioctl_args * uap,__unused int32_t * retval)843 ioctl(struct proc *p, struct ioctl_args *uap, __unused int32_t *retval)
844 {
845 struct fileproc *fp = NULL;
846 int error = 0;
847 u_int size = 0;
848 caddr_t datap = NULL, memp = NULL;
849 boolean_t is64bit = FALSE;
850 int tmp = 0;
851 #define STK_PARAMS 128
852 char stkbuf[STK_PARAMS] = {};
853 int fd = uap->fd;
854 u_long com = uap->com;
855 struct vfs_context context = *vfs_context_current();
856
857 AUDIT_ARG(fd, uap->fd);
858 AUDIT_ARG(addr, uap->data);
859
860 is64bit = proc_is64bit(p);
861 #if CONFIG_AUDIT
862 if (is64bit) {
863 AUDIT_ARG(value64, com);
864 } else {
865 AUDIT_ARG(cmd, CAST_DOWN_EXPLICIT(int, com));
866 }
867 #endif /* CONFIG_AUDIT */
868
869 /*
870 * Interpret high order word to find amount of data to be
871 * copied to/from the user's address space.
872 */
873 size = IOCPARM_LEN(com);
874 if (size > IOCPARM_MAX) {
875 return ENOTTY;
876 }
877 if (size > sizeof(stkbuf)) {
878 memp = (caddr_t)kalloc_data(size, Z_WAITOK);
879 if (memp == 0) {
880 return ENOMEM;
881 }
882 datap = memp;
883 } else {
884 datap = &stkbuf[0];
885 }
886 if (com & IOC_IN) {
887 if (size) {
888 error = copyin(uap->data, datap, size);
889 if (error) {
890 goto out_nofp;
891 }
892 } else {
893 /* XXX - IOC_IN and no size? we should proably return an error here!! */
894 if (is64bit) {
895 *(user_addr_t *)datap = uap->data;
896 } else {
897 *(uint32_t *)datap = (uint32_t)uap->data;
898 }
899 }
900 } else if ((com & IOC_OUT) && size) {
901 /*
902 * Zero the buffer so the user always
903 * gets back something deterministic.
904 */
905 bzero(datap, size);
906 } else if (com & IOC_VOID) {
907 /* XXX - this is odd since IOC_VOID means no parameters */
908 if (is64bit) {
909 *(user_addr_t *)datap = uap->data;
910 } else {
911 *(uint32_t *)datap = (uint32_t)uap->data;
912 }
913 }
914
915 proc_fdlock(p);
916 error = fp_lookup(p, fd, &fp, 1);
917 if (error) {
918 proc_fdunlock(p);
919 goto out_nofp;
920 }
921
922 AUDIT_ARG(file, p, fp);
923
924 if ((fp->f_flag & (FREAD | FWRITE)) == 0) {
925 error = EBADF;
926 goto out;
927 }
928
929 context.vc_ucred = fp->fp_glob->fg_cred;
930
931 #if CONFIG_MACF
932 error = mac_file_check_ioctl(context.vc_ucred, fp->fp_glob, com);
933 if (error) {
934 goto out;
935 }
936 #endif
937
938 switch (com) {
939 case FIONCLEX:
940 fp->fp_flags &= ~FP_CLOEXEC;
941 break;
942
943 case FIOCLEX:
944 fp->fp_flags |= FP_CLOEXEC;
945 break;
946
947 case FIONBIO:
948 // FIXME (rdar://54898652)
949 //
950 // this code is broken if fnctl(F_SETFL), ioctl() are
951 // called concurrently for the same fileglob.
952 if ((tmp = *(int *)datap)) {
953 os_atomic_or(&fp->f_flag, FNONBLOCK, relaxed);
954 } else {
955 os_atomic_andnot(&fp->f_flag, FNONBLOCK, relaxed);
956 }
957 error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, &context);
958 break;
959
960 case FIOASYNC:
961 // FIXME (rdar://54898652)
962 //
963 // this code is broken if fnctl(F_SETFL), ioctl() are
964 // called concurrently for the same fileglob.
965 if ((tmp = *(int *)datap)) {
966 os_atomic_or(&fp->f_flag, FASYNC, relaxed);
967 } else {
968 os_atomic_andnot(&fp->f_flag, FASYNC, relaxed);
969 }
970 error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, &context);
971 break;
972
973 case FIOSETOWN:
974 tmp = *(int *)datap;
975 if (fp->f_type == DTYPE_SOCKET) {
976 ((struct socket *)fp_get_data(fp))->so_pgid = tmp;
977 break;
978 }
979 if (fp->f_type == DTYPE_PIPE) {
980 error = fo_ioctl(fp, TIOCSPGRP, (caddr_t)&tmp, &context);
981 break;
982 }
983 if (tmp <= 0) {
984 tmp = -tmp;
985 } else {
986 struct proc *p1 = proc_find(tmp);
987 if (p1 == 0) {
988 error = ESRCH;
989 break;
990 }
991 tmp = p1->p_pgrpid;
992 proc_rele(p1);
993 }
994 error = fo_ioctl(fp, TIOCSPGRP, (caddr_t)&tmp, &context);
995 break;
996
997 case FIOGETOWN:
998 if (fp->f_type == DTYPE_SOCKET) {
999 *(int *)datap = ((struct socket *)fp_get_data(fp))->so_pgid;
1000 break;
1001 }
1002 error = fo_ioctl(fp, TIOCGPGRP, datap, &context);
1003 *(int *)datap = -*(int *)datap;
1004 break;
1005
1006 default:
1007 error = fo_ioctl(fp, com, datap, &context);
1008 /*
1009 * Copy any data to user, size was
1010 * already set and checked above.
1011 */
1012 if (error == 0 && (com & IOC_OUT) && size) {
1013 error = copyout(datap, uap->data, (u_int)size);
1014 }
1015 break;
1016 }
1017 out:
1018 fp_drop(p, fd, fp, 1);
1019 proc_fdunlock(p);
1020
1021 out_nofp:
1022 if (memp) {
1023 kfree_data(memp, size);
1024 }
1025 return error;
1026 }
1027
1028 int selwait;
1029 #define SEL_FIRSTPASS 1
1030 #define SEL_SECONDPASS 2
1031 static int selprocess(struct proc *p, int error, int sel_pass);
1032 static int selscan(struct proc *p, struct _select * sel, struct _select_data * seldata,
1033 int nfd, int32_t *retval, int sel_pass, struct select_set *selset);
1034 static int selcount(struct proc *p, u_int32_t *ibits, int nfd, int *count);
1035 static int seldrop_locked(struct proc *p, u_int32_t *ibits, int nfd, int lim, int *need_wakeup);
1036 static int seldrop(struct proc *p, u_int32_t *ibits, int nfd, int lim);
1037 static int select_internal(struct proc *p, struct select_nocancel_args *uap, uint64_t timeout, int32_t *retval);
1038
1039 /*
1040 * This is used for the special device nodes that do not implement
1041 * a proper kevent filter (see filt_specattach).
1042 *
1043 * In order to enable kevents on those, the spec_filtops will pretend
1044 * to call select, and try to sniff the selrecord(), if it observes one,
1045 * the knote is attached, which pairs with selwakeup() or selthreadclear().
1046 *
1047 * The last issue remaining, is that we need to serialize filt_specdetach()
1048 * with this, but it really can't know the "selinfo" or any locking domain.
1049 * To make up for this, We protect knote list operations with a global lock,
1050 * which give us a safe shared locking domain.
1051 *
1052 * Note: It is a little distasteful, but we really have very few of those.
1053 * The big problem here is that sharing a lock domain without
1054 * any kind of shared knowledge is a little complicated.
1055 *
1056 * 1. filters can really implement their own kqueue integration
1057 * to side step this,
1058 *
1059 * 2. There's an opportunity to pick a private lock in selspec_attach()
1060 * because both the selinfo and the knote are locked at that time.
1061 * The cleanup story is however a little complicated.
1062 */
1063 static LCK_GRP_DECLARE(selspec_grp, "spec_filtops");
1064 static LCK_SPIN_DECLARE(selspec_lock, &selspec_grp);
1065
1066 /*
1067 * The "primitive" lock is held.
1068 * The knote lock is held.
1069 */
1070 void
selspec_attach(struct knote * kn,struct selinfo * si)1071 selspec_attach(struct knote *kn, struct selinfo *si)
1072 {
1073 struct selinfo *cur = knote_kn_hook_get_raw(kn);
1074
1075 if (cur == NULL) {
1076 si->si_flags |= SI_SELSPEC;
1077 lck_spin_lock(&selspec_lock);
1078 knote_kn_hook_set_raw(kn, (void *) si);
1079 KNOTE_ATTACH(&si->si_note, kn);
1080 lck_spin_unlock(&selspec_lock);
1081 } else {
1082 /*
1083 * selspec_attach() can be called from e.g. filt_spectouch()
1084 * which might be called before any event was dequeued.
1085 *
1086 * It is hence not impossible for the knote already be hooked.
1087 *
1088 * Note that selwakeup_internal() could possibly
1089 * already have cleared this pointer. This is a race
1090 * that filt_specprocess will debounce.
1091 */
1092 assert(si->si_flags & SI_SELSPEC);
1093 assert(cur == si);
1094 }
1095 }
1096
1097 /*
1098 * The "primitive" lock is _not_ held.
1099 *
1100 * knote "lock" is held
1101 */
1102 void
selspec_detach(struct knote * kn)1103 selspec_detach(struct knote *kn)
1104 {
1105 lck_spin_lock(&selspec_lock);
1106
1107 if (!KNOTE_IS_AUTODETACHED(kn)) {
1108 struct selinfo *sip = knote_kn_hook_get_raw(kn);
1109 if (sip) {
1110 KNOTE_DETACH(&sip->si_note, kn);
1111 }
1112 }
1113
1114 knote_kn_hook_set_raw(kn, NULL);
1115
1116 lck_spin_unlock(&selspec_lock);
1117 }
1118
1119 /*
1120 * Select system call.
1121 *
1122 * Returns: 0 Success
1123 * EINVAL Invalid argument
1124 * EAGAIN Nonconformant error if allocation fails
1125 */
1126 int
select(struct proc * p,struct select_args * uap,int32_t * retval)1127 select(struct proc *p, struct select_args *uap, int32_t *retval)
1128 {
1129 __pthread_testcancel(1);
1130 return select_nocancel(p, (struct select_nocancel_args *)uap, retval);
1131 }
1132
1133 int
select_nocancel(struct proc * p,struct select_nocancel_args * uap,int32_t * retval)1134 select_nocancel(struct proc *p, struct select_nocancel_args *uap, int32_t *retval)
1135 {
1136 uint64_t timeout = 0;
1137
1138 if (uap->tv) {
1139 int err;
1140 struct timeval atv;
1141 if (IS_64BIT_PROCESS(p)) {
1142 struct user64_timeval atv64;
1143 err = copyin(uap->tv, (caddr_t)&atv64, sizeof(atv64));
1144 /* Loses resolution - assume timeout < 68 years */
1145 atv.tv_sec = (__darwin_time_t)atv64.tv_sec;
1146 atv.tv_usec = atv64.tv_usec;
1147 } else {
1148 struct user32_timeval atv32;
1149 err = copyin(uap->tv, (caddr_t)&atv32, sizeof(atv32));
1150 atv.tv_sec = atv32.tv_sec;
1151 atv.tv_usec = atv32.tv_usec;
1152 }
1153 if (err) {
1154 return err;
1155 }
1156
1157 if (itimerfix(&atv)) {
1158 err = EINVAL;
1159 return err;
1160 }
1161
1162 clock_absolutetime_interval_to_deadline(tvtoabstime(&atv), &timeout);
1163 }
1164
1165 return select_internal(p, uap, timeout, retval);
1166 }
1167
1168 int
pselect(struct proc * p,struct pselect_args * uap,int32_t * retval)1169 pselect(struct proc *p, struct pselect_args *uap, int32_t *retval)
1170 {
1171 __pthread_testcancel(1);
1172 return pselect_nocancel(p, (struct pselect_nocancel_args *)uap, retval);
1173 }
1174
1175 int
pselect_nocancel(struct proc * p,struct pselect_nocancel_args * uap,int32_t * retval)1176 pselect_nocancel(struct proc *p, struct pselect_nocancel_args *uap, int32_t *retval)
1177 {
1178 int err;
1179 struct uthread *ut;
1180 uint64_t timeout = 0;
1181
1182 if (uap->ts) {
1183 struct timespec ts;
1184
1185 if (IS_64BIT_PROCESS(p)) {
1186 struct user64_timespec ts64;
1187 err = copyin(uap->ts, (caddr_t)&ts64, sizeof(ts64));
1188 ts.tv_sec = (__darwin_time_t)ts64.tv_sec;
1189 ts.tv_nsec = (long)ts64.tv_nsec;
1190 } else {
1191 struct user32_timespec ts32;
1192 err = copyin(uap->ts, (caddr_t)&ts32, sizeof(ts32));
1193 ts.tv_sec = ts32.tv_sec;
1194 ts.tv_nsec = ts32.tv_nsec;
1195 }
1196 if (err) {
1197 return err;
1198 }
1199
1200 if (!timespec_is_valid(&ts)) {
1201 return EINVAL;
1202 }
1203 clock_absolutetime_interval_to_deadline(tstoabstime(&ts), &timeout);
1204 }
1205
1206 ut = current_uthread();
1207
1208 if (uap->mask != USER_ADDR_NULL) {
1209 /* save current mask, then copyin and set new mask */
1210 sigset_t newset;
1211 err = copyin(uap->mask, &newset, sizeof(sigset_t));
1212 if (err) {
1213 return err;
1214 }
1215 ut->uu_oldmask = ut->uu_sigmask;
1216 ut->uu_flag |= UT_SAS_OLDMASK;
1217 ut->uu_sigmask = (newset & ~sigcantmask);
1218 }
1219
1220 err = select_internal(p, (struct select_nocancel_args *)uap, timeout, retval);
1221
1222 if (err != EINTR && ut->uu_flag & UT_SAS_OLDMASK) {
1223 /*
1224 * Restore old mask (direct return case). NOTE: EINTR can also be returned
1225 * if the thread is cancelled. In that case, we don't reset the signal
1226 * mask to its original value (which usually happens in the signal
1227 * delivery path). This behavior is permitted by POSIX.
1228 */
1229 ut->uu_sigmask = ut->uu_oldmask;
1230 ut->uu_oldmask = 0;
1231 ut->uu_flag &= ~UT_SAS_OLDMASK;
1232 }
1233
1234 return err;
1235 }
1236
1237 void
select_cleanup_uthread(struct _select * sel)1238 select_cleanup_uthread(struct _select *sel)
1239 {
1240 kfree_data(sel->ibits, 2 * sel->nbytes);
1241 sel->ibits = sel->obits = NULL;
1242 sel->nbytes = 0;
1243 }
1244
1245 static int
select_grow_uthread_cache(struct _select * sel,uint32_t nbytes)1246 select_grow_uthread_cache(struct _select *sel, uint32_t nbytes)
1247 {
1248 uint32_t *buf;
1249
1250 buf = kalloc_data(2 * nbytes, Z_WAITOK | Z_ZERO);
1251 if (buf) {
1252 select_cleanup_uthread(sel);
1253 sel->ibits = buf;
1254 sel->obits = buf + nbytes / sizeof(uint32_t);
1255 sel->nbytes = nbytes;
1256 return true;
1257 }
1258 return false;
1259 }
1260
1261 static void
select_bzero_uthread_cache(struct _select * sel)1262 select_bzero_uthread_cache(struct _select *sel)
1263 {
1264 bzero(sel->ibits, sel->nbytes * 2);
1265 }
1266
1267 /*
1268 * Generic implementation of {,p}select. Care: we type-pun uap across the two
1269 * syscalls, which differ slightly. The first 4 arguments (nfds and the fd sets)
1270 * are identical. The 5th (timeout) argument points to different types, so we
1271 * unpack in the syscall-specific code, but the generic code still does a null
1272 * check on this argument to determine if a timeout was specified.
1273 */
1274 static int
select_internal(struct proc * p,struct select_nocancel_args * uap,uint64_t timeout,int32_t * retval)1275 select_internal(struct proc *p, struct select_nocancel_args *uap, uint64_t timeout, int32_t *retval)
1276 {
1277 struct uthread *uth = current_uthread();
1278 struct _select *sel = &uth->uu_select;
1279 struct _select_data *seldata = &uth->uu_save.uus_select_data;
1280 int error = 0;
1281 u_int ni, nw;
1282
1283 *retval = 0;
1284
1285 seldata->abstime = timeout;
1286 seldata->args = uap;
1287 seldata->retval = retval;
1288 seldata->count = 0;
1289
1290 if (uap->nd < 0) {
1291 return EINVAL;
1292 }
1293
1294 if (uap->nd > p->p_fd.fd_nfiles) {
1295 uap->nd = p->p_fd.fd_nfiles; /* forgiving; slightly wrong */
1296 }
1297 nw = howmany(uap->nd, NFDBITS);
1298 ni = nw * sizeof(fd_mask);
1299
1300 /*
1301 * if the previously allocated space for the bits is smaller than
1302 * what is requested or no space has yet been allocated for this
1303 * thread, allocate enough space now.
1304 *
1305 * Note: If this process fails, select() will return EAGAIN; this
1306 * is the same thing pool() returns in a no-memory situation, but
1307 * it is not a POSIX compliant error code for select().
1308 */
1309 if (sel->nbytes >= (3 * ni)) {
1310 select_bzero_uthread_cache(sel);
1311 } else if (!select_grow_uthread_cache(sel, 3 * ni)) {
1312 return EAGAIN;
1313 }
1314
1315 /*
1316 * get the bits from the user address space
1317 */
1318 #define getbits(name, x) \
1319 (uap->name ? copyin(uap->name, &sel->ibits[(x) * nw], ni) : 0)
1320
1321 if ((error = getbits(in, 0))) {
1322 return error;
1323 }
1324 if ((error = getbits(ou, 1))) {
1325 return error;
1326 }
1327 if ((error = getbits(ex, 2))) {
1328 return error;
1329 }
1330 #undef getbits
1331
1332 if ((error = selcount(p, sel->ibits, uap->nd, &seldata->count))) {
1333 return error;
1334 }
1335
1336 if (uth->uu_selset == NULL) {
1337 uth->uu_selset = select_set_alloc();
1338 }
1339 return selprocess(p, 0, SEL_FIRSTPASS);
1340 }
1341
1342 static int
selcontinue(int error)1343 selcontinue(int error)
1344 {
1345 return selprocess(current_proc(), error, SEL_SECONDPASS);
1346 }
1347
1348
1349 /*
1350 * selprocess
1351 *
1352 * Parameters: error The error code from our caller
1353 * sel_pass The pass we are on
1354 */
1355 int
selprocess(struct proc * p,int error,int sel_pass)1356 selprocess(struct proc *p, int error, int sel_pass)
1357 {
1358 struct uthread *uth = current_uthread();
1359 struct _select *sel = &uth->uu_select;
1360 struct _select_data *seldata = &uth->uu_save.uus_select_data;
1361 struct select_nocancel_args *uap = seldata->args;
1362 int *retval = seldata->retval;
1363
1364 int unwind = 1;
1365 int prepost = 0;
1366 int somewakeup = 0;
1367 int doretry = 0;
1368 wait_result_t wait_result;
1369
1370 if ((error != 0) && (sel_pass == SEL_FIRSTPASS)) {
1371 unwind = 0;
1372 }
1373 if (seldata->count == 0) {
1374 unwind = 0;
1375 }
1376 retry:
1377 if (error != 0) {
1378 goto done;
1379 }
1380
1381 OSBitOrAtomic(P_SELECT, &p->p_flag);
1382
1383 /* skip scans if the select is just for timeouts */
1384 if (seldata->count) {
1385 error = selscan(p, sel, seldata, uap->nd, retval, sel_pass,
1386 uth->uu_selset);
1387 if (error || *retval) {
1388 goto done;
1389 }
1390 if (prepost || somewakeup) {
1391 /*
1392 * if the select of log, then we can wakeup and
1393 * discover some one else already read the data;
1394 * go to select again if time permits
1395 */
1396 prepost = 0;
1397 somewakeup = 0;
1398 doretry = 1;
1399 }
1400 }
1401
1402 if (uap->tv) {
1403 uint64_t now;
1404
1405 clock_get_uptime(&now);
1406 if (now >= seldata->abstime) {
1407 goto done;
1408 }
1409 }
1410
1411 if (doretry) {
1412 /* cleanup obits and try again */
1413 doretry = 0;
1414 sel_pass = SEL_FIRSTPASS;
1415 goto retry;
1416 }
1417
1418 /*
1419 * To effect a poll, the timeout argument should be
1420 * non-nil, pointing to a zero-valued timeval structure.
1421 */
1422 if (uap->tv && seldata->abstime == 0) {
1423 goto done;
1424 }
1425
1426 /* No spurious wakeups due to colls,no need to check for them */
1427 if ((sel_pass == SEL_SECONDPASS) || ((p->p_flag & P_SELECT) == 0)) {
1428 sel_pass = SEL_FIRSTPASS;
1429 goto retry;
1430 }
1431
1432 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1433
1434 /* if the select is just for timeout skip check */
1435 if (seldata->count && (sel_pass == SEL_SECONDPASS)) {
1436 panic("selprocess: 2nd pass assertwaiting");
1437 }
1438
1439 wait_result = waitq_assert_wait64_leeway(uth->uu_selset,
1440 NO_EVENT64, THREAD_ABORTSAFE,
1441 TIMEOUT_URGENCY_USER_NORMAL,
1442 seldata->abstime,
1443 TIMEOUT_NO_LEEWAY);
1444 if (wait_result != THREAD_AWAKENED) {
1445 /* there are no preposted events */
1446 error = tsleep1(NULL, PSOCK | PCATCH,
1447 "select", 0, selcontinue);
1448 } else {
1449 prepost = 1;
1450 error = 0;
1451 }
1452
1453 if (error == 0) {
1454 sel_pass = SEL_SECONDPASS;
1455 if (!prepost) {
1456 somewakeup = 1;
1457 }
1458 goto retry;
1459 }
1460 done:
1461 if (unwind) {
1462 seldrop(p, sel->ibits, uap->nd, seldata->count);
1463 select_set_reset(uth->uu_selset);
1464 }
1465 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1466 /* select is not restarted after signals... */
1467 if (error == ERESTART) {
1468 error = EINTR;
1469 }
1470 if (error == EWOULDBLOCK) {
1471 error = 0;
1472 }
1473
1474 if (error == 0) {
1475 uint32_t nw = howmany(uap->nd, NFDBITS);
1476 uint32_t ni = nw * sizeof(fd_mask);
1477
1478 #define putbits(name, x) \
1479 (uap->name ? copyout(&sel->obits[(x) * nw], uap->name, ni) : 0)
1480 int e0 = putbits(in, 0);
1481 int e1 = putbits(ou, 1);
1482 int e2 = putbits(ex, 2);
1483
1484 error = e0 ?: e1 ?: e2;
1485 #undef putbits
1486 }
1487
1488 if (error != EINTR && sel_pass == SEL_SECONDPASS && uth->uu_flag & UT_SAS_OLDMASK) {
1489 /* restore signal mask - continuation case */
1490 uth->uu_sigmask = uth->uu_oldmask;
1491 uth->uu_oldmask = 0;
1492 uth->uu_flag &= ~UT_SAS_OLDMASK;
1493 }
1494
1495 return error;
1496 }
1497
1498
1499 /**
1500 * remove the fileproc's underlying waitq from the supplied waitq set;
1501 * clear FP_INSELECT when appropriate
1502 *
1503 * Parameters:
1504 * fp File proc that is potentially currently in select
1505 * selset Waitq set to which the fileproc may belong
1506 * (usually this is the thread's private waitq set)
1507 * Conditions:
1508 * proc_fdlock is held
1509 */
1510 static void
selunlinkfp(struct fileproc * fp,struct select_set * selset)1511 selunlinkfp(struct fileproc *fp, struct select_set *selset)
1512 {
1513 if (fp->fp_flags & FP_INSELECT) {
1514 if (fp->fp_guard_attrs) {
1515 if (fp->fp_guard->fpg_wset == selset) {
1516 fp->fp_guard->fpg_wset = NULL;
1517 fp->fp_flags &= ~FP_INSELECT;
1518 }
1519 } else {
1520 if (fp->fp_wset == selset) {
1521 fp->fp_wset = NULL;
1522 fp->fp_flags &= ~FP_INSELECT;
1523 }
1524 }
1525 }
1526 }
1527
1528 /**
1529 * connect a fileproc to the given selset, potentially bridging to a waitq
1530 * pointed to indirectly by wq_data
1531 *
1532 * Parameters:
1533 * fp File proc potentially currently in select
1534 * selset Waitq set to which the fileproc should now belong
1535 * (usually this is the thread's private waitq set)
1536 *
1537 * Conditions:
1538 * proc_fdlock is held
1539 */
1540 static void
sellinkfp(struct fileproc * fp,struct select_set * selset,waitq_link_t * linkp)1541 sellinkfp(struct fileproc *fp, struct select_set *selset, waitq_link_t *linkp)
1542 {
1543 if ((fp->fp_flags & FP_INSELECT) == 0) {
1544 if (fp->fp_guard_attrs) {
1545 fp->fp_guard->fpg_wset = selset;
1546 } else {
1547 fp->fp_wset = selset;
1548 }
1549 fp->fp_flags |= FP_INSELECT;
1550 } else {
1551 fp->fp_flags |= FP_SELCONFLICT;
1552 if (linkp->wqlh == NULL) {
1553 *linkp = waitq_link_alloc(WQT_SELECT_SET);
1554 }
1555 select_set_link(&select_conflict_queue, selset, linkp);
1556 }
1557 }
1558
1559
1560 /*
1561 * selscan
1562 *
1563 * Parameters: p Process performing the select
1564 * sel The per-thread select context structure
1565 * nfd The number of file descriptors to scan
1566 * retval The per thread system call return area
1567 * sel_pass Which pass this is; allowed values are
1568 * SEL_FIRSTPASS and SEL_SECONDPASS
1569 * selset The per thread wait queue set
1570 *
1571 * Returns: 0 Success
1572 * EIO Invalid p->p_fd field XXX Obsolete?
1573 * EBADF One of the files in the bit vector is
1574 * invalid.
1575 */
1576 static int
selscan(struct proc * p,struct _select * sel,struct _select_data * seldata,int nfd,int32_t * retval,int sel_pass,struct select_set * selset)1577 selscan(struct proc *p, struct _select *sel, struct _select_data * seldata,
1578 int nfd, int32_t *retval, int sel_pass, struct select_set *selset)
1579 {
1580 int msk, i, j, fd;
1581 u_int32_t bits;
1582 struct fileproc *fp;
1583 int n = 0; /* count of bits */
1584 int nc = 0; /* bit vector offset (nc'th bit) */
1585 static int flag[3] = { FREAD, FWRITE, 0 };
1586 u_int32_t *iptr, *optr;
1587 u_int nw;
1588 u_int32_t *ibits, *obits;
1589 int count;
1590 struct vfs_context context = {
1591 .vc_thread = current_thread(),
1592 };
1593 waitq_link_t link = WQL_NULL;
1594 void *s_data;
1595
1596 ibits = sel->ibits;
1597 obits = sel->obits;
1598
1599 nw = howmany(nfd, NFDBITS);
1600
1601 count = seldata->count;
1602
1603 nc = 0;
1604 if (!count) {
1605 *retval = 0;
1606 return 0;
1607 }
1608
1609 if (sel_pass == SEL_FIRSTPASS) {
1610 /*
1611 * Make sure the waitq-set is all clean:
1612 *
1613 * select loops until it finds at least one event, however it
1614 * doesn't mean that the event that woke up select is still
1615 * fired by the time the second pass runs, and then
1616 * select_internal will loop back to a first pass.
1617 */
1618 select_set_reset(selset);
1619 s_data = &link;
1620 } else {
1621 s_data = NULL;
1622 }
1623
1624 proc_fdlock(p);
1625 for (msk = 0; msk < 3; msk++) {
1626 iptr = (u_int32_t *)&ibits[msk * nw];
1627 optr = (u_int32_t *)&obits[msk * nw];
1628
1629 for (i = 0; i < nfd; i += NFDBITS) {
1630 bits = iptr[i / NFDBITS];
1631
1632 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
1633 bits &= ~(1U << j);
1634
1635 fp = fp_get_noref_locked(p, fd);
1636 if (fp == NULL) {
1637 /*
1638 * If we abort because of a bad
1639 * fd, let the caller unwind...
1640 */
1641 proc_fdunlock(p);
1642 return EBADF;
1643 }
1644 if (sel_pass == SEL_SECONDPASS) {
1645 selunlinkfp(fp, selset);
1646 } else if (link.wqlh == NULL) {
1647 link = waitq_link_alloc(WQT_SELECT_SET);
1648 }
1649
1650 context.vc_ucred = fp->f_cred;
1651
1652 /* The select; set the bit, if true */
1653 if (fo_select(fp, flag[msk], s_data, &context)) {
1654 optr[fd / NFDBITS] |= (1U << (fd % NFDBITS));
1655 n++;
1656 }
1657 if (sel_pass == SEL_FIRSTPASS) {
1658 /*
1659 * Hook up the thread's waitq set either to
1660 * the fileproc structure, or to the global
1661 * conflict queue: but only on the first
1662 * select pass.
1663 */
1664 sellinkfp(fp, selset, &link);
1665 }
1666 nc++;
1667 }
1668 }
1669 }
1670 proc_fdunlock(p);
1671
1672 if (link.wqlh) {
1673 waitq_link_free(WQT_SELECT_SET, link);
1674 }
1675
1676 *retval = n;
1677 return 0;
1678 }
1679
1680 static int poll_callback(struct kevent_qos_s *, kevent_ctx_t);
1681
1682 int
poll(struct proc * p,struct poll_args * uap,int32_t * retval)1683 poll(struct proc *p, struct poll_args *uap, int32_t *retval)
1684 {
1685 __pthread_testcancel(1);
1686 return poll_nocancel(p, (struct poll_nocancel_args *)uap, retval);
1687 }
1688
1689
1690 int
poll_nocancel(struct proc * p,struct poll_nocancel_args * uap,int32_t * retval)1691 poll_nocancel(struct proc *p, struct poll_nocancel_args *uap, int32_t *retval)
1692 {
1693 struct pollfd *fds = NULL;
1694 struct kqueue *kq = NULL;
1695 int error = 0;
1696 u_int nfds = uap->nfds;
1697 u_int rfds = 0;
1698 rlim_t nofile = proc_limitgetcur(p, RLIMIT_NOFILE);
1699 size_t ni = nfds * sizeof(struct pollfd);
1700
1701 /*
1702 * This is kinda bogus. We have fd limits, but that is not
1703 * really related to the size of the pollfd array. Make sure
1704 * we let the process use at least FD_SETSIZE entries and at
1705 * least enough for the current limits. We want to be reasonably
1706 * safe, but not overly restrictive.
1707 */
1708 if (nfds > OPEN_MAX ||
1709 (nfds > nofile && (proc_suser(p) || nfds > FD_SETSIZE))) {
1710 return EINVAL;
1711 }
1712
1713 kq = kqueue_alloc(p);
1714 if (kq == NULL) {
1715 return EAGAIN;
1716 }
1717
1718 if (nfds) {
1719 fds = (struct pollfd *)kalloc_data(ni, Z_WAITOK);
1720 if (NULL == fds) {
1721 error = EAGAIN;
1722 goto out;
1723 }
1724
1725 error = copyin(uap->fds, fds, nfds * sizeof(struct pollfd));
1726 if (error) {
1727 goto out;
1728 }
1729 }
1730
1731 /* JMM - all this P_SELECT stuff is bogus */
1732 OSBitOrAtomic(P_SELECT, &p->p_flag);
1733 for (u_int i = 0; i < nfds; i++) {
1734 short events = fds[i].events;
1735 __assert_only int rc;
1736
1737 /* per spec, ignore fd values below zero */
1738 if (fds[i].fd < 0) {
1739 fds[i].revents = 0;
1740 continue;
1741 }
1742
1743 /* convert the poll event into a kqueue kevent */
1744 struct kevent_qos_s kev = {
1745 .ident = fds[i].fd,
1746 .flags = EV_ADD | EV_ONESHOT | EV_POLL,
1747 .udata = i, /* Index into pollfd array */
1748 };
1749
1750 /* Handle input events */
1751 if (events & (POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND | POLLHUP)) {
1752 kev.filter = EVFILT_READ;
1753 if (events & (POLLPRI | POLLRDBAND)) {
1754 kev.flags |= EV_OOBAND;
1755 }
1756 rc = kevent_register(kq, &kev, NULL);
1757 assert((rc & FILTER_REGISTER_WAIT) == 0);
1758 }
1759
1760 /* Handle output events */
1761 if ((kev.flags & EV_ERROR) == 0 &&
1762 (events & (POLLOUT | POLLWRNORM | POLLWRBAND))) {
1763 kev.filter = EVFILT_WRITE;
1764 rc = kevent_register(kq, &kev, NULL);
1765 assert((rc & FILTER_REGISTER_WAIT) == 0);
1766 }
1767
1768 /* Handle BSD extension vnode events */
1769 if ((kev.flags & EV_ERROR) == 0 &&
1770 (events & (POLLEXTEND | POLLATTRIB | POLLNLINK | POLLWRITE))) {
1771 kev.filter = EVFILT_VNODE;
1772 kev.fflags = 0;
1773 if (events & POLLEXTEND) {
1774 kev.fflags |= NOTE_EXTEND;
1775 }
1776 if (events & POLLATTRIB) {
1777 kev.fflags |= NOTE_ATTRIB;
1778 }
1779 if (events & POLLNLINK) {
1780 kev.fflags |= NOTE_LINK;
1781 }
1782 if (events & POLLWRITE) {
1783 kev.fflags |= NOTE_WRITE;
1784 }
1785 rc = kevent_register(kq, &kev, NULL);
1786 assert((rc & FILTER_REGISTER_WAIT) == 0);
1787 }
1788
1789 if (kev.flags & EV_ERROR) {
1790 fds[i].revents = POLLNVAL;
1791 rfds++;
1792 } else {
1793 fds[i].revents = 0;
1794 }
1795 }
1796
1797 /*
1798 * Did we have any trouble registering?
1799 * If user space passed 0 FDs, then respect any timeout value passed.
1800 * This is an extremely inefficient sleep. If user space passed one or
1801 * more FDs, and we had trouble registering _all_ of them, then bail
1802 * out. If a subset of the provided FDs failed to register, then we
1803 * will still call the kqueue_scan function.
1804 */
1805 if (nfds && (rfds == nfds)) {
1806 goto done;
1807 }
1808
1809 /* scan for, and possibly wait for, the kevents to trigger */
1810 kevent_ctx_t kectx = kevent_get_context(current_thread());
1811 *kectx = (struct kevent_ctx_s){
1812 .kec_process_noutputs = rfds,
1813 .kec_process_flags = KEVENT_FLAG_POLL,
1814 .kec_deadline = 0, /* wait forever */
1815 .kec_poll_fds = fds,
1816 };
1817
1818 /*
1819 * If any events have trouble registering, an event has fired and we
1820 * shouldn't wait for events in kqueue_scan.
1821 */
1822 if (rfds) {
1823 kectx->kec_process_flags |= KEVENT_FLAG_IMMEDIATE;
1824 } else if (uap->timeout != -1) {
1825 clock_interval_to_deadline(uap->timeout, NSEC_PER_MSEC,
1826 &kectx->kec_deadline);
1827 }
1828
1829 error = kqueue_scan(kq, kectx->kec_process_flags, kectx, poll_callback);
1830 rfds = kectx->kec_process_noutputs;
1831
1832 done:
1833 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1834 /* poll is not restarted after signals... */
1835 if (error == ERESTART) {
1836 error = EINTR;
1837 }
1838 if (error == 0) {
1839 error = copyout(fds, uap->fds, nfds * sizeof(struct pollfd));
1840 *retval = rfds;
1841 }
1842
1843 out:
1844 kfree_data(fds, ni);
1845
1846 kqueue_dealloc(kq);
1847 return error;
1848 }
1849
1850 static int
poll_callback(struct kevent_qos_s * kevp,kevent_ctx_t kectx)1851 poll_callback(struct kevent_qos_s *kevp, kevent_ctx_t kectx)
1852 {
1853 assert(kectx->kec_process_flags & KEVENT_FLAG_POLL);
1854 struct pollfd *fds = &kectx->kec_poll_fds[kevp->udata];
1855
1856 short prev_revents = fds->revents;
1857 short mask = 0;
1858
1859 /* convert the results back into revents */
1860 if (kevp->flags & EV_EOF) {
1861 fds->revents |= POLLHUP;
1862 }
1863 if (kevp->flags & EV_ERROR) {
1864 fds->revents |= POLLERR;
1865 }
1866
1867 switch (kevp->filter) {
1868 case EVFILT_READ:
1869 if (fds->revents & POLLHUP) {
1870 mask = (POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND);
1871 } else {
1872 mask = (POLLIN | POLLRDNORM);
1873 if (kevp->flags & EV_OOBAND) {
1874 mask |= (POLLPRI | POLLRDBAND);
1875 }
1876 }
1877 fds->revents |= (fds->events & mask);
1878 break;
1879
1880 case EVFILT_WRITE:
1881 if (!(fds->revents & POLLHUP)) {
1882 fds->revents |= (fds->events & (POLLOUT | POLLWRNORM | POLLWRBAND));
1883 }
1884 break;
1885
1886 case EVFILT_VNODE:
1887 if (kevp->fflags & NOTE_EXTEND) {
1888 fds->revents |= (fds->events & POLLEXTEND);
1889 }
1890 if (kevp->fflags & NOTE_ATTRIB) {
1891 fds->revents |= (fds->events & POLLATTRIB);
1892 }
1893 if (kevp->fflags & NOTE_LINK) {
1894 fds->revents |= (fds->events & POLLNLINK);
1895 }
1896 if (kevp->fflags & NOTE_WRITE) {
1897 fds->revents |= (fds->events & POLLWRITE);
1898 }
1899 break;
1900 }
1901
1902 if (fds->revents != 0 && prev_revents == 0) {
1903 kectx->kec_process_noutputs++;
1904 }
1905
1906 return 0;
1907 }
1908
1909 int
seltrue(__unused dev_t dev,__unused int flag,__unused struct proc * p)1910 seltrue(__unused dev_t dev, __unused int flag, __unused struct proc *p)
1911 {
1912 return 1;
1913 }
1914
1915 /*
1916 * selcount
1917 *
1918 * Count the number of bits set in the input bit vector, and establish an
1919 * outstanding fp->fp_iocount for each of the descriptors which will be in
1920 * use in the select operation.
1921 *
1922 * Parameters: p The process doing the select
1923 * ibits The input bit vector
1924 * nfd The number of fd's in the vector
1925 * countp Pointer to where to store the bit count
1926 *
1927 * Returns: 0 Success
1928 * EIO Bad per process open file table
1929 * EBADF One of the bits in the input bit vector
1930 * references an invalid fd
1931 *
1932 * Implicit: *countp (modified) Count of fd's
1933 *
1934 * Notes: This function is the first pass under the proc_fdlock() that
1935 * permits us to recognize invalid descriptors in the bit vector;
1936 * the may, however, not remain valid through the drop and
1937 * later reacquisition of the proc_fdlock().
1938 */
1939 static int
selcount(struct proc * p,u_int32_t * ibits,int nfd,int * countp)1940 selcount(struct proc *p, u_int32_t *ibits, int nfd, int *countp)
1941 {
1942 int msk, i, j, fd;
1943 u_int32_t bits;
1944 struct fileproc *fp;
1945 int n = 0;
1946 u_int32_t *iptr;
1947 u_int nw;
1948 int error = 0;
1949 int need_wakeup = 0;
1950
1951 nw = howmany(nfd, NFDBITS);
1952
1953 proc_fdlock(p);
1954 for (msk = 0; msk < 3; msk++) {
1955 iptr = (u_int32_t *)&ibits[msk * nw];
1956 for (i = 0; i < nfd; i += NFDBITS) {
1957 bits = iptr[i / NFDBITS];
1958 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
1959 bits &= ~(1U << j);
1960
1961 fp = fp_get_noref_locked(p, fd);
1962 if (fp == NULL) {
1963 *countp = 0;
1964 error = EBADF;
1965 goto bad;
1966 }
1967 os_ref_retain_locked(&fp->fp_iocount);
1968 n++;
1969 }
1970 }
1971 }
1972 proc_fdunlock(p);
1973
1974 *countp = n;
1975 return 0;
1976
1977 bad:
1978 if (n == 0) {
1979 goto out;
1980 }
1981 /* Ignore error return; it's already EBADF */
1982 (void)seldrop_locked(p, ibits, nfd, n, &need_wakeup);
1983
1984 out:
1985 proc_fdunlock(p);
1986 if (need_wakeup) {
1987 wakeup(&p->p_fd.fd_fpdrainwait);
1988 }
1989 return error;
1990 }
1991
1992
1993 /*
1994 * seldrop_locked
1995 *
1996 * Drop outstanding wait queue references set up during selscan(); drop the
1997 * outstanding per fileproc fp_iocount picked up during the selcount().
1998 *
1999 * Parameters: p Process performing the select
2000 * ibits Input bit bector of fd's
2001 * nfd Number of fd's
2002 * lim Limit to number of vector entries to
2003 * consider, or -1 for "all"
2004 * inselect True if
2005 * need_wakeup Pointer to flag to set to do a wakeup
2006 * if f_iocont on any descriptor goes to 0
2007 *
2008 * Returns: 0 Success
2009 * EBADF One or more fds in the bit vector
2010 * were invalid, but the rest
2011 * were successfully dropped
2012 *
2013 * Notes: An fd make become bad while the proc_fdlock() is not held,
2014 * if a multithreaded application closes the fd out from under
2015 * the in progress select. In this case, we still have to
2016 * clean up after the set up on the remaining fds.
2017 */
2018 static int
seldrop_locked(struct proc * p,u_int32_t * ibits,int nfd,int lim,int * need_wakeup)2019 seldrop_locked(struct proc *p, u_int32_t *ibits, int nfd, int lim, int *need_wakeup)
2020 {
2021 int msk, i, j, nc, fd;
2022 u_int32_t bits;
2023 struct fileproc *fp;
2024 u_int32_t *iptr;
2025 u_int nw;
2026 int error = 0;
2027 uthread_t uth = current_uthread();
2028 struct _select_data *seldata;
2029
2030 *need_wakeup = 0;
2031
2032 nw = howmany(nfd, NFDBITS);
2033 seldata = &uth->uu_save.uus_select_data;
2034
2035 nc = 0;
2036 for (msk = 0; msk < 3; msk++) {
2037 iptr = (u_int32_t *)&ibits[msk * nw];
2038 for (i = 0; i < nfd; i += NFDBITS) {
2039 bits = iptr[i / NFDBITS];
2040 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
2041 bits &= ~(1U << j);
2042 /*
2043 * If we've already dropped as many as were
2044 * counted/scanned, then we are done.
2045 */
2046 if (nc >= lim) {
2047 goto done;
2048 }
2049
2050 /*
2051 * We took an I/O reference in selcount,
2052 * so the fp can't possibly be NULL.
2053 */
2054 fp = fp_get_noref_locked_with_iocount(p, fd);
2055 selunlinkfp(fp, uth->uu_selset);
2056
2057 nc++;
2058
2059 const os_ref_count_t refc = os_ref_release_locked(&fp->fp_iocount);
2060 if (0 == refc) {
2061 panic("fp_iocount overdecrement!");
2062 }
2063
2064 if (1 == refc) {
2065 /*
2066 * The last iocount is responsible for clearing
2067 * selconfict flag - even if we didn't set it -
2068 * and is also responsible for waking up anyone
2069 * waiting on iocounts to drain.
2070 */
2071 if (fp->fp_flags & FP_SELCONFLICT) {
2072 fp->fp_flags &= ~FP_SELCONFLICT;
2073 }
2074 if (p->p_fd.fd_fpdrainwait) {
2075 p->p_fd.fd_fpdrainwait = 0;
2076 *need_wakeup = 1;
2077 }
2078 }
2079 }
2080 }
2081 }
2082 done:
2083 return error;
2084 }
2085
2086
2087 static int
seldrop(struct proc * p,u_int32_t * ibits,int nfd,int lim)2088 seldrop(struct proc *p, u_int32_t *ibits, int nfd, int lim)
2089 {
2090 int error;
2091 int need_wakeup = 0;
2092
2093 proc_fdlock(p);
2094 error = seldrop_locked(p, ibits, nfd, lim, &need_wakeup);
2095 proc_fdunlock(p);
2096 if (need_wakeup) {
2097 wakeup(&p->p_fd.fd_fpdrainwait);
2098 }
2099 return error;
2100 }
2101
2102 /*
2103 * Record a select request.
2104 */
2105 void
selrecord(__unused struct proc * selector,struct selinfo * sip,void * s_data)2106 selrecord(__unused struct proc *selector, struct selinfo *sip, void *s_data)
2107 {
2108 struct select_set *selset = current_uthread()->uu_selset;
2109
2110 /* do not record if this is second pass of select */
2111 if (!s_data) {
2112 return;
2113 }
2114
2115 if (selset == SELSPEC_RECORD_MARKER) {
2116 /*
2117 * The kevent subsystem is trying to sniff
2118 * the selinfo::si_note to attach to.
2119 */
2120 ((selspec_record_hook_t)s_data)(sip);
2121 } else {
2122 waitq_link_t *linkp = s_data;
2123
2124 if (!waitq_is_valid(&sip->si_waitq)) {
2125 waitq_init(&sip->si_waitq, WQT_SELECT, SYNC_POLICY_FIFO);
2126 }
2127
2128 /* note: this checks for pre-existing linkage */
2129 select_set_link(&sip->si_waitq, selset, linkp);
2130 }
2131 }
2132
2133 static void
selwakeup_internal(struct selinfo * sip,long hint,wait_result_t wr)2134 selwakeup_internal(struct selinfo *sip, long hint, wait_result_t wr)
2135 {
2136 if (sip->si_flags & SI_SELSPEC) {
2137 /*
2138 * The "primitive" lock is held.
2139 * The knote lock is not held.
2140 *
2141 * All knotes will transition their kn_hook to NULL and we will
2142 * reeinitialize the primitive's klist
2143 */
2144 lck_spin_lock(&selspec_lock);
2145 knote(&sip->si_note, hint, /*autodetach=*/ true);
2146 lck_spin_unlock(&selspec_lock);
2147 sip->si_flags &= ~SI_SELSPEC;
2148 }
2149
2150 /*
2151 * After selrecord() has been called, selinfo owners must call
2152 * at least one of selwakeup() or selthreadclear().
2153 *
2154 * Use this opportunity to deinit the waitq
2155 * so that all linkages are garbage collected
2156 * in a combined wakeup-all + unlink + deinit call.
2157 */
2158 select_waitq_wakeup_and_deinit(&sip->si_waitq, NO_EVENT64, wr);
2159 }
2160
2161
2162 void
selwakeup(struct selinfo * sip)2163 selwakeup(struct selinfo *sip)
2164 {
2165 selwakeup_internal(sip, 0, THREAD_AWAKENED);
2166 }
2167
2168 void
selthreadclear(struct selinfo * sip)2169 selthreadclear(struct selinfo *sip)
2170 {
2171 selwakeup_internal(sip, NOTE_REVOKE, THREAD_RESTART);
2172 }
2173
2174
2175 /*
2176 * gethostuuid
2177 *
2178 * Description: Get the host UUID from IOKit and return it to user space.
2179 *
2180 * Parameters: uuid_buf Pointer to buffer to receive UUID
2181 * timeout Timespec for timout
2182 *
2183 * Returns: 0 Success
2184 * EWOULDBLOCK Timeout is too short
2185 * copyout:EFAULT Bad user buffer
2186 * mac_system_check_info:EPERM Client not allowed to perform this operation
2187 *
2188 * Notes: A timeout seems redundant, since if it's tolerable to not
2189 * have a system UUID in hand, then why ask for one?
2190 */
2191 int
gethostuuid(struct proc * p,struct gethostuuid_args * uap,__unused int32_t * retval)2192 gethostuuid(struct proc *p, struct gethostuuid_args *uap, __unused int32_t *retval)
2193 {
2194 kern_return_t kret;
2195 int error;
2196 mach_timespec_t mach_ts; /* for IOKit call */
2197 __darwin_uuid_t uuid_kern = {}; /* for IOKit call */
2198
2199 /* Check entitlement */
2200 if (!IOCurrentTaskHasEntitlement("com.apple.private.getprivatesysid")) {
2201 #if !defined(XNU_TARGET_OS_OSX)
2202 #if CONFIG_MACF
2203 if ((error = mac_system_check_info(kauth_cred_get(), "hw.uuid")) != 0) {
2204 /* EPERM invokes userspace upcall if present */
2205 return error;
2206 }
2207 #endif
2208 #endif
2209 }
2210
2211 /* Convert the 32/64 bit timespec into a mach_timespec_t */
2212 if (proc_is64bit(p)) {
2213 struct user64_timespec ts;
2214 error = copyin(uap->timeoutp, &ts, sizeof(ts));
2215 if (error) {
2216 return error;
2217 }
2218 mach_ts.tv_sec = (unsigned int)ts.tv_sec;
2219 mach_ts.tv_nsec = (clock_res_t)ts.tv_nsec;
2220 } else {
2221 struct user32_timespec ts;
2222 error = copyin(uap->timeoutp, &ts, sizeof(ts));
2223 if (error) {
2224 return error;
2225 }
2226 mach_ts.tv_sec = ts.tv_sec;
2227 mach_ts.tv_nsec = ts.tv_nsec;
2228 }
2229
2230 /* Call IOKit with the stack buffer to get the UUID */
2231 kret = IOBSDGetPlatformUUID(uuid_kern, mach_ts);
2232
2233 /*
2234 * If we get it, copy out the data to the user buffer; note that a
2235 * uuid_t is an array of characters, so this is size invariant for
2236 * 32 vs. 64 bit.
2237 */
2238 if (kret == KERN_SUCCESS) {
2239 error = copyout(uuid_kern, uap->uuid_buf, sizeof(uuid_kern));
2240 } else {
2241 error = EWOULDBLOCK;
2242 }
2243
2244 return error;
2245 }
2246
2247 /*
2248 * ledger
2249 *
2250 * Description: Omnibus system call for ledger operations
2251 */
2252 int
ledger(struct proc * p,struct ledger_args * args,__unused int32_t * retval)2253 ledger(struct proc *p, struct ledger_args *args, __unused int32_t *retval)
2254 {
2255 #if !CONFIG_MACF
2256 #pragma unused(p)
2257 #endif
2258 int rval, pid, len, error;
2259 #ifdef LEDGER_DEBUG
2260 struct ledger_limit_args lla;
2261 #endif
2262 task_t task;
2263 proc_t proc;
2264
2265 /* Finish copying in the necessary args before taking the proc lock */
2266 error = 0;
2267 len = 0;
2268 if (args->cmd == LEDGER_ENTRY_INFO) {
2269 error = copyin(args->arg3, (char *)&len, sizeof(len));
2270 } else if (args->cmd == LEDGER_TEMPLATE_INFO) {
2271 error = copyin(args->arg2, (char *)&len, sizeof(len));
2272 } else if (args->cmd == LEDGER_LIMIT)
2273 #ifdef LEDGER_DEBUG
2274 { error = copyin(args->arg2, (char *)&lla, sizeof(lla));}
2275 #else
2276 { return EINVAL; }
2277 #endif
2278 else if ((args->cmd < 0) || (args->cmd > LEDGER_MAX_CMD)) {
2279 return EINVAL;
2280 }
2281
2282 if (error) {
2283 return error;
2284 }
2285 if (len < 0) {
2286 return EINVAL;
2287 }
2288
2289 rval = 0;
2290 if (args->cmd != LEDGER_TEMPLATE_INFO) {
2291 pid = (int)args->arg1;
2292 proc = proc_find(pid);
2293 if (proc == NULL) {
2294 return ESRCH;
2295 }
2296
2297 #if CONFIG_MACF
2298 error = mac_proc_check_ledger(p, proc, args->cmd);
2299 if (error) {
2300 proc_rele(proc);
2301 return error;
2302 }
2303 #endif
2304
2305 task = proc_task(proc);
2306 }
2307
2308 switch (args->cmd) {
2309 #ifdef LEDGER_DEBUG
2310 case LEDGER_LIMIT: {
2311 if (!kauth_cred_issuser(kauth_cred_get())) {
2312 rval = EPERM;
2313 }
2314 rval = ledger_limit(task, &lla);
2315 proc_rele(proc);
2316 break;
2317 }
2318 #endif
2319 case LEDGER_INFO: {
2320 struct ledger_info info = {};
2321
2322 rval = ledger_info(task, &info);
2323 proc_rele(proc);
2324 if (rval == 0) {
2325 rval = copyout(&info, args->arg2,
2326 sizeof(info));
2327 }
2328 break;
2329 }
2330
2331 case LEDGER_ENTRY_INFO: {
2332 void *buf;
2333 int sz;
2334
2335 #if CONFIG_MEMORYSTATUS
2336 task_ledger_settle_dirty_time(task);
2337 #endif /* CONFIG_MEMORYSTATUS */
2338
2339 rval = ledger_get_task_entry_info_multiple(task, &buf, &len);
2340 proc_rele(proc);
2341 if ((rval == 0) && (len >= 0)) {
2342 sz = len * sizeof(struct ledger_entry_info);
2343 rval = copyout(buf, args->arg2, sz);
2344 kfree_data(buf, sz);
2345 }
2346 if (rval == 0) {
2347 rval = copyout(&len, args->arg3, sizeof(len));
2348 }
2349 break;
2350 }
2351
2352 case LEDGER_TEMPLATE_INFO: {
2353 void *buf;
2354 int sz;
2355
2356 rval = ledger_template_info(&buf, &len);
2357 if ((rval == 0) && (len >= 0)) {
2358 sz = len * sizeof(struct ledger_template_info);
2359 rval = copyout(buf, args->arg1, sz);
2360 kfree_data(buf, sz);
2361 }
2362 if (rval == 0) {
2363 rval = copyout(&len, args->arg2, sizeof(len));
2364 }
2365 break;
2366 }
2367
2368 default:
2369 panic("ledger syscall logic error -- command type %d", args->cmd);
2370 proc_rele(proc);
2371 rval = EINVAL;
2372 }
2373
2374 return rval;
2375 }
2376
2377 int
telemetry(__unused struct proc * p,struct telemetry_args * args,__unused int32_t * retval)2378 telemetry(__unused struct proc *p, struct telemetry_args *args, __unused int32_t *retval)
2379 {
2380 int error = 0;
2381
2382 switch (args->cmd) {
2383 #if CONFIG_TELEMETRY
2384 case TELEMETRY_CMD_TIMER_EVENT:
2385 error = telemetry_timer_event(args->deadline, args->interval, args->leeway);
2386 break;
2387 case TELEMETRY_CMD_PMI_SETUP:
2388 error = telemetry_pmi_setup((enum telemetry_pmi)args->deadline, args->interval);
2389 break;
2390 #endif /* CONFIG_TELEMETRY */
2391 case TELEMETRY_CMD_VOUCHER_NAME:
2392 if (thread_set_voucher_name((mach_port_name_t)args->deadline)) {
2393 error = EINVAL;
2394 }
2395 break;
2396
2397 default:
2398 error = EINVAL;
2399 break;
2400 }
2401
2402 return error;
2403 }
2404
2405 /*
2406 * Logging
2407 *
2408 * Description: syscall to access kernel logging from userspace
2409 *
2410 * Args:
2411 * tag - used for syncing with userspace on the version.
2412 * flags - flags used by the syscall.
2413 * buffer - userspace address of string to copy.
2414 * size - size of buffer.
2415 */
2416 int
log_data(__unused struct proc * p,struct log_data_args * args,int * retval)2417 log_data(__unused struct proc *p, struct log_data_args *args, int *retval)
2418 {
2419 unsigned int tag = args->tag;
2420 unsigned int flags = args->flags;
2421 user_addr_t buffer = args->buffer;
2422 unsigned int size = args->size;
2423 int ret = 0;
2424 *retval = 0;
2425
2426 /* Only DEXTs are suppose to use this syscall. */
2427 if (!task_is_driver(current_task())) {
2428 return EPERM;
2429 }
2430
2431 /*
2432 * Tag synchronize the syscall version with userspace.
2433 * Tag == 0 => flags == OS_LOG_TYPE
2434 */
2435 if (tag != 0) {
2436 return EINVAL;
2437 }
2438
2439 /*
2440 * OS_LOG_TYPE are defined in libkern/os/log.h
2441 * In userspace they are defined in libtrace/os/log.h
2442 */
2443 if (flags != OS_LOG_TYPE_DEFAULT &&
2444 flags != OS_LOG_TYPE_INFO &&
2445 flags != OS_LOG_TYPE_DEBUG &&
2446 flags != OS_LOG_TYPE_ERROR &&
2447 flags != OS_LOG_TYPE_FAULT) {
2448 return EINVAL;
2449 }
2450
2451 if (size == 0) {
2452 return EINVAL;
2453 }
2454
2455 /* truncate to OS_LOG_DATA_MAX_SIZE */
2456 if (size > OS_LOG_DATA_MAX_SIZE) {
2457 size = OS_LOG_DATA_MAX_SIZE;
2458 }
2459
2460 char *log_msg = (char *)kalloc_data(size, Z_WAITOK);
2461 if (!log_msg) {
2462 return ENOMEM;
2463 }
2464
2465 if (copyin(buffer, log_msg, size) != 0) {
2466 ret = EFAULT;
2467 goto out;
2468 }
2469 log_msg[size - 1] = '\0';
2470
2471 /*
2472 * This will log to dmesg and logd.
2473 * The call will fail if the current
2474 * process is not a driverKit process.
2475 */
2476 os_log_driverKit(&ret, OS_LOG_DEFAULT, (os_log_type_t)flags, "%s", log_msg);
2477
2478 out:
2479 if (log_msg != NULL) {
2480 kfree_data(log_msg, size);
2481 }
2482
2483 return ret;
2484 }
2485
2486 #if DEVELOPMENT || DEBUG
2487
2488 static int
2489 sysctl_mpsc_test_pingpong SYSCTL_HANDLER_ARGS
2490 {
2491 #pragma unused(oidp, arg1, arg2)
2492 uint64_t value = 0;
2493 int error;
2494
2495 error = SYSCTL_IN(req, &value, sizeof(value));
2496 if (error) {
2497 return error;
2498 }
2499
2500 if (error == 0 && req->newptr) {
2501 error = mpsc_test_pingpong(value, &value);
2502 if (error == 0) {
2503 error = SYSCTL_OUT(req, &value, sizeof(value));
2504 }
2505 }
2506
2507 return error;
2508 }
2509 SYSCTL_PROC(_kern, OID_AUTO, mpsc_test_pingpong, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
2510 0, 0, sysctl_mpsc_test_pingpong, "Q", "MPSC tests: pingpong");
2511
2512 #endif /* DEVELOPMENT || DEBUG */
2513
2514 /* Telemetry, microstackshots */
2515
2516 SYSCTL_NODE(_kern, OID_AUTO, microstackshot, CTLFLAG_RD | CTLFLAG_LOCKED, 0,
2517 "microstackshot info");
2518
2519 extern uint32_t telemetry_sample_rate;
2520 SYSCTL_UINT(_kern_microstackshot, OID_AUTO, interrupt_sample_rate,
2521 CTLFLAG_RD | CTLFLAG_LOCKED, &telemetry_sample_rate, 0,
2522 "interrupt-based sampling rate in Hz");
2523
2524 #if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES)
2525
2526 extern uint64_t mt_microstackshot_period;
2527 SYSCTL_QUAD(_kern_microstackshot, OID_AUTO, pmi_sample_period,
2528 CTLFLAG_RD | CTLFLAG_LOCKED, &mt_microstackshot_period,
2529 "PMI sampling rate");
2530 extern unsigned int mt_microstackshot_ctr;
2531 SYSCTL_UINT(_kern_microstackshot, OID_AUTO, pmi_sample_counter,
2532 CTLFLAG_RD | CTLFLAG_LOCKED, &mt_microstackshot_ctr, 0,
2533 "PMI counter");
2534
2535 #endif /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */
2536
2537 /*Remote Time api*/
2538 SYSCTL_NODE(_machdep, OID_AUTO, remotetime, CTLFLAG_RD | CTLFLAG_LOCKED, 0, "Remote time api");
2539
2540 #if DEVELOPMENT || DEBUG
2541 #if CONFIG_MACH_BRIDGE_SEND_TIME
2542 extern _Atomic uint32_t bt_init_flag;
2543 extern uint32_t mach_bridge_timer_enable(uint32_t, int);
2544
2545 SYSCTL_INT(_machdep_remotetime, OID_AUTO, bridge_timer_init_flag,
2546 CTLFLAG_RD | CTLFLAG_LOCKED, &bt_init_flag, 0, "");
2547
2548 static int sysctl_mach_bridge_timer_enable SYSCTL_HANDLER_ARGS
2549 {
2550 #pragma unused(oidp, arg1, arg2)
2551 uint32_t value = 0;
2552 int error = 0;
2553 /* User is querying buffer size */
2554 if (req->oldptr == USER_ADDR_NULL && req->newptr == USER_ADDR_NULL) {
2555 req->oldidx = sizeof(value);
2556 return 0;
2557 }
2558 if (os_atomic_load(&bt_init_flag, acquire)) {
2559 if (req->newptr) {
2560 int new_value = 0;
2561 error = SYSCTL_IN(req, &new_value, sizeof(new_value));
2562 if (error) {
2563 return error;
2564 }
2565 if (new_value == 0 || new_value == 1) {
2566 value = mach_bridge_timer_enable(new_value, 1);
2567 } else {
2568 return EPERM;
2569 }
2570 } else {
2571 value = mach_bridge_timer_enable(0, 0);
2572 }
2573 }
2574 error = SYSCTL_OUT(req, &value, sizeof(value));
2575 return error;
2576 }
2577
2578 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, bridge_timer_enable,
2579 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2580 0, 0, sysctl_mach_bridge_timer_enable, "I", "");
2581
2582 #endif /* CONFIG_MACH_BRIDGE_SEND_TIME */
2583
2584 static int sysctl_mach_bridge_remote_time SYSCTL_HANDLER_ARGS
2585 {
2586 #pragma unused(oidp, arg1, arg2)
2587 uint64_t ltime = 0, rtime = 0;
2588 if (req->oldptr == USER_ADDR_NULL) {
2589 req->oldidx = sizeof(rtime);
2590 return 0;
2591 }
2592 if (req->newptr) {
2593 int error = SYSCTL_IN(req, <ime, sizeof(ltime));
2594 if (error) {
2595 return error;
2596 }
2597 }
2598 rtime = mach_bridge_remote_time(ltime);
2599 return SYSCTL_OUT(req, &rtime, sizeof(rtime));
2600 }
2601 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, mach_bridge_remote_time,
2602 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
2603 0, 0, sysctl_mach_bridge_remote_time, "Q", "");
2604
2605 #endif /* DEVELOPMENT || DEBUG */
2606
2607 #if CONFIG_MACH_BRIDGE_RECV_TIME
2608 extern struct bt_params bt_params_get_latest(void);
2609
2610 static int sysctl_mach_bridge_conversion_params SYSCTL_HANDLER_ARGS
2611 {
2612 #pragma unused(oidp, arg1, arg2)
2613 struct bt_params params = {};
2614 if (req->oldptr == USER_ADDR_NULL) {
2615 req->oldidx = sizeof(struct bt_params);
2616 return 0;
2617 }
2618 if (req->newptr) {
2619 return EPERM;
2620 }
2621 params = bt_params_get_latest();
2622 return SYSCTL_OUT(req, ¶ms, MIN(sizeof(params), req->oldlen));
2623 }
2624
2625 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, conversion_params,
2626 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0,
2627 0, sysctl_mach_bridge_conversion_params, "S,bt_params", "");
2628
2629 #endif /* CONFIG_MACH_BRIDGE_RECV_TIME */
2630
2631 #if DEVELOPMENT || DEBUG
2632
2633 #include <pexpert/pexpert.h>
2634 extern int32_t sysctl_get_bound_cpuid(void);
2635 extern kern_return_t sysctl_thread_bind_cpuid(int32_t cpuid);
2636 static int
2637 sysctl_kern_sched_thread_bind_cpu SYSCTL_HANDLER_ARGS
2638 {
2639 #pragma unused(oidp, arg1, arg2)
2640
2641 /*
2642 * DO NOT remove this bootarg guard or make this non-development.
2643 * This kind of binding should only be used for tests and
2644 * experiments in a custom configuration, never shipping code.
2645 */
2646
2647 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2648 return ENOENT;
2649 }
2650
2651 int32_t cpuid = sysctl_get_bound_cpuid();
2652
2653 int32_t new_value;
2654 int changed;
2655 int error = sysctl_io_number(req, cpuid, sizeof(cpuid), &new_value, &changed);
2656 if (error) {
2657 return error;
2658 }
2659
2660 if (changed) {
2661 kern_return_t kr = sysctl_thread_bind_cpuid(new_value);
2662
2663 if (kr == KERN_NOT_SUPPORTED) {
2664 return ENOTSUP;
2665 }
2666
2667 if (kr == KERN_INVALID_VALUE) {
2668 return ERANGE;
2669 }
2670 }
2671
2672 return error;
2673 }
2674
2675 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cpu, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2676 0, 0, sysctl_kern_sched_thread_bind_cpu, "I", "");
2677
2678 #if __AMP__
2679 extern char sysctl_get_bound_cluster_type(void);
2680 extern void sysctl_thread_bind_cluster_type(char cluster_type);
2681 static int
2682 sysctl_kern_sched_thread_bind_cluster_type SYSCTL_HANDLER_ARGS
2683 {
2684 #pragma unused(oidp, arg1, arg2)
2685 char buff[4];
2686
2687 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2688 return ENOENT;
2689 }
2690
2691 int error = SYSCTL_IN(req, buff, 1);
2692 if (error) {
2693 return error;
2694 }
2695 char cluster_type = buff[0];
2696
2697 if (!req->newptr) {
2698 goto out;
2699 }
2700
2701 if (cluster_type != 'E' &&
2702 cluster_type != 'e' &&
2703 cluster_type != 'P' &&
2704 cluster_type != 'p') {
2705 return EINVAL;
2706 }
2707
2708 sysctl_thread_bind_cluster_type(cluster_type);
2709
2710 out:
2711 cluster_type = sysctl_get_bound_cluster_type();
2712 buff[0] = cluster_type;
2713
2714 return SYSCTL_OUT(req, buff, 1);
2715 }
2716
2717 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cluster_type, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
2718 0, 0, sysctl_kern_sched_thread_bind_cluster_type, "A", "");
2719
2720 extern char sysctl_get_task_cluster_type(void);
2721 extern void sysctl_task_set_cluster_type(char cluster_type);
2722 static int
2723 sysctl_kern_sched_task_set_cluster_type SYSCTL_HANDLER_ARGS
2724 {
2725 #pragma unused(oidp, arg1, arg2)
2726 char buff[4];
2727
2728 if (!PE_parse_boot_argn("enable_skstsct", NULL, 0)) {
2729 return ENOENT;
2730 }
2731
2732 int error = SYSCTL_IN(req, buff, 1);
2733 if (error) {
2734 return error;
2735 }
2736 char cluster_type = buff[0];
2737
2738 if (!req->newptr) {
2739 goto out;
2740 }
2741
2742 if (cluster_type != 'E' &&
2743 cluster_type != 'e' &&
2744 cluster_type != 'P' &&
2745 cluster_type != 'p') {
2746 return EINVAL;
2747 }
2748
2749 sysctl_task_set_cluster_type(cluster_type);
2750 out:
2751 cluster_type = sysctl_get_task_cluster_type();
2752 buff[0] = cluster_type;
2753
2754 return SYSCTL_OUT(req, buff, 1);
2755 }
2756
2757 SYSCTL_PROC(_kern, OID_AUTO, sched_task_set_cluster_type, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
2758 0, 0, sysctl_kern_sched_task_set_cluster_type, "A", "");
2759
2760 extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options);
2761 extern uint32_t thread_bound_cluster_id(thread_t);
2762 static int
2763 sysctl_kern_sched_thread_bind_cluster_id SYSCTL_HANDLER_ARGS
2764 {
2765 #pragma unused(oidp, arg1, arg2)
2766 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2767 return ENOENT;
2768 }
2769
2770 thread_t self = current_thread();
2771 int32_t cluster_id = thread_bound_cluster_id(self);
2772 int32_t new_value;
2773 int changed;
2774 int error = sysctl_io_number(req, cluster_id, sizeof(cluster_id), &new_value, &changed);
2775 if (error) {
2776 return error;
2777 }
2778
2779 if (changed) {
2780 /*
2781 * This sysctl binds the thread to the cluster without any flags, which
2782 * means it will be hard bound and not check eligibility.
2783 */
2784 kern_return_t kr = thread_bind_cluster_id(self, new_value, 0);
2785 if (kr == KERN_INVALID_VALUE) {
2786 return ERANGE;
2787 }
2788
2789 if (kr != KERN_SUCCESS) {
2790 return EINVAL;
2791 }
2792 }
2793
2794 return error;
2795 }
2796
2797 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cluster_id, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2798 0, 0, sysctl_kern_sched_thread_bind_cluster_id, "I", "");
2799
2800 #if CONFIG_SCHED_EDGE
2801
2802 extern int sched_edge_restrict_ut;
2803 SYSCTL_INT(_kern, OID_AUTO, sched_edge_restrict_ut, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_restrict_ut, 0, "Edge Scheduler Restrict UT Threads");
2804 extern int sched_edge_restrict_bg;
2805 SYSCTL_INT(_kern, OID_AUTO, sched_edge_restrict_bg, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_restrict_ut, 0, "Edge Scheduler Restrict BG Threads");
2806 extern int sched_edge_migrate_ipi_immediate;
2807 SYSCTL_INT(_kern, OID_AUTO, sched_edge_migrate_ipi_immediate, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_migrate_ipi_immediate, 0, "Edge Scheduler uses immediate IPIs for migration event based on execution latency");
2808
2809 #endif /* CONFIG_SCHED_EDGE */
2810
2811 #endif /* __AMP__ */
2812
2813 #if SCHED_HYGIENE_DEBUG
2814
2815 SYSCTL_QUAD(_kern, OID_AUTO, interrupt_masked_threshold_mt, CTLFLAG_RW | CTLFLAG_LOCKED,
2816 &interrupt_masked_timeout,
2817 "Interrupt masked duration after which a tracepoint is emitted or the device panics (in mach timebase units)");
2818
2819 SYSCTL_INT(_kern, OID_AUTO, interrupt_masked_debug_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
2820 &interrupt_masked_debug_mode, 0,
2821 "Enable interrupt masked tracing or panic (0: off, 1: trace, 2: panic)");
2822
2823 SYSCTL_QUAD(_kern, OID_AUTO, sched_preemption_disable_threshold_mt, CTLFLAG_RW | CTLFLAG_LOCKED,
2824 &sched_preemption_disable_threshold_mt,
2825 "Preemption disablement duration after which a tracepoint is emitted or the device panics (in mach timebase units)");
2826
2827 SYSCTL_INT(_kern, OID_AUTO, sched_preemption_disable_debug_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
2828 &sched_preemption_disable_debug_mode, 0,
2829 "Enable preemption disablement tracing or panic (0: off, 1: trace, 2: panic)");
2830
2831 static int
sysctl_sched_preemption_disable_stats(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)2832 sysctl_sched_preemption_disable_stats(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
2833 {
2834 extern unsigned int preemption_disable_get_max_durations(uint64_t *durations, size_t count);
2835 extern void preemption_disable_reset_max_durations(void);
2836
2837 uint64_t stats[MAX_CPUS]; // maximum per CPU
2838
2839 unsigned int ncpus = preemption_disable_get_max_durations(stats, MAX_CPUS);
2840 if (req->newlen > 0) {
2841 /* Reset when attempting to write to the sysctl. */
2842 preemption_disable_reset_max_durations();
2843 }
2844
2845 return sysctl_io_opaque(req, stats, ncpus * sizeof(uint64_t), NULL);
2846 }
2847
2848 SYSCTL_PROC(_kern, OID_AUTO, sched_preemption_disable_stats,
2849 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_LOCKED,
2850 0, 0, sysctl_sched_preemption_disable_stats, "I", "Preemption disablement statistics");
2851
2852 #endif /* SCHED_HYGIENE_DEBUG */
2853
2854 /* used for testing by exception_tests */
2855 extern uint32_t ipc_control_port_options;
2856 SYSCTL_INT(_kern, OID_AUTO, ipc_control_port_options,
2857 CTLFLAG_RD | CTLFLAG_LOCKED, &ipc_control_port_options, 0, "");
2858
2859 #endif /* DEVELOPMENT || DEBUG */
2860
2861 extern uint32_t task_exc_guard_default;
2862
2863 SYSCTL_INT(_kern, OID_AUTO, task_exc_guard_default,
2864 CTLFLAG_RD | CTLFLAG_LOCKED, &task_exc_guard_default, 0, "");
2865
2866
2867 static int
2868 sysctl_kern_tcsm_available SYSCTL_HANDLER_ARGS
2869 {
2870 #pragma unused(oidp, arg1, arg2)
2871 uint32_t value = machine_csv(CPUVN_CI) ? 1 : 0;
2872
2873 if (req->newptr) {
2874 return EINVAL;
2875 }
2876
2877 return SYSCTL_OUT(req, &value, sizeof(value));
2878 }
2879 SYSCTL_PROC(_kern, OID_AUTO, tcsm_available,
2880 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED | CTLFLAG_ANYBODY,
2881 0, 0, sysctl_kern_tcsm_available, "I", "");
2882
2883
2884 static int
2885 sysctl_kern_tcsm_enable SYSCTL_HANDLER_ARGS
2886 {
2887 #pragma unused(oidp, arg1, arg2)
2888 uint32_t soflags = 0;
2889 uint32_t old_value = thread_get_no_smt() ? 1 : 0;
2890
2891 int error = SYSCTL_IN(req, &soflags, sizeof(soflags));
2892 if (error) {
2893 return error;
2894 }
2895
2896 if (soflags && machine_csv(CPUVN_CI)) {
2897 thread_set_no_smt(true);
2898 machine_tecs(current_thread());
2899 }
2900
2901 return SYSCTL_OUT(req, &old_value, sizeof(old_value));
2902 }
2903 SYSCTL_PROC(_kern, OID_AUTO, tcsm_enable,
2904 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_MASKED | CTLFLAG_ANYBODY,
2905 0, 0, sysctl_kern_tcsm_enable, "I", "");
2906
2907 static int
2908 sysctl_kern_debug_get_preoslog SYSCTL_HANDLER_ARGS
2909 {
2910 #pragma unused(oidp, arg1, arg2)
2911 static bool oneshot_executed = false;
2912 size_t preoslog_size = 0;
2913 const char *preoslog = NULL;
2914 int ret = 0;
2915
2916 // DumpPanic passes a non-zero write value when it needs oneshot behaviour
2917 if (req->newptr != USER_ADDR_NULL) {
2918 uint8_t oneshot = 0;
2919 int error = SYSCTL_IN(req, &oneshot, sizeof(oneshot));
2920 if (error) {
2921 return error;
2922 }
2923
2924 if (oneshot) {
2925 if (!os_atomic_cmpxchg(&oneshot_executed, false, true, acq_rel)) {
2926 return EPERM;
2927 }
2928 }
2929 }
2930
2931 preoslog = sysctl_debug_get_preoslog(&preoslog_size);
2932 if (preoslog != NULL && preoslog_size == 0) {
2933 sysctl_debug_free_preoslog();
2934 return 0;
2935 }
2936
2937 if (preoslog == NULL || preoslog_size == 0) {
2938 return 0;
2939 }
2940
2941 if (req->oldptr == USER_ADDR_NULL) {
2942 req->oldidx = preoslog_size;
2943 return 0;
2944 }
2945
2946 ret = SYSCTL_OUT(req, preoslog, preoslog_size);
2947 sysctl_debug_free_preoslog();
2948 return ret;
2949 }
2950
2951 SYSCTL_PROC(_kern, OID_AUTO, preoslog, CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_LOCKED,
2952 0, 0, sysctl_kern_debug_get_preoslog, "-", "");
2953
2954 #if DEVELOPMENT || DEBUG
2955 extern void sysctl_task_set_no_smt(char no_smt);
2956 extern char sysctl_task_get_no_smt(void);
2957
2958 static int
2959 sysctl_kern_sched_task_set_no_smt SYSCTL_HANDLER_ARGS
2960 {
2961 #pragma unused(oidp, arg1, arg2)
2962 char buff[4];
2963
2964 int error = SYSCTL_IN(req, buff, 1);
2965 if (error) {
2966 return error;
2967 }
2968 char no_smt = buff[0];
2969
2970 if (!req->newptr) {
2971 goto out;
2972 }
2973
2974 sysctl_task_set_no_smt(no_smt);
2975 out:
2976 no_smt = sysctl_task_get_no_smt();
2977 buff[0] = no_smt;
2978
2979 return SYSCTL_OUT(req, buff, 1);
2980 }
2981
2982 SYSCTL_PROC(_kern, OID_AUTO, sched_task_set_no_smt, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY,
2983 0, 0, sysctl_kern_sched_task_set_no_smt, "A", "");
2984
2985 static int
sysctl_kern_sched_thread_set_no_smt(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)2986 sysctl_kern_sched_thread_set_no_smt(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
2987 {
2988 int new_value, changed;
2989 int old_value = thread_get_no_smt() ? 1 : 0;
2990 int error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
2991
2992 if (changed) {
2993 thread_set_no_smt(!!new_value);
2994 }
2995
2996 return error;
2997 }
2998
2999 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_set_no_smt,
3000 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY,
3001 0, 0, sysctl_kern_sched_thread_set_no_smt, "I", "");
3002
3003 #if CONFIG_SCHED_RT_ALLOW
3004
3005 #if DEVELOPMENT || DEBUG
3006 #define RT_ALLOW_CTLFLAGS CTLFLAG_RW
3007 #else
3008 #define RT_ALLOW_CTLFLAGS CTLFLAG_RD
3009 #endif /* DEVELOPMENT || DEBUG */
3010
3011 static int
sysctl_kern_rt_allow_limit_percent(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)3012 sysctl_kern_rt_allow_limit_percent(__unused struct sysctl_oid *oidp,
3013 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3014 {
3015 extern uint8_t rt_allow_limit_percent;
3016
3017 int new_value = 0;
3018 int old_value = rt_allow_limit_percent;
3019 int changed = 0;
3020
3021 int error = sysctl_io_number(req, old_value, sizeof(old_value),
3022 &new_value, &changed);
3023 if (error != 0) {
3024 return error;
3025 }
3026
3027 /* Only accept a percentage between 1 and 99 inclusive. */
3028 if (changed) {
3029 if (new_value >= 100 || new_value <= 0) {
3030 return EINVAL;
3031 }
3032
3033 rt_allow_limit_percent = (uint8_t)new_value;
3034 }
3035
3036 return 0;
3037 }
3038
3039 SYSCTL_PROC(_kern, OID_AUTO, rt_allow_limit_percent,
3040 RT_ALLOW_CTLFLAGS | CTLTYPE_INT | CTLFLAG_LOCKED,
3041 0, 0, sysctl_kern_rt_allow_limit_percent, "I", "");
3042
3043 static int
sysctl_kern_rt_allow_limit_interval_ms(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)3044 sysctl_kern_rt_allow_limit_interval_ms(__unused struct sysctl_oid *oidp,
3045 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3046 {
3047 extern uint16_t rt_allow_limit_interval_ms;
3048
3049 uint64_t new_value = 0;
3050 uint64_t old_value = rt_allow_limit_interval_ms;
3051 int changed = 0;
3052
3053 int error = sysctl_io_number(req, old_value, sizeof(old_value),
3054 &new_value, &changed);
3055 if (error != 0) {
3056 return error;
3057 }
3058
3059 /* Value is in ns. Must be at least 1ms. */
3060 if (changed) {
3061 if (new_value < 1 || new_value > UINT16_MAX) {
3062 return EINVAL;
3063 }
3064
3065 rt_allow_limit_interval_ms = (uint16_t)new_value;
3066 }
3067
3068 return 0;
3069 }
3070
3071 SYSCTL_PROC(_kern, OID_AUTO, rt_allow_limit_interval_ms,
3072 RT_ALLOW_CTLFLAGS | CTLTYPE_QUAD | CTLFLAG_LOCKED,
3073 0, 0, sysctl_kern_rt_allow_limit_interval_ms, "Q", "");
3074
3075 #endif /* CONFIG_SCHED_RT_ALLOW */
3076
3077
3078 static int
3079 sysctl_kern_task_set_filter_msg_flag SYSCTL_HANDLER_ARGS
3080 {
3081 #pragma unused(oidp, arg1, arg2)
3082 int new_value, changed;
3083 int old_value = task_get_filter_msg_flag(current_task()) ? 1 : 0;
3084 int error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
3085
3086 if (changed) {
3087 task_set_filter_msg_flag(current_task(), !!new_value);
3088 }
3089
3090 return error;
3091 }
3092
3093 SYSCTL_PROC(_kern, OID_AUTO, task_set_filter_msg_flag, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3094 0, 0, sysctl_kern_task_set_filter_msg_flag, "I", "");
3095
3096 #if CONFIG_PROC_RESOURCE_LIMITS
3097
3098 extern mach_port_name_t current_task_get_fatal_port_name(void);
3099
3100 static int
3101 sysctl_kern_task_get_fatal_port SYSCTL_HANDLER_ARGS
3102 {
3103 #pragma unused(oidp, arg1, arg2)
3104 int port = 0;
3105 int flag = 0;
3106
3107 if (req->oldptr == USER_ADDR_NULL) {
3108 req->oldidx = sizeof(mach_port_t);
3109 return 0;
3110 }
3111
3112 int error = SYSCTL_IN(req, &flag, sizeof(flag));
3113 if (error) {
3114 return error;
3115 }
3116
3117 if (flag == 1) {
3118 port = (int)current_task_get_fatal_port_name();
3119 }
3120 return SYSCTL_OUT(req, &port, sizeof(port));
3121 }
3122
3123 SYSCTL_PROC(_machdep, OID_AUTO, task_get_fatal_port, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3124 0, 0, sysctl_kern_task_get_fatal_port, "I", "");
3125
3126 #endif /* CONFIG_PROC_RESOURCE_LIMITS */
3127
3128 extern unsigned int ipc_entry_table_count_max(void);
3129
3130 static int
3131 sysctl_mach_max_port_table_size SYSCTL_HANDLER_ARGS
3132 {
3133 #pragma unused(oidp, arg1, arg2)
3134 int old_value = ipc_entry_table_count_max();
3135 int error = sysctl_io_number(req, old_value, sizeof(int), NULL, NULL);
3136
3137 return error;
3138 }
3139
3140 SYSCTL_PROC(_machdep, OID_AUTO, max_port_table_size, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3141 0, 0, sysctl_mach_max_port_table_size, "I", "");
3142
3143 #endif /* DEVELOPMENT || DEBUG */
3144
3145 #if defined(CONFIG_KDP_INTERACTIVE_DEBUGGING) && defined(CONFIG_KDP_COREDUMP_ENCRYPTION)
3146
3147 #define COREDUMP_ENCRYPTION_KEY_ENTITLEMENT "com.apple.private.coredump-encryption-key"
3148
3149 static int
3150 sysctl_coredump_encryption_key_update SYSCTL_HANDLER_ARGS
3151 {
3152 kern_return_t ret = KERN_SUCCESS;
3153 int error = 0;
3154 struct kdp_core_encryption_key_descriptor key_descriptor = {
3155 .kcekd_format = MACH_CORE_FILEHEADER_V2_FLAG_NEXT_COREFILE_KEY_FORMAT_NIST_P256,
3156 };
3157
3158 /* Need to be root and have entitlement */
3159 if (!kauth_cred_issuser(kauth_cred_get()) && !IOCurrentTaskHasEntitlement(COREDUMP_ENCRYPTION_KEY_ENTITLEMENT)) {
3160 return EPERM;
3161 }
3162
3163 // Sanity-check the given key length
3164 if (req->newlen > UINT16_MAX) {
3165 return EINVAL;
3166 }
3167
3168 // It is allowed for the caller to pass in a NULL buffer.
3169 // This indicates that they want us to forget about any public key we might have.
3170 if (req->newptr) {
3171 key_descriptor.kcekd_size = (uint16_t) req->newlen;
3172 key_descriptor.kcekd_key = kalloc_data(key_descriptor.kcekd_size, Z_WAITOK);
3173
3174 if (key_descriptor.kcekd_key == NULL) {
3175 return ENOMEM;
3176 }
3177
3178 error = SYSCTL_IN(req, key_descriptor.kcekd_key, key_descriptor.kcekd_size);
3179 if (error) {
3180 goto out;
3181 }
3182 }
3183
3184 ret = IOProvideCoreFileAccess(kdp_core_handle_new_encryption_key, (void *)&key_descriptor);
3185 if (KERN_SUCCESS != ret) {
3186 printf("Failed to handle the new encryption key. Error 0x%x", ret);
3187 error = EFAULT;
3188 }
3189
3190 out:
3191 kfree_data(key_descriptor.kcekd_key, key_descriptor.kcekd_size);
3192 return 0;
3193 }
3194
3195 SYSCTL_PROC(_kern, OID_AUTO, coredump_encryption_key, CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3196 0, 0, &sysctl_coredump_encryption_key_update, "-", "Set a new encryption key for coredumps");
3197
3198 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING && CONFIG_KDP_COREDUMP_ENCRYPTION*/
3199