1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1982, 1986, 1989, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95
67 */
68 /*
69 * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
70 * support for mandatory and extensible security protections. This notice
71 * is included in support of clause 2.2 (b) of the Apple Public License,
72 * Version 2.0.
73 */
74
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/filedesc.h>
78 #include <sys/ioctl.h>
79 #include <sys/file_internal.h>
80 #include <sys/proc_internal.h>
81 #include <sys/socketvar.h>
82 #include <sys/uio_internal.h>
83 #include <sys/kernel.h>
84 #include <sys/guarded.h>
85 #include <sys/stat.h>
86 #include <sys/malloc.h>
87 #include <sys/sysproto.h>
88
89 #include <sys/mount_internal.h>
90 #include <sys/protosw.h>
91 #include <sys/ev.h>
92 #include <sys/user.h>
93 #include <sys/kdebug.h>
94 #include <sys/poll.h>
95 #include <sys/event.h>
96 #include <sys/eventvar.h>
97 #include <sys/proc.h>
98 #include <sys/kauth.h>
99
100 #include <machine/smp.h>
101 #include <mach/mach_types.h>
102 #include <kern/kern_types.h>
103 #include <kern/assert.h>
104 #include <kern/kalloc.h>
105 #include <kern/thread.h>
106 #include <kern/clock.h>
107 #include <kern/ledger.h>
108 #include <kern/monotonic.h>
109 #include <kern/task.h>
110 #include <kern/telemetry.h>
111 #include <kern/waitq.h>
112 #include <kern/sched_hygiene.h>
113 #include <kern/sched_prim.h>
114 #include <kern/mpsc_queue.h>
115 #include <kern/debug.h>
116
117 #include <sys/mbuf.h>
118 #include <sys/domain.h>
119 #include <sys/socket.h>
120 #include <sys/socketvar.h>
121 #include <sys/errno.h>
122 #include <sys/syscall.h>
123 #include <sys/pipe.h>
124
125 #include <security/audit/audit.h>
126
127 #include <net/if.h>
128 #include <net/route.h>
129
130 #include <netinet/in.h>
131 #include <netinet/in_systm.h>
132 #include <netinet/ip.h>
133 #include <netinet/in_pcb.h>
134 #include <netinet/ip_var.h>
135 #include <netinet/ip6.h>
136 #include <netinet/tcp.h>
137 #include <netinet/tcp_fsm.h>
138 #include <netinet/tcp_seq.h>
139 #include <netinet/tcp_timer.h>
140 #include <netinet/tcp_var.h>
141 #include <netinet/tcpip.h>
142 #include <netinet/tcp_debug.h>
143 /* for wait queue based select */
144 #include <kern/waitq.h>
145 #include <sys/vnode_internal.h>
146 /* for remote time api*/
147 #include <kern/remote_time.h>
148 #include <os/log.h>
149 #include <sys/log_data.h>
150
151 #if CONFIG_MACF
152 #include <security/mac_framework.h>
153 #endif
154
155 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
156 #include <mach_debug/mach_debug_types.h>
157 #endif
158
159 #if MONOTONIC
160 #include <machine/monotonic.h>
161 #endif /* MONOTONIC */
162
163 /* for entitlement check */
164 #include <IOKit/IOBSD.h>
165 /*
166 * If you need accounting for KM_SELECT consider using
167 * KALLOC_HEAP_DEFINE to define a view.
168 */
169 #define KM_SELECT KHEAP_DEFAULT
170
171 /* XXX should be in a header file somewhere */
172 extern kern_return_t IOBSDGetPlatformUUID(__darwin_uuid_t uuid, mach_timespec_t timeoutp);
173
174 int do_uiowrite(struct proc *p, struct fileproc *fp, uio_t uio, int flags, user_ssize_t *retval);
175 __private_extern__ int dofileread(vfs_context_t ctx, struct fileproc *fp,
176 user_addr_t bufp, user_size_t nbyte,
177 off_t offset, int flags, user_ssize_t *retval);
178 __private_extern__ int dofilewrite(vfs_context_t ctx, struct fileproc *fp,
179 user_addr_t bufp, user_size_t nbyte,
180 off_t offset, int flags, user_ssize_t *retval);
181 static int preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_vnode);
182
183 /* needed by guarded_writev, etc. */
184 int write_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
185 off_t offset, int flags, guardid_t *puguard, user_ssize_t *retval);
186 int writev_uio(struct proc *p, int fd, user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
187 guardid_t *puguard, user_ssize_t *retval);
188
189 #define f_flag fp_glob->fg_flag
190 #define f_type fp_glob->fg_ops->fo_type
191 #define f_cred fp_glob->fg_cred
192 #define f_ops fp_glob->fg_ops
193
194 /*
195 * Validate if the file can be used for random access (pread, pwrite, etc).
196 *
197 * Conditions:
198 * proc_fdlock is held
199 *
200 * Returns: 0 Success
201 * ESPIPE
202 * ENXIO
203 */
204 static int
valid_for_random_access(struct fileproc * fp)205 valid_for_random_access(struct fileproc *fp)
206 {
207 if (__improbable(fp->f_type != DTYPE_VNODE)) {
208 return ESPIPE;
209 }
210
211 vnode_t vp = (struct vnode *)fp_get_data(fp);
212 if (__improbable(vnode_isfifo(vp))) {
213 return ESPIPE;
214 }
215
216 if (__improbable(vp->v_flag & VISTTY)) {
217 return ENXIO;
218 }
219
220 return 0;
221 }
222
223 /*
224 * Returns: 0 Success
225 * EBADF
226 * ESPIPE
227 * ENXIO
228 * fp_lookup:EBADF
229 * valid_for_random_access:ESPIPE
230 * valid_for_random_access:ENXIO
231 */
232 static int
preparefileread(struct proc * p,struct fileproc ** fp_ret,int fd,int check_for_pread)233 preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_pread)
234 {
235 int error;
236 struct fileproc *fp;
237
238 AUDIT_ARG(fd, fd);
239
240 proc_fdlock_spin(p);
241
242 error = fp_lookup(p, fd, &fp, 1);
243
244 if (error) {
245 proc_fdunlock(p);
246 return error;
247 }
248 if ((fp->f_flag & FREAD) == 0) {
249 error = EBADF;
250 goto out;
251 }
252 if (check_for_pread) {
253 if ((error = valid_for_random_access(fp))) {
254 goto out;
255 }
256 }
257
258 *fp_ret = fp;
259
260 proc_fdunlock(p);
261 return 0;
262
263 out:
264 fp_drop(p, fd, fp, 1);
265 proc_fdunlock(p);
266 return error;
267 }
268
269 static int
fp_readv(vfs_context_t ctx,struct fileproc * fp,uio_t uio,int flags,user_ssize_t * retval)270 fp_readv(vfs_context_t ctx, struct fileproc *fp, uio_t uio, int flags,
271 user_ssize_t *retval)
272 {
273 int error;
274 user_ssize_t count;
275
276 if ((error = uio_calculateresid(uio))) {
277 *retval = 0;
278 return error;
279 }
280
281 count = uio_resid(uio);
282 error = fo_read(fp, uio, flags, ctx);
283
284 switch (error) {
285 case ERESTART:
286 case EINTR:
287 case EWOULDBLOCK:
288 if (uio_resid(uio) != count) {
289 error = 0;
290 }
291 break;
292
293 default:
294 break;
295 }
296
297 *retval = count - uio_resid(uio);
298 return error;
299 }
300
301 /*
302 * Returns: 0 Success
303 * EINVAL
304 * fo_read:???
305 */
306 __private_extern__ int
dofileread(vfs_context_t ctx,struct fileproc * fp,user_addr_t bufp,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)307 dofileread(vfs_context_t ctx, struct fileproc *fp,
308 user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
309 user_ssize_t *retval)
310 {
311 uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
312 uio_t uio;
313 int spacetype;
314
315 if (nbyte > INT_MAX) {
316 *retval = 0;
317 return EINVAL;
318 }
319
320 spacetype = vfs_context_is64bit(ctx) ? UIO_USERSPACE64 : UIO_USERSPACE32;
321 uio = uio_createwithbuffer(1, offset, spacetype, UIO_READ, &uio_buf[0],
322 sizeof(uio_buf));
323
324 if (uio_addiov(uio, bufp, nbyte) != 0) {
325 *retval = 0;
326 return EINVAL;
327 }
328
329 return fp_readv(ctx, fp, uio, flags, retval);
330 }
331
332 static int
readv_internal(struct proc * p,int fd,uio_t uio,int flags,user_ssize_t * retval)333 readv_internal(struct proc *p, int fd, uio_t uio, int flags,
334 user_ssize_t *retval)
335 {
336 struct fileproc *fp = NULL;
337 struct vfs_context context;
338 int error;
339
340 if ((error = preparefileread(p, &fp, fd, flags & FOF_OFFSET))) {
341 *retval = 0;
342 return error;
343 }
344
345 context = *(vfs_context_current());
346 context.vc_ucred = fp->fp_glob->fg_cred;
347
348 error = fp_readv(&context, fp, uio, flags, retval);
349
350 fp_drop(p, fd, fp, 0);
351 return error;
352 }
353
354 static int
read_internal(struct proc * p,int fd,user_addr_t buf,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)355 read_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
356 off_t offset, int flags, user_ssize_t *retval)
357 {
358 uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
359 uio_t uio;
360 int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
361
362 if (nbyte > INT_MAX) {
363 *retval = 0;
364 return EINVAL;
365 }
366
367 uio = uio_createwithbuffer(1, offset, spacetype, UIO_READ,
368 &uio_buf[0], sizeof(uio_buf));
369
370 if (uio_addiov(uio, buf, nbyte) != 0) {
371 *retval = 0;
372 return EINVAL;
373 }
374
375 return readv_internal(p, fd, uio, flags, retval);
376 }
377
378 int
read_nocancel(struct proc * p,struct read_nocancel_args * uap,user_ssize_t * retval)379 read_nocancel(struct proc *p, struct read_nocancel_args *uap, user_ssize_t *retval)
380 {
381 return read_internal(p, uap->fd, uap->cbuf, uap->nbyte, (off_t)-1, 0,
382 retval);
383 }
384
385 /*
386 * Read system call.
387 *
388 * Returns: 0 Success
389 * preparefileread:EBADF
390 * preparefileread:ESPIPE
391 * preparefileread:ENXIO
392 * preparefileread:EBADF
393 * dofileread:???
394 */
395 int
read(struct proc * p,struct read_args * uap,user_ssize_t * retval)396 read(struct proc *p, struct read_args *uap, user_ssize_t *retval)
397 {
398 __pthread_testcancel(1);
399 return read_nocancel(p, (struct read_nocancel_args *)uap, retval);
400 }
401
402 int
pread_nocancel(struct proc * p,struct pread_nocancel_args * uap,user_ssize_t * retval)403 pread_nocancel(struct proc *p, struct pread_nocancel_args *uap, user_ssize_t *retval)
404 {
405 KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_pread) | DBG_FUNC_NONE),
406 uap->fd, uap->nbyte, (unsigned int)((uap->offset >> 32)), (unsigned int)(uap->offset), 0);
407
408 return read_internal(p, uap->fd, uap->buf, uap->nbyte, uap->offset,
409 FOF_OFFSET, retval);
410 }
411
412 /*
413 * Pread system call
414 *
415 * Returns: 0 Success
416 * preparefileread:EBADF
417 * preparefileread:ESPIPE
418 * preparefileread:ENXIO
419 * preparefileread:EBADF
420 * dofileread:???
421 */
422 int
pread(struct proc * p,struct pread_args * uap,user_ssize_t * retval)423 pread(struct proc *p, struct pread_args *uap, user_ssize_t *retval)
424 {
425 __pthread_testcancel(1);
426 return pread_nocancel(p, (struct pread_nocancel_args *)uap, retval);
427 }
428
429 /*
430 * Vector read.
431 *
432 * Returns: 0 Success
433 * EINVAL
434 * ENOMEM
435 * preparefileread:EBADF
436 * preparefileread:ESPIPE
437 * preparefileread:ENXIO
438 * preparefileread:EBADF
439 * copyin:EFAULT
440 * rd_uio:???
441 */
442 static int
readv_uio(struct proc * p,int fd,user_addr_t user_iovp,int iovcnt,off_t offset,int flags,user_ssize_t * retval)443 readv_uio(struct proc *p, int fd,
444 user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
445 user_ssize_t *retval)
446 {
447 uio_t uio = NULL;
448 int error;
449 struct user_iovec *iovp;
450
451 if (iovcnt <= 0 || iovcnt > UIO_MAXIOV) {
452 error = EINVAL;
453 goto out;
454 }
455
456 uio = uio_create(iovcnt, offset,
457 (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
458 UIO_READ);
459
460 iovp = uio_iovsaddr(uio);
461 if (iovp == NULL) {
462 error = ENOMEM;
463 goto out;
464 }
465
466 error = copyin_user_iovec_array(user_iovp,
467 IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32,
468 iovcnt, iovp);
469
470 if (error) {
471 goto out;
472 }
473
474 error = readv_internal(p, fd, uio, flags, retval);
475
476 out:
477 if (uio != NULL) {
478 uio_free(uio);
479 }
480
481 return error;
482 }
483
484 int
readv_nocancel(struct proc * p,struct readv_nocancel_args * uap,user_ssize_t * retval)485 readv_nocancel(struct proc *p, struct readv_nocancel_args *uap, user_ssize_t *retval)
486 {
487 return readv_uio(p, uap->fd, uap->iovp, uap->iovcnt, 0, 0, retval);
488 }
489
490 /*
491 * Scatter read system call.
492 */
493 int
readv(struct proc * p,struct readv_args * uap,user_ssize_t * retval)494 readv(struct proc *p, struct readv_args *uap, user_ssize_t *retval)
495 {
496 __pthread_testcancel(1);
497 return readv_nocancel(p, (struct readv_nocancel_args *)uap, retval);
498 }
499
500 int
sys_preadv_nocancel(struct proc * p,struct preadv_nocancel_args * uap,user_ssize_t * retval)501 sys_preadv_nocancel(struct proc *p, struct preadv_nocancel_args *uap, user_ssize_t *retval)
502 {
503 return readv_uio(p, uap->fd, uap->iovp, uap->iovcnt, uap->offset,
504 FOF_OFFSET, retval);
505 }
506
507 /*
508 * Preadv system call
509 */
510 int
sys_preadv(struct proc * p,struct preadv_args * uap,user_ssize_t * retval)511 sys_preadv(struct proc *p, struct preadv_args *uap, user_ssize_t *retval)
512 {
513 __pthread_testcancel(1);
514 return sys_preadv_nocancel(p, (struct preadv_nocancel_args *)uap, retval);
515 }
516
517 /*
518 * Returns: 0 Success
519 * EBADF
520 * ESPIPE
521 * ENXIO
522 * fp_lookup:EBADF
523 * fp_guard_exception:???
524 * valid_for_random_access:ESPIPE
525 * valid_for_random_access:ENXIO
526 */
527 static int
preparefilewrite(struct proc * p,struct fileproc ** fp_ret,int fd,int check_for_pwrite,guardid_t * puguard)528 preparefilewrite(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_pwrite,
529 guardid_t *puguard)
530 {
531 int error;
532 struct fileproc *fp;
533
534 AUDIT_ARG(fd, fd);
535
536 proc_fdlock_spin(p);
537
538 if (puguard) {
539 error = fp_lookup_guarded(p, fd, *puguard, &fp, 1);
540 if (error) {
541 proc_fdunlock(p);
542 return error;
543 }
544
545 if ((fp->f_flag & FWRITE) == 0) {
546 error = EBADF;
547 goto out;
548 }
549 } else {
550 error = fp_lookup(p, fd, &fp, 1);
551 if (error) {
552 proc_fdunlock(p);
553 return error;
554 }
555
556 /* Allow EBADF first. */
557 if ((fp->f_flag & FWRITE) == 0) {
558 error = EBADF;
559 goto out;
560 }
561
562 if (fp_isguarded(fp, GUARD_WRITE)) {
563 error = fp_guard_exception(p, fd, fp, kGUARD_EXC_WRITE);
564 goto out;
565 }
566 }
567
568 if (check_for_pwrite) {
569 if ((error = valid_for_random_access(fp))) {
570 goto out;
571 }
572 }
573
574 *fp_ret = fp;
575
576 proc_fdunlock(p);
577 return 0;
578
579 out:
580 fp_drop(p, fd, fp, 1);
581 proc_fdunlock(p);
582 return error;
583 }
584
585 static int
fp_writev(vfs_context_t ctx,struct fileproc * fp,uio_t uio,int flags,user_ssize_t * retval)586 fp_writev(vfs_context_t ctx, struct fileproc *fp, uio_t uio, int flags,
587 user_ssize_t *retval)
588 {
589 int error;
590 user_ssize_t count;
591
592 if ((error = uio_calculateresid(uio))) {
593 *retval = 0;
594 return error;
595 }
596
597 count = uio_resid(uio);
598 error = fo_write(fp, uio, flags, ctx);
599
600 switch (error) {
601 case ERESTART:
602 case EINTR:
603 case EWOULDBLOCK:
604 if (uio_resid(uio) != count) {
605 error = 0;
606 }
607 break;
608
609 case EPIPE:
610 if (fp->f_type != DTYPE_SOCKET &&
611 (fp->fp_glob->fg_lflags & FG_NOSIGPIPE) == 0) {
612 /* XXX Raise the signal on the thread? */
613 psignal(vfs_context_proc(ctx), SIGPIPE);
614 }
615 break;
616
617 default:
618 break;
619 }
620
621 if ((*retval = count - uio_resid(uio))) {
622 os_atomic_or(&fp->fp_glob->fg_flag, FWASWRITTEN, relaxed);
623 }
624
625 return error;
626 }
627
628 /*
629 * Returns: 0 Success
630 * EINVAL
631 * <fo_write>:EPIPE
632 * <fo_write>:??? [indirect through struct fileops]
633 */
634 __private_extern__ int
dofilewrite(vfs_context_t ctx,struct fileproc * fp,user_addr_t bufp,user_size_t nbyte,off_t offset,int flags,user_ssize_t * retval)635 dofilewrite(vfs_context_t ctx, struct fileproc *fp,
636 user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
637 user_ssize_t *retval)
638 {
639 uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
640 uio_t uio;
641 int spacetype;
642
643 if (nbyte > INT_MAX) {
644 *retval = 0;
645 return EINVAL;
646 }
647
648 spacetype = vfs_context_is64bit(ctx) ? UIO_USERSPACE64 : UIO_USERSPACE32;
649 uio = uio_createwithbuffer(1, offset, spacetype, UIO_WRITE, &uio_buf[0],
650 sizeof(uio_buf));
651
652 if (uio_addiov(uio, bufp, nbyte) != 0) {
653 *retval = 0;
654 return EINVAL;
655 }
656
657 return fp_writev(ctx, fp, uio, flags, retval);
658 }
659
660 static int
writev_internal(struct proc * p,int fd,uio_t uio,int flags,guardid_t * puguard,user_ssize_t * retval)661 writev_internal(struct proc *p, int fd, uio_t uio, int flags,
662 guardid_t *puguard, user_ssize_t *retval)
663 {
664 struct fileproc *fp = NULL;
665 struct vfs_context context;
666 int error;
667
668 if ((error = preparefilewrite(p, &fp, fd, flags & FOF_OFFSET, puguard))) {
669 *retval = 0;
670 return error;
671 }
672
673 context = *(vfs_context_current());
674 context.vc_ucred = fp->fp_glob->fg_cred;
675
676 error = fp_writev(&context, fp, uio, flags, retval);
677
678 fp_drop(p, fd, fp, 0);
679 return error;
680 }
681
682 int
write_internal(struct proc * p,int fd,user_addr_t buf,user_size_t nbyte,off_t offset,int flags,guardid_t * puguard,user_ssize_t * retval)683 write_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
684 off_t offset, int flags, guardid_t *puguard, user_ssize_t *retval)
685 {
686 uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
687 uio_t uio;
688 int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
689
690 if (nbyte > INT_MAX) {
691 *retval = 0;
692 return EINVAL;
693 }
694
695 uio = uio_createwithbuffer(1, offset, spacetype, UIO_WRITE,
696 &uio_buf[0], sizeof(uio_buf));
697
698 if (uio_addiov(uio, buf, nbyte) != 0) {
699 *retval = 0;
700 return EINVAL;
701 }
702
703 return writev_internal(p, fd, uio, flags, puguard, retval);
704 }
705
706 int
write_nocancel(struct proc * p,struct write_nocancel_args * uap,user_ssize_t * retval)707 write_nocancel(struct proc *p, struct write_nocancel_args *uap, user_ssize_t *retval)
708 {
709 return write_internal(p, uap->fd, uap->cbuf, uap->nbyte, (off_t)-1, 0,
710 NULL, retval);
711 }
712
713 /*
714 * Write system call
715 *
716 * Returns: 0 Success
717 * EBADF
718 * fp_lookup:EBADF
719 * dofilewrite:???
720 */
721 int
write(struct proc * p,struct write_args * uap,user_ssize_t * retval)722 write(struct proc *p, struct write_args *uap, user_ssize_t *retval)
723 {
724 __pthread_testcancel(1);
725 return write_nocancel(p, (struct write_nocancel_args *)uap, retval);
726 }
727
728 int
pwrite_nocancel(struct proc * p,struct pwrite_nocancel_args * uap,user_ssize_t * retval)729 pwrite_nocancel(struct proc *p, struct pwrite_nocancel_args *uap, user_ssize_t *retval)
730 {
731 KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_pwrite) | DBG_FUNC_NONE),
732 uap->fd, uap->nbyte, (unsigned int)((uap->offset >> 32)), (unsigned int)(uap->offset), 0);
733
734 /* XXX: Should be < 0 instead? (See man page + pwritev) */
735 if (uap->offset == (off_t)-1) {
736 return EINVAL;
737 }
738
739 return write_internal(p, uap->fd, uap->buf, uap->nbyte, uap->offset,
740 FOF_OFFSET, NULL, retval);
741 }
742
743 /*
744 * pwrite system call
745 *
746 * Returns: 0 Success
747 * EBADF
748 * ESPIPE
749 * ENXIO
750 * EINVAL
751 * fp_lookup:EBADF
752 * dofilewrite:???
753 */
754 int
pwrite(struct proc * p,struct pwrite_args * uap,user_ssize_t * retval)755 pwrite(struct proc *p, struct pwrite_args *uap, user_ssize_t *retval)
756 {
757 __pthread_testcancel(1);
758 return pwrite_nocancel(p, (struct pwrite_nocancel_args *)uap, retval);
759 }
760
761 int
writev_uio(struct proc * p,int fd,user_addr_t user_iovp,int iovcnt,off_t offset,int flags,guardid_t * puguard,user_ssize_t * retval)762 writev_uio(struct proc *p, int fd,
763 user_addr_t user_iovp, int iovcnt, off_t offset, int flags,
764 guardid_t *puguard, user_ssize_t *retval)
765 {
766 uio_t uio = NULL;
767 int error;
768 struct user_iovec *iovp;
769
770 if (iovcnt <= 0 || iovcnt > UIO_MAXIOV || offset < 0) {
771 error = EINVAL;
772 goto out;
773 }
774
775 uio = uio_create(iovcnt, offset,
776 (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
777 UIO_WRITE);
778
779 iovp = uio_iovsaddr(uio);
780 if (iovp == NULL) {
781 error = ENOMEM;
782 goto out;
783 }
784
785 error = copyin_user_iovec_array(user_iovp,
786 IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32,
787 iovcnt, iovp);
788
789 if (error) {
790 goto out;
791 }
792
793 error = writev_internal(p, fd, uio, flags, puguard, retval);
794
795 out:
796 if (uio != NULL) {
797 uio_free(uio);
798 }
799
800 return error;
801 }
802
803 int
writev_nocancel(struct proc * p,struct writev_nocancel_args * uap,user_ssize_t * retval)804 writev_nocancel(struct proc *p, struct writev_nocancel_args *uap, user_ssize_t *retval)
805 {
806 return writev_uio(p, uap->fd, uap->iovp, uap->iovcnt, 0, 0, NULL, retval);
807 }
808
809 /*
810 * Gather write system call
811 */
812 int
writev(struct proc * p,struct writev_args * uap,user_ssize_t * retval)813 writev(struct proc *p, struct writev_args *uap, user_ssize_t *retval)
814 {
815 __pthread_testcancel(1);
816 return writev_nocancel(p, (struct writev_nocancel_args *)uap, retval);
817 }
818
819 int
sys_pwritev_nocancel(struct proc * p,struct pwritev_nocancel_args * uap,user_ssize_t * retval)820 sys_pwritev_nocancel(struct proc *p, struct pwritev_nocancel_args *uap, user_ssize_t *retval)
821 {
822 return writev_uio(p, uap->fd, uap->iovp, uap->iovcnt, uap->offset,
823 FOF_OFFSET, NULL, retval);
824 }
825
826 /*
827 * Pwritev system call
828 */
829 int
sys_pwritev(struct proc * p,struct pwritev_args * uap,user_ssize_t * retval)830 sys_pwritev(struct proc *p, struct pwritev_args *uap, user_ssize_t *retval)
831 {
832 __pthread_testcancel(1);
833 return sys_pwritev_nocancel(p, (struct pwritev_nocancel_args *)uap, retval);
834 }
835
836 /*
837 * Ioctl system call
838 *
839 * Returns: 0 Success
840 * EBADF
841 * ENOTTY
842 * ENOMEM
843 * ESRCH
844 * copyin:EFAULT
845 * copyoutEFAULT
846 * fp_lookup:EBADF Bad file descriptor
847 * fo_ioctl:???
848 */
849 int
ioctl(struct proc * p,struct ioctl_args * uap,__unused int32_t * retval)850 ioctl(struct proc *p, struct ioctl_args *uap, __unused int32_t *retval)
851 {
852 struct fileproc *fp = NULL;
853 int error = 0;
854 u_int size = 0;
855 caddr_t datap = NULL, memp = NULL;
856 boolean_t is64bit = FALSE;
857 int tmp = 0;
858 #define STK_PARAMS 128
859 char stkbuf[STK_PARAMS] = {};
860 int fd = uap->fd;
861 u_long com = uap->com;
862 struct vfs_context context = *vfs_context_current();
863
864 AUDIT_ARG(fd, uap->fd);
865 AUDIT_ARG(addr, uap->data);
866
867 is64bit = proc_is64bit(p);
868 #if CONFIG_AUDIT
869 if (is64bit) {
870 AUDIT_ARG(value64, com);
871 } else {
872 AUDIT_ARG(cmd, CAST_DOWN_EXPLICIT(int, com));
873 }
874 #endif /* CONFIG_AUDIT */
875
876 /*
877 * Interpret high order word to find amount of data to be
878 * copied to/from the user's address space.
879 */
880 size = IOCPARM_LEN(com);
881 if (size > IOCPARM_MAX) {
882 return ENOTTY;
883 }
884 if (size > sizeof(stkbuf)) {
885 memp = (caddr_t)kalloc_data(size, Z_WAITOK);
886 if (memp == 0) {
887 return ENOMEM;
888 }
889 datap = memp;
890 } else {
891 datap = &stkbuf[0];
892 }
893 if (com & IOC_IN) {
894 if (size) {
895 error = copyin(uap->data, datap, size);
896 if (error) {
897 goto out_nofp;
898 }
899 } else {
900 /* XXX - IOC_IN and no size? we should proably return an error here!! */
901 if (is64bit) {
902 *(user_addr_t *)datap = uap->data;
903 } else {
904 *(uint32_t *)datap = (uint32_t)uap->data;
905 }
906 }
907 } else if ((com & IOC_OUT) && size) {
908 /*
909 * Zero the buffer so the user always
910 * gets back something deterministic.
911 */
912 bzero(datap, size);
913 } else if (com & IOC_VOID) {
914 /* XXX - this is odd since IOC_VOID means no parameters */
915 if (is64bit) {
916 *(user_addr_t *)datap = uap->data;
917 } else {
918 *(uint32_t *)datap = (uint32_t)uap->data;
919 }
920 }
921
922 proc_fdlock(p);
923 error = fp_lookup(p, fd, &fp, 1);
924 if (error) {
925 proc_fdunlock(p);
926 goto out_nofp;
927 }
928
929 AUDIT_ARG(file, p, fp);
930
931 if ((fp->f_flag & (FREAD | FWRITE)) == 0) {
932 error = EBADF;
933 goto out;
934 }
935
936 context.vc_ucred = fp->fp_glob->fg_cred;
937
938 #if CONFIG_MACF
939 error = mac_file_check_ioctl(context.vc_ucred, fp->fp_glob, com);
940 if (error) {
941 goto out;
942 }
943 #endif
944
945 switch (com) {
946 case FIONCLEX:
947 fp->fp_flags &= ~FP_CLOEXEC;
948 break;
949
950 case FIOCLEX:
951 fp->fp_flags |= FP_CLOEXEC;
952 break;
953
954 case FIONBIO:
955 // FIXME (rdar://54898652)
956 //
957 // this code is broken if fnctl(F_SETFL), ioctl() are
958 // called concurrently for the same fileglob.
959 if ((tmp = *(int *)datap)) {
960 os_atomic_or(&fp->f_flag, FNONBLOCK, relaxed);
961 } else {
962 os_atomic_andnot(&fp->f_flag, FNONBLOCK, relaxed);
963 }
964 error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, &context);
965 break;
966
967 case FIOASYNC:
968 // FIXME (rdar://54898652)
969 //
970 // this code is broken if fnctl(F_SETFL), ioctl() are
971 // called concurrently for the same fileglob.
972 if ((tmp = *(int *)datap)) {
973 os_atomic_or(&fp->f_flag, FASYNC, relaxed);
974 } else {
975 os_atomic_andnot(&fp->f_flag, FASYNC, relaxed);
976 }
977 error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, &context);
978 break;
979
980 case FIOSETOWN:
981 tmp = *(int *)datap;
982 if (fp->f_type == DTYPE_SOCKET) {
983 ((struct socket *)fp_get_data(fp))->so_pgid = tmp;
984 break;
985 }
986 if (fp->f_type == DTYPE_PIPE) {
987 error = fo_ioctl(fp, TIOCSPGRP, (caddr_t)&tmp, &context);
988 break;
989 }
990 if (tmp <= 0) {
991 tmp = -tmp;
992 } else {
993 struct proc *p1 = proc_find(tmp);
994 if (p1 == 0) {
995 error = ESRCH;
996 break;
997 }
998 tmp = p1->p_pgrpid;
999 proc_rele(p1);
1000 }
1001 error = fo_ioctl(fp, TIOCSPGRP, (caddr_t)&tmp, &context);
1002 break;
1003
1004 case FIOGETOWN:
1005 if (fp->f_type == DTYPE_SOCKET) {
1006 *(int *)datap = ((struct socket *)fp_get_data(fp))->so_pgid;
1007 break;
1008 }
1009 error = fo_ioctl(fp, TIOCGPGRP, datap, &context);
1010 *(int *)datap = -*(int *)datap;
1011 break;
1012
1013 default:
1014 error = fo_ioctl(fp, com, datap, &context);
1015 /*
1016 * Copy any data to user, size was
1017 * already set and checked above.
1018 */
1019 if (error == 0 && (com & IOC_OUT) && size) {
1020 error = copyout(datap, uap->data, (u_int)size);
1021 }
1022 break;
1023 }
1024 out:
1025 fp_drop(p, fd, fp, 1);
1026 proc_fdunlock(p);
1027
1028 out_nofp:
1029 if (memp) {
1030 kfree_data(memp, size);
1031 }
1032 return error;
1033 }
1034
1035 int selwait;
1036 #define SEL_FIRSTPASS 1
1037 #define SEL_SECONDPASS 2
1038 static int selprocess(struct proc *p, int error, int sel_pass);
1039 static int selscan(struct proc *p, struct _select * sel, struct _select_data * seldata,
1040 int nfd, int32_t *retval, int sel_pass, struct select_set *selset);
1041 static int selcount(struct proc *p, u_int32_t *ibits, int nfd, int *count);
1042 static int seldrop_locked(struct proc *p, u_int32_t *ibits, int nfd, int lim, int *need_wakeup);
1043 static int seldrop(struct proc *p, u_int32_t *ibits, int nfd, int lim);
1044 static int select_internal(struct proc *p, struct select_nocancel_args *uap, uint64_t timeout, int32_t *retval);
1045
1046 /*
1047 * This is used for the special device nodes that do not implement
1048 * a proper kevent filter (see filt_specattach).
1049 *
1050 * In order to enable kevents on those, the spec_filtops will pretend
1051 * to call select, and try to sniff the selrecord(), if it observes one,
1052 * the knote is attached, which pairs with selwakeup() or selthreadclear().
1053 *
1054 * The last issue remaining, is that we need to serialize filt_specdetach()
1055 * with this, but it really can't know the "selinfo" or any locking domain.
1056 * To make up for this, We protect knote list operations with a global lock,
1057 * which give us a safe shared locking domain.
1058 *
1059 * Note: It is a little distasteful, but we really have very few of those.
1060 * The big problem here is that sharing a lock domain without
1061 * any kind of shared knowledge is a little complicated.
1062 *
1063 * 1. filters can really implement their own kqueue integration
1064 * to side step this,
1065 *
1066 * 2. There's an opportunity to pick a private lock in selspec_attach()
1067 * because both the selinfo and the knote are locked at that time.
1068 * The cleanup story is however a little complicated.
1069 */
1070 static LCK_GRP_DECLARE(selspec_grp, "spec_filtops");
1071 static LCK_SPIN_DECLARE(selspec_lock, &selspec_grp);
1072
1073 /*
1074 * The "primitive" lock is held.
1075 * The knote lock is held.
1076 */
1077 void
selspec_attach(struct knote * kn,struct selinfo * si)1078 selspec_attach(struct knote *kn, struct selinfo *si)
1079 {
1080 struct selinfo *cur = os_atomic_load(&kn->kn_hook, relaxed);
1081
1082 if (cur == NULL) {
1083 si->si_flags |= SI_SELSPEC;
1084 lck_spin_lock(&selspec_lock);
1085 kn->kn_hook = si;
1086 KNOTE_ATTACH(&si->si_note, kn);
1087 lck_spin_unlock(&selspec_lock);
1088 } else {
1089 /*
1090 * selspec_attach() can be called from e.g. filt_spectouch()
1091 * which might be called before any event was dequeued.
1092 *
1093 * It is hence not impossible for the knote already be hooked.
1094 *
1095 * Note that selwakeup_internal() could possibly
1096 * already have cleared this pointer. This is a race
1097 * that filt_specprocess will debounce.
1098 */
1099 assert(si->si_flags & SI_SELSPEC);
1100 assert(cur == si);
1101 }
1102 }
1103
1104 /*
1105 * The "primitive" lock is _not_ held.
1106 * The knote lock is held.
1107 */
1108 void
selspec_detach(struct knote * kn)1109 selspec_detach(struct knote *kn)
1110 {
1111 /*
1112 * kn_hook always becomes non NULL under the knote lock.
1113 * Seeing "NULL" can't be a false positive.
1114 */
1115 if (kn->kn_hook == NULL) {
1116 return;
1117 }
1118
1119 lck_spin_lock(&selspec_lock);
1120 if (kn->kn_hook) {
1121 struct selinfo *sip = kn->kn_hook;
1122
1123 kn->kn_hook = NULL;
1124 KNOTE_DETACH(&sip->si_note, kn);
1125 }
1126 lck_spin_unlock(&selspec_lock);
1127 }
1128
1129 /*
1130 * Select system call.
1131 *
1132 * Returns: 0 Success
1133 * EINVAL Invalid argument
1134 * EAGAIN Nonconformant error if allocation fails
1135 */
1136 int
select(struct proc * p,struct select_args * uap,int32_t * retval)1137 select(struct proc *p, struct select_args *uap, int32_t *retval)
1138 {
1139 __pthread_testcancel(1);
1140 return select_nocancel(p, (struct select_nocancel_args *)uap, retval);
1141 }
1142
1143 int
select_nocancel(struct proc * p,struct select_nocancel_args * uap,int32_t * retval)1144 select_nocancel(struct proc *p, struct select_nocancel_args *uap, int32_t *retval)
1145 {
1146 uint64_t timeout = 0;
1147
1148 if (uap->tv) {
1149 int err;
1150 struct timeval atv;
1151 if (IS_64BIT_PROCESS(p)) {
1152 struct user64_timeval atv64;
1153 err = copyin(uap->tv, (caddr_t)&atv64, sizeof(atv64));
1154 /* Loses resolution - assume timeout < 68 years */
1155 atv.tv_sec = (__darwin_time_t)atv64.tv_sec;
1156 atv.tv_usec = atv64.tv_usec;
1157 } else {
1158 struct user32_timeval atv32;
1159 err = copyin(uap->tv, (caddr_t)&atv32, sizeof(atv32));
1160 atv.tv_sec = atv32.tv_sec;
1161 atv.tv_usec = atv32.tv_usec;
1162 }
1163 if (err) {
1164 return err;
1165 }
1166
1167 if (itimerfix(&atv)) {
1168 err = EINVAL;
1169 return err;
1170 }
1171
1172 clock_absolutetime_interval_to_deadline(tvtoabstime(&atv), &timeout);
1173 }
1174
1175 return select_internal(p, uap, timeout, retval);
1176 }
1177
1178 int
pselect(struct proc * p,struct pselect_args * uap,int32_t * retval)1179 pselect(struct proc *p, struct pselect_args *uap, int32_t *retval)
1180 {
1181 __pthread_testcancel(1);
1182 return pselect_nocancel(p, (struct pselect_nocancel_args *)uap, retval);
1183 }
1184
1185 int
pselect_nocancel(struct proc * p,struct pselect_nocancel_args * uap,int32_t * retval)1186 pselect_nocancel(struct proc *p, struct pselect_nocancel_args *uap, int32_t *retval)
1187 {
1188 int err;
1189 struct uthread *ut;
1190 uint64_t timeout = 0;
1191
1192 if (uap->ts) {
1193 struct timespec ts;
1194
1195 if (IS_64BIT_PROCESS(p)) {
1196 struct user64_timespec ts64;
1197 err = copyin(uap->ts, (caddr_t)&ts64, sizeof(ts64));
1198 ts.tv_sec = (__darwin_time_t)ts64.tv_sec;
1199 ts.tv_nsec = (long)ts64.tv_nsec;
1200 } else {
1201 struct user32_timespec ts32;
1202 err = copyin(uap->ts, (caddr_t)&ts32, sizeof(ts32));
1203 ts.tv_sec = ts32.tv_sec;
1204 ts.tv_nsec = ts32.tv_nsec;
1205 }
1206 if (err) {
1207 return err;
1208 }
1209
1210 if (!timespec_is_valid(&ts)) {
1211 return EINVAL;
1212 }
1213 clock_absolutetime_interval_to_deadline(tstoabstime(&ts), &timeout);
1214 }
1215
1216 ut = current_uthread();
1217
1218 if (uap->mask != USER_ADDR_NULL) {
1219 /* save current mask, then copyin and set new mask */
1220 sigset_t newset;
1221 err = copyin(uap->mask, &newset, sizeof(sigset_t));
1222 if (err) {
1223 return err;
1224 }
1225 ut->uu_oldmask = ut->uu_sigmask;
1226 ut->uu_flag |= UT_SAS_OLDMASK;
1227 ut->uu_sigmask = (newset & ~sigcantmask);
1228 }
1229
1230 err = select_internal(p, (struct select_nocancel_args *)uap, timeout, retval);
1231
1232 if (err != EINTR && ut->uu_flag & UT_SAS_OLDMASK) {
1233 /*
1234 * Restore old mask (direct return case). NOTE: EINTR can also be returned
1235 * if the thread is cancelled. In that case, we don't reset the signal
1236 * mask to its original value (which usually happens in the signal
1237 * delivery path). This behavior is permitted by POSIX.
1238 */
1239 ut->uu_sigmask = ut->uu_oldmask;
1240 ut->uu_oldmask = 0;
1241 ut->uu_flag &= ~UT_SAS_OLDMASK;
1242 }
1243
1244 return err;
1245 }
1246
1247 void
select_cleanup_uthread(struct _select * sel)1248 select_cleanup_uthread(struct _select *sel)
1249 {
1250 kfree_data(sel->ibits, 2 * sel->nbytes);
1251 sel->ibits = sel->obits = NULL;
1252 sel->nbytes = 0;
1253 }
1254
1255 static int
select_grow_uthread_cache(struct _select * sel,uint32_t nbytes)1256 select_grow_uthread_cache(struct _select *sel, uint32_t nbytes)
1257 {
1258 uint32_t *buf;
1259
1260 buf = kalloc_data(2 * nbytes, Z_WAITOK | Z_ZERO);
1261 if (buf) {
1262 select_cleanup_uthread(sel);
1263 sel->ibits = buf;
1264 sel->obits = buf + nbytes / sizeof(uint32_t);
1265 sel->nbytes = nbytes;
1266 return true;
1267 }
1268 return false;
1269 }
1270
1271 static void
select_bzero_uthread_cache(struct _select * sel)1272 select_bzero_uthread_cache(struct _select *sel)
1273 {
1274 bzero(sel->ibits, sel->nbytes * 2);
1275 }
1276
1277 /*
1278 * Generic implementation of {,p}select. Care: we type-pun uap across the two
1279 * syscalls, which differ slightly. The first 4 arguments (nfds and the fd sets)
1280 * are identical. The 5th (timeout) argument points to different types, so we
1281 * unpack in the syscall-specific code, but the generic code still does a null
1282 * check on this argument to determine if a timeout was specified.
1283 */
1284 static int
select_internal(struct proc * p,struct select_nocancel_args * uap,uint64_t timeout,int32_t * retval)1285 select_internal(struct proc *p, struct select_nocancel_args *uap, uint64_t timeout, int32_t *retval)
1286 {
1287 struct uthread *uth = current_uthread();
1288 struct _select *sel = &uth->uu_select;
1289 struct _select_data *seldata = &uth->uu_save.uus_select_data;
1290 int error = 0;
1291 u_int ni, nw;
1292
1293 *retval = 0;
1294
1295 seldata->abstime = timeout;
1296 seldata->args = uap;
1297 seldata->retval = retval;
1298 seldata->count = 0;
1299
1300 if (uap->nd < 0) {
1301 return EINVAL;
1302 }
1303
1304 if (uap->nd > p->p_fd.fd_nfiles) {
1305 uap->nd = p->p_fd.fd_nfiles; /* forgiving; slightly wrong */
1306 }
1307 nw = howmany(uap->nd, NFDBITS);
1308 ni = nw * sizeof(fd_mask);
1309
1310 /*
1311 * if the previously allocated space for the bits is smaller than
1312 * what is requested or no space has yet been allocated for this
1313 * thread, allocate enough space now.
1314 *
1315 * Note: If this process fails, select() will return EAGAIN; this
1316 * is the same thing pool() returns in a no-memory situation, but
1317 * it is not a POSIX compliant error code for select().
1318 */
1319 if (sel->nbytes >= (3 * ni)) {
1320 select_bzero_uthread_cache(sel);
1321 } else if (!select_grow_uthread_cache(sel, 3 * ni)) {
1322 return EAGAIN;
1323 }
1324
1325 /*
1326 * get the bits from the user address space
1327 */
1328 #define getbits(name, x) \
1329 (uap->name ? copyin(uap->name, &sel->ibits[(x) * nw], ni) : 0)
1330
1331 if ((error = getbits(in, 0))) {
1332 return error;
1333 }
1334 if ((error = getbits(ou, 1))) {
1335 return error;
1336 }
1337 if ((error = getbits(ex, 2))) {
1338 return error;
1339 }
1340 #undef getbits
1341
1342 if ((error = selcount(p, sel->ibits, uap->nd, &seldata->count))) {
1343 return error;
1344 }
1345
1346 if (uth->uu_selset == NULL) {
1347 uth->uu_selset = select_set_alloc();
1348 }
1349 return selprocess(p, 0, SEL_FIRSTPASS);
1350 }
1351
1352 static int
selcontinue(int error)1353 selcontinue(int error)
1354 {
1355 return selprocess(current_proc(), error, SEL_SECONDPASS);
1356 }
1357
1358
1359 /*
1360 * selprocess
1361 *
1362 * Parameters: error The error code from our caller
1363 * sel_pass The pass we are on
1364 */
1365 int
selprocess(struct proc * p,int error,int sel_pass)1366 selprocess(struct proc *p, int error, int sel_pass)
1367 {
1368 struct uthread *uth = current_uthread();
1369 struct _select *sel = &uth->uu_select;
1370 struct _select_data *seldata = &uth->uu_save.uus_select_data;
1371 struct select_nocancel_args *uap = seldata->args;
1372 int *retval = seldata->retval;
1373
1374 int unwind = 1;
1375 int prepost = 0;
1376 int somewakeup = 0;
1377 int doretry = 0;
1378 wait_result_t wait_result;
1379
1380 if ((error != 0) && (sel_pass == SEL_FIRSTPASS)) {
1381 unwind = 0;
1382 }
1383 if (seldata->count == 0) {
1384 unwind = 0;
1385 }
1386 retry:
1387 if (error != 0) {
1388 goto done;
1389 }
1390
1391 OSBitOrAtomic(P_SELECT, &p->p_flag);
1392
1393 /* skip scans if the select is just for timeouts */
1394 if (seldata->count) {
1395 error = selscan(p, sel, seldata, uap->nd, retval, sel_pass,
1396 uth->uu_selset);
1397 if (error || *retval) {
1398 goto done;
1399 }
1400 if (prepost || somewakeup) {
1401 /*
1402 * if the select of log, then we can wakeup and
1403 * discover some one else already read the data;
1404 * go to select again if time permits
1405 */
1406 prepost = 0;
1407 somewakeup = 0;
1408 doretry = 1;
1409 }
1410 }
1411
1412 if (uap->tv) {
1413 uint64_t now;
1414
1415 clock_get_uptime(&now);
1416 if (now >= seldata->abstime) {
1417 goto done;
1418 }
1419 }
1420
1421 if (doretry) {
1422 /* cleanup obits and try again */
1423 doretry = 0;
1424 sel_pass = SEL_FIRSTPASS;
1425 goto retry;
1426 }
1427
1428 /*
1429 * To effect a poll, the timeout argument should be
1430 * non-nil, pointing to a zero-valued timeval structure.
1431 */
1432 if (uap->tv && seldata->abstime == 0) {
1433 goto done;
1434 }
1435
1436 /* No spurious wakeups due to colls,no need to check for them */
1437 if ((sel_pass == SEL_SECONDPASS) || ((p->p_flag & P_SELECT) == 0)) {
1438 sel_pass = SEL_FIRSTPASS;
1439 goto retry;
1440 }
1441
1442 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1443
1444 /* if the select is just for timeout skip check */
1445 if (seldata->count && (sel_pass == SEL_SECONDPASS)) {
1446 panic("selprocess: 2nd pass assertwaiting");
1447 }
1448
1449 wait_result = waitq_assert_wait64_leeway(uth->uu_selset,
1450 NO_EVENT64, THREAD_ABORTSAFE,
1451 TIMEOUT_URGENCY_USER_NORMAL,
1452 seldata->abstime,
1453 TIMEOUT_NO_LEEWAY);
1454 if (wait_result != THREAD_AWAKENED) {
1455 /* there are no preposted events */
1456 error = tsleep1(NULL, PSOCK | PCATCH,
1457 "select", 0, selcontinue);
1458 } else {
1459 prepost = 1;
1460 error = 0;
1461 }
1462
1463 if (error == 0) {
1464 sel_pass = SEL_SECONDPASS;
1465 if (!prepost) {
1466 somewakeup = 1;
1467 }
1468 goto retry;
1469 }
1470 done:
1471 if (unwind) {
1472 seldrop(p, sel->ibits, uap->nd, seldata->count);
1473 select_set_reset(uth->uu_selset);
1474 }
1475 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1476 /* select is not restarted after signals... */
1477 if (error == ERESTART) {
1478 error = EINTR;
1479 }
1480 if (error == EWOULDBLOCK) {
1481 error = 0;
1482 }
1483
1484 if (error == 0) {
1485 uint32_t nw = howmany(uap->nd, NFDBITS);
1486 uint32_t ni = nw * sizeof(fd_mask);
1487
1488 #define putbits(name, x) \
1489 (uap->name ? copyout(&sel->obits[(x) * nw], uap->name, ni) : 0)
1490 int e0 = putbits(in, 0);
1491 int e1 = putbits(ou, 1);
1492 int e2 = putbits(ex, 2);
1493
1494 error = e0 ?: e1 ?: e2;
1495 #undef putbits
1496 }
1497
1498 if (error != EINTR && sel_pass == SEL_SECONDPASS && uth->uu_flag & UT_SAS_OLDMASK) {
1499 /* restore signal mask - continuation case */
1500 uth->uu_sigmask = uth->uu_oldmask;
1501 uth->uu_oldmask = 0;
1502 uth->uu_flag &= ~UT_SAS_OLDMASK;
1503 }
1504
1505 return error;
1506 }
1507
1508
1509 /**
1510 * remove the fileproc's underlying waitq from the supplied waitq set;
1511 * clear FP_INSELECT when appropriate
1512 *
1513 * Parameters:
1514 * fp File proc that is potentially currently in select
1515 * selset Waitq set to which the fileproc may belong
1516 * (usually this is the thread's private waitq set)
1517 * Conditions:
1518 * proc_fdlock is held
1519 */
1520 static void
selunlinkfp(struct fileproc * fp,struct select_set * selset)1521 selunlinkfp(struct fileproc *fp, struct select_set *selset)
1522 {
1523 if (fp->fp_flags & FP_INSELECT) {
1524 if (fp->fp_guard_attrs) {
1525 if (fp->fp_guard->fpg_wset == selset) {
1526 fp->fp_guard->fpg_wset = NULL;
1527 fp->fp_flags &= ~FP_INSELECT;
1528 }
1529 } else {
1530 if (fp->fp_wset == selset) {
1531 fp->fp_wset = NULL;
1532 fp->fp_flags &= ~FP_INSELECT;
1533 }
1534 }
1535 }
1536 }
1537
1538 /**
1539 * connect a fileproc to the given selset, potentially bridging to a waitq
1540 * pointed to indirectly by wq_data
1541 *
1542 * Parameters:
1543 * fp File proc potentially currently in select
1544 * selset Waitq set to which the fileproc should now belong
1545 * (usually this is the thread's private waitq set)
1546 *
1547 * Conditions:
1548 * proc_fdlock is held
1549 */
1550 static void
sellinkfp(struct fileproc * fp,struct select_set * selset,waitq_link_t * linkp)1551 sellinkfp(struct fileproc *fp, struct select_set *selset, waitq_link_t *linkp)
1552 {
1553 if ((fp->fp_flags & FP_INSELECT) == 0) {
1554 if (fp->fp_guard_attrs) {
1555 fp->fp_guard->fpg_wset = selset;
1556 } else {
1557 fp->fp_wset = selset;
1558 }
1559 fp->fp_flags |= FP_INSELECT;
1560 } else {
1561 fp->fp_flags |= FP_SELCONFLICT;
1562 if (linkp->wqlh == NULL) {
1563 *linkp = waitq_link_alloc(WQT_SELECT_SET);
1564 }
1565 select_set_link(&select_conflict_queue, selset, linkp);
1566 }
1567 }
1568
1569
1570 /*
1571 * selscan
1572 *
1573 * Parameters: p Process performing the select
1574 * sel The per-thread select context structure
1575 * nfd The number of file descriptors to scan
1576 * retval The per thread system call return area
1577 * sel_pass Which pass this is; allowed values are
1578 * SEL_FIRSTPASS and SEL_SECONDPASS
1579 * selset The per thread wait queue set
1580 *
1581 * Returns: 0 Success
1582 * EIO Invalid p->p_fd field XXX Obsolete?
1583 * EBADF One of the files in the bit vector is
1584 * invalid.
1585 */
1586 static int
selscan(struct proc * p,struct _select * sel,struct _select_data * seldata,int nfd,int32_t * retval,int sel_pass,struct select_set * selset)1587 selscan(struct proc *p, struct _select *sel, struct _select_data * seldata,
1588 int nfd, int32_t *retval, int sel_pass, struct select_set *selset)
1589 {
1590 int msk, i, j, fd;
1591 u_int32_t bits;
1592 struct fileproc *fp;
1593 int n = 0; /* count of bits */
1594 int nc = 0; /* bit vector offset (nc'th bit) */
1595 static int flag[3] = { FREAD, FWRITE, 0 };
1596 u_int32_t *iptr, *optr;
1597 u_int nw;
1598 u_int32_t *ibits, *obits;
1599 int count;
1600 struct vfs_context context = {
1601 .vc_thread = current_thread(),
1602 };
1603 waitq_link_t link = WQL_NULL;
1604 void *s_data;
1605
1606 ibits = sel->ibits;
1607 obits = sel->obits;
1608
1609 nw = howmany(nfd, NFDBITS);
1610
1611 count = seldata->count;
1612
1613 nc = 0;
1614 if (!count) {
1615 *retval = 0;
1616 return 0;
1617 }
1618
1619 if (sel_pass == SEL_FIRSTPASS) {
1620 /*
1621 * Make sure the waitq-set is all clean:
1622 *
1623 * select loops until it finds at least one event, however it
1624 * doesn't mean that the event that woke up select is still
1625 * fired by the time the second pass runs, and then
1626 * select_internal will loop back to a first pass.
1627 */
1628 select_set_reset(selset);
1629 s_data = &link;
1630 } else {
1631 s_data = NULL;
1632 }
1633
1634 proc_fdlock(p);
1635 for (msk = 0; msk < 3; msk++) {
1636 iptr = (u_int32_t *)&ibits[msk * nw];
1637 optr = (u_int32_t *)&obits[msk * nw];
1638
1639 for (i = 0; i < nfd; i += NFDBITS) {
1640 bits = iptr[i / NFDBITS];
1641
1642 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
1643 bits &= ~(1U << j);
1644
1645 fp = fp_get_noref_locked(p, fd);
1646 if (fp == NULL) {
1647 /*
1648 * If we abort because of a bad
1649 * fd, let the caller unwind...
1650 */
1651 proc_fdunlock(p);
1652 return EBADF;
1653 }
1654 if (sel_pass == SEL_SECONDPASS) {
1655 selunlinkfp(fp, selset);
1656 } else if (link.wqlh == NULL) {
1657 link = waitq_link_alloc(WQT_SELECT_SET);
1658 }
1659
1660 context.vc_ucred = fp->f_cred;
1661
1662 /* The select; set the bit, if true */
1663 if (fo_select(fp, flag[msk], s_data, &context)) {
1664 optr[fd / NFDBITS] |= (1U << (fd % NFDBITS));
1665 n++;
1666 }
1667 if (sel_pass == SEL_FIRSTPASS) {
1668 /*
1669 * Hook up the thread's waitq set either to
1670 * the fileproc structure, or to the global
1671 * conflict queue: but only on the first
1672 * select pass.
1673 */
1674 sellinkfp(fp, selset, &link);
1675 }
1676 nc++;
1677 }
1678 }
1679 }
1680 proc_fdunlock(p);
1681
1682 if (link.wqlh) {
1683 waitq_link_free(WQT_SELECT_SET, link);
1684 }
1685
1686 *retval = n;
1687 return 0;
1688 }
1689
1690 static int poll_callback(struct kevent_qos_s *, kevent_ctx_t);
1691
1692 int
poll(struct proc * p,struct poll_args * uap,int32_t * retval)1693 poll(struct proc *p, struct poll_args *uap, int32_t *retval)
1694 {
1695 __pthread_testcancel(1);
1696 return poll_nocancel(p, (struct poll_nocancel_args *)uap, retval);
1697 }
1698
1699
1700 int
poll_nocancel(struct proc * p,struct poll_nocancel_args * uap,int32_t * retval)1701 poll_nocancel(struct proc *p, struct poll_nocancel_args *uap, int32_t *retval)
1702 {
1703 struct pollfd *fds = NULL;
1704 struct kqueue *kq = NULL;
1705 int error = 0;
1706 u_int nfds = uap->nfds;
1707 u_int rfds = 0;
1708 rlim_t nofile = proc_limitgetcur(p, RLIMIT_NOFILE);
1709 size_t ni = nfds * sizeof(struct pollfd);
1710
1711 /*
1712 * This is kinda bogus. We have fd limits, but that is not
1713 * really related to the size of the pollfd array. Make sure
1714 * we let the process use at least FD_SETSIZE entries and at
1715 * least enough for the current limits. We want to be reasonably
1716 * safe, but not overly restrictive.
1717 */
1718 if (nfds > OPEN_MAX ||
1719 (nfds > nofile && (proc_suser(p) || nfds > FD_SETSIZE))) {
1720 return EINVAL;
1721 }
1722
1723 kq = kqueue_alloc(p);
1724 if (kq == NULL) {
1725 return EAGAIN;
1726 }
1727
1728 if (nfds) {
1729 fds = (struct pollfd *)kalloc_data(ni, Z_WAITOK);
1730 if (NULL == fds) {
1731 error = EAGAIN;
1732 goto out;
1733 }
1734
1735 error = copyin(uap->fds, fds, nfds * sizeof(struct pollfd));
1736 if (error) {
1737 goto out;
1738 }
1739 }
1740
1741 /* JMM - all this P_SELECT stuff is bogus */
1742 OSBitOrAtomic(P_SELECT, &p->p_flag);
1743 for (u_int i = 0; i < nfds; i++) {
1744 short events = fds[i].events;
1745 __assert_only int rc;
1746
1747 /* per spec, ignore fd values below zero */
1748 if (fds[i].fd < 0) {
1749 fds[i].revents = 0;
1750 continue;
1751 }
1752
1753 /* convert the poll event into a kqueue kevent */
1754 struct kevent_qos_s kev = {
1755 .ident = fds[i].fd,
1756 .flags = EV_ADD | EV_ONESHOT | EV_POLL,
1757 .udata = CAST_USER_ADDR_T(&fds[i])
1758 };
1759
1760 /* Handle input events */
1761 if (events & (POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND | POLLHUP)) {
1762 kev.filter = EVFILT_READ;
1763 if (events & (POLLPRI | POLLRDBAND)) {
1764 kev.flags |= EV_OOBAND;
1765 }
1766 rc = kevent_register(kq, &kev, NULL);
1767 assert((rc & FILTER_REGISTER_WAIT) == 0);
1768 }
1769
1770 /* Handle output events */
1771 if ((kev.flags & EV_ERROR) == 0 &&
1772 (events & (POLLOUT | POLLWRNORM | POLLWRBAND))) {
1773 kev.filter = EVFILT_WRITE;
1774 rc = kevent_register(kq, &kev, NULL);
1775 assert((rc & FILTER_REGISTER_WAIT) == 0);
1776 }
1777
1778 /* Handle BSD extension vnode events */
1779 if ((kev.flags & EV_ERROR) == 0 &&
1780 (events & (POLLEXTEND | POLLATTRIB | POLLNLINK | POLLWRITE))) {
1781 kev.filter = EVFILT_VNODE;
1782 kev.fflags = 0;
1783 if (events & POLLEXTEND) {
1784 kev.fflags |= NOTE_EXTEND;
1785 }
1786 if (events & POLLATTRIB) {
1787 kev.fflags |= NOTE_ATTRIB;
1788 }
1789 if (events & POLLNLINK) {
1790 kev.fflags |= NOTE_LINK;
1791 }
1792 if (events & POLLWRITE) {
1793 kev.fflags |= NOTE_WRITE;
1794 }
1795 rc = kevent_register(kq, &kev, NULL);
1796 assert((rc & FILTER_REGISTER_WAIT) == 0);
1797 }
1798
1799 if (kev.flags & EV_ERROR) {
1800 fds[i].revents = POLLNVAL;
1801 rfds++;
1802 } else {
1803 fds[i].revents = 0;
1804 }
1805 }
1806
1807 /*
1808 * Did we have any trouble registering?
1809 * If user space passed 0 FDs, then respect any timeout value passed.
1810 * This is an extremely inefficient sleep. If user space passed one or
1811 * more FDs, and we had trouble registering _all_ of them, then bail
1812 * out. If a subset of the provided FDs failed to register, then we
1813 * will still call the kqueue_scan function.
1814 */
1815 if (nfds && (rfds == nfds)) {
1816 goto done;
1817 }
1818
1819 /* scan for, and possibly wait for, the kevents to trigger */
1820 kevent_ctx_t kectx = kevent_get_context(current_thread());
1821 *kectx = (struct kevent_ctx_s){
1822 .kec_process_noutputs = rfds,
1823 .kec_process_flags = KEVENT_FLAG_POLL,
1824 .kec_deadline = 0, /* wait forever */
1825 };
1826
1827 /*
1828 * If any events have trouble registering, an event has fired and we
1829 * shouldn't wait for events in kqueue_scan.
1830 */
1831 if (rfds) {
1832 kectx->kec_process_flags |= KEVENT_FLAG_IMMEDIATE;
1833 } else if (uap->timeout != -1) {
1834 clock_interval_to_deadline(uap->timeout, NSEC_PER_MSEC,
1835 &kectx->kec_deadline);
1836 }
1837
1838 error = kqueue_scan(kq, kectx->kec_process_flags, kectx, poll_callback);
1839 rfds = kectx->kec_process_noutputs;
1840
1841 done:
1842 OSBitAndAtomic(~((uint32_t)P_SELECT), &p->p_flag);
1843 /* poll is not restarted after signals... */
1844 if (error == ERESTART) {
1845 error = EINTR;
1846 }
1847 if (error == 0) {
1848 error = copyout(fds, uap->fds, nfds * sizeof(struct pollfd));
1849 *retval = rfds;
1850 }
1851
1852 out:
1853 kfree_data(fds, ni);
1854
1855 kqueue_dealloc(kq);
1856 return error;
1857 }
1858
1859 static int
poll_callback(struct kevent_qos_s * kevp,kevent_ctx_t kectx)1860 poll_callback(struct kevent_qos_s *kevp, kevent_ctx_t kectx)
1861 {
1862 struct pollfd *fds = CAST_DOWN(struct pollfd *, kevp->udata);
1863 short prev_revents = fds->revents;
1864 short mask = 0;
1865
1866 /* convert the results back into revents */
1867 if (kevp->flags & EV_EOF) {
1868 fds->revents |= POLLHUP;
1869 }
1870 if (kevp->flags & EV_ERROR) {
1871 fds->revents |= POLLERR;
1872 }
1873
1874 switch (kevp->filter) {
1875 case EVFILT_READ:
1876 if (fds->revents & POLLHUP) {
1877 mask = (POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND);
1878 } else {
1879 mask = (POLLIN | POLLRDNORM);
1880 if (kevp->flags & EV_OOBAND) {
1881 mask |= (POLLPRI | POLLRDBAND);
1882 }
1883 }
1884 fds->revents |= (fds->events & mask);
1885 break;
1886
1887 case EVFILT_WRITE:
1888 if (!(fds->revents & POLLHUP)) {
1889 fds->revents |= (fds->events & (POLLOUT | POLLWRNORM | POLLWRBAND));
1890 }
1891 break;
1892
1893 case EVFILT_VNODE:
1894 if (kevp->fflags & NOTE_EXTEND) {
1895 fds->revents |= (fds->events & POLLEXTEND);
1896 }
1897 if (kevp->fflags & NOTE_ATTRIB) {
1898 fds->revents |= (fds->events & POLLATTRIB);
1899 }
1900 if (kevp->fflags & NOTE_LINK) {
1901 fds->revents |= (fds->events & POLLNLINK);
1902 }
1903 if (kevp->fflags & NOTE_WRITE) {
1904 fds->revents |= (fds->events & POLLWRITE);
1905 }
1906 break;
1907 }
1908
1909 if (fds->revents != 0 && prev_revents == 0) {
1910 kectx->kec_process_noutputs++;
1911 }
1912
1913 return 0;
1914 }
1915
1916 int
seltrue(__unused dev_t dev,__unused int flag,__unused struct proc * p)1917 seltrue(__unused dev_t dev, __unused int flag, __unused struct proc *p)
1918 {
1919 return 1;
1920 }
1921
1922 /*
1923 * selcount
1924 *
1925 * Count the number of bits set in the input bit vector, and establish an
1926 * outstanding fp->fp_iocount for each of the descriptors which will be in
1927 * use in the select operation.
1928 *
1929 * Parameters: p The process doing the select
1930 * ibits The input bit vector
1931 * nfd The number of fd's in the vector
1932 * countp Pointer to where to store the bit count
1933 *
1934 * Returns: 0 Success
1935 * EIO Bad per process open file table
1936 * EBADF One of the bits in the input bit vector
1937 * references an invalid fd
1938 *
1939 * Implicit: *countp (modified) Count of fd's
1940 *
1941 * Notes: This function is the first pass under the proc_fdlock() that
1942 * permits us to recognize invalid descriptors in the bit vector;
1943 * the may, however, not remain valid through the drop and
1944 * later reacquisition of the proc_fdlock().
1945 */
1946 static int
selcount(struct proc * p,u_int32_t * ibits,int nfd,int * countp)1947 selcount(struct proc *p, u_int32_t *ibits, int nfd, int *countp)
1948 {
1949 int msk, i, j, fd;
1950 u_int32_t bits;
1951 struct fileproc *fp;
1952 int n = 0;
1953 u_int32_t *iptr;
1954 u_int nw;
1955 int error = 0;
1956 int need_wakeup = 0;
1957
1958 nw = howmany(nfd, NFDBITS);
1959
1960 proc_fdlock(p);
1961 for (msk = 0; msk < 3; msk++) {
1962 iptr = (u_int32_t *)&ibits[msk * nw];
1963 for (i = 0; i < nfd; i += NFDBITS) {
1964 bits = iptr[i / NFDBITS];
1965 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
1966 bits &= ~(1U << j);
1967
1968 fp = fp_get_noref_locked(p, fd);
1969 if (fp == NULL) {
1970 *countp = 0;
1971 error = EBADF;
1972 goto bad;
1973 }
1974 os_ref_retain_locked(&fp->fp_iocount);
1975 n++;
1976 }
1977 }
1978 }
1979 proc_fdunlock(p);
1980
1981 *countp = n;
1982 return 0;
1983
1984 bad:
1985 if (n == 0) {
1986 goto out;
1987 }
1988 /* Ignore error return; it's already EBADF */
1989 (void)seldrop_locked(p, ibits, nfd, n, &need_wakeup);
1990
1991 out:
1992 proc_fdunlock(p);
1993 if (need_wakeup) {
1994 wakeup(&p->p_fd.fd_fpdrainwait);
1995 }
1996 return error;
1997 }
1998
1999
2000 /*
2001 * seldrop_locked
2002 *
2003 * Drop outstanding wait queue references set up during selscan(); drop the
2004 * outstanding per fileproc fp_iocount picked up during the selcount().
2005 *
2006 * Parameters: p Process performing the select
2007 * ibits Input bit bector of fd's
2008 * nfd Number of fd's
2009 * lim Limit to number of vector entries to
2010 * consider, or -1 for "all"
2011 * inselect True if
2012 * need_wakeup Pointer to flag to set to do a wakeup
2013 * if f_iocont on any descriptor goes to 0
2014 *
2015 * Returns: 0 Success
2016 * EBADF One or more fds in the bit vector
2017 * were invalid, but the rest
2018 * were successfully dropped
2019 *
2020 * Notes: An fd make become bad while the proc_fdlock() is not held,
2021 * if a multithreaded application closes the fd out from under
2022 * the in progress select. In this case, we still have to
2023 * clean up after the set up on the remaining fds.
2024 */
2025 static int
seldrop_locked(struct proc * p,u_int32_t * ibits,int nfd,int lim,int * need_wakeup)2026 seldrop_locked(struct proc *p, u_int32_t *ibits, int nfd, int lim, int *need_wakeup)
2027 {
2028 int msk, i, j, nc, fd;
2029 u_int32_t bits;
2030 struct fileproc *fp;
2031 u_int32_t *iptr;
2032 u_int nw;
2033 int error = 0;
2034 uthread_t uth = current_uthread();
2035 struct _select_data *seldata;
2036
2037 *need_wakeup = 0;
2038
2039 nw = howmany(nfd, NFDBITS);
2040 seldata = &uth->uu_save.uus_select_data;
2041
2042 nc = 0;
2043 for (msk = 0; msk < 3; msk++) {
2044 iptr = (u_int32_t *)&ibits[msk * nw];
2045 for (i = 0; i < nfd; i += NFDBITS) {
2046 bits = iptr[i / NFDBITS];
2047 while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
2048 bits &= ~(1U << j);
2049 /*
2050 * If we've already dropped as many as were
2051 * counted/scanned, then we are done.
2052 */
2053 if (nc >= lim) {
2054 goto done;
2055 }
2056
2057 /*
2058 * We took an I/O reference in selcount,
2059 * so the fp can't possibly be NULL.
2060 */
2061 fp = fp_get_noref_locked_with_iocount(p, fd);
2062 selunlinkfp(fp, uth->uu_selset);
2063
2064 nc++;
2065
2066 const os_ref_count_t refc = os_ref_release_locked(&fp->fp_iocount);
2067 if (0 == refc) {
2068 panic("fp_iocount overdecrement!");
2069 }
2070
2071 if (1 == refc) {
2072 /*
2073 * The last iocount is responsible for clearing
2074 * selconfict flag - even if we didn't set it -
2075 * and is also responsible for waking up anyone
2076 * waiting on iocounts to drain.
2077 */
2078 if (fp->fp_flags & FP_SELCONFLICT) {
2079 fp->fp_flags &= ~FP_SELCONFLICT;
2080 }
2081 if (p->p_fd.fd_fpdrainwait) {
2082 p->p_fd.fd_fpdrainwait = 0;
2083 *need_wakeup = 1;
2084 }
2085 }
2086 }
2087 }
2088 }
2089 done:
2090 return error;
2091 }
2092
2093
2094 static int
seldrop(struct proc * p,u_int32_t * ibits,int nfd,int lim)2095 seldrop(struct proc *p, u_int32_t *ibits, int nfd, int lim)
2096 {
2097 int error;
2098 int need_wakeup = 0;
2099
2100 proc_fdlock(p);
2101 error = seldrop_locked(p, ibits, nfd, lim, &need_wakeup);
2102 proc_fdunlock(p);
2103 if (need_wakeup) {
2104 wakeup(&p->p_fd.fd_fpdrainwait);
2105 }
2106 return error;
2107 }
2108
2109 /*
2110 * Record a select request.
2111 */
2112 void
selrecord(__unused struct proc * selector,struct selinfo * sip,void * s_data)2113 selrecord(__unused struct proc *selector, struct selinfo *sip, void *s_data)
2114 {
2115 struct select_set *selset = current_uthread()->uu_selset;
2116
2117 /* do not record if this is second pass of select */
2118 if (!s_data) {
2119 return;
2120 }
2121
2122 if (selset == SELSPEC_RECORD_MARKER) {
2123 /*
2124 * The kevent subsystem is trying to sniff
2125 * the selinfo::si_note to attach to.
2126 */
2127 ((selspec_record_hook_t)s_data)(sip);
2128 } else {
2129 waitq_link_t *linkp = s_data;
2130
2131 if (!waitq_is_valid(&sip->si_waitq)) {
2132 waitq_init(&sip->si_waitq, WQT_SELECT, SYNC_POLICY_FIFO);
2133 }
2134
2135 /* note: this checks for pre-existing linkage */
2136 select_set_link(&sip->si_waitq, selset, linkp);
2137 }
2138 }
2139
2140 static void
selwakeup_internal(struct selinfo * sip,long hint,wait_result_t wr)2141 selwakeup_internal(struct selinfo *sip, long hint, wait_result_t wr)
2142 {
2143 if (sip->si_flags & SI_SELSPEC) {
2144 /*
2145 * The "primitive" lock is held.
2146 * The knote lock is not held.
2147 *
2148 * All knotes will transition their kn_hook to NULL.
2149 */
2150 lck_spin_lock(&selspec_lock);
2151 KNOTE(&sip->si_note, hint);
2152 klist_init(&sip->si_note);
2153 lck_spin_unlock(&selspec_lock);
2154 sip->si_flags &= ~SI_SELSPEC;
2155 }
2156
2157 /*
2158 * After selrecord() has been called, selinfo owners must call
2159 * at least one of selwakeup() or selthreadclear().
2160 *
2161 * Use this opportunity to deinit the waitq
2162 * so that all linkages are garbage collected
2163 * in a combined wakeup-all + unlink + deinit call.
2164 */
2165 select_waitq_wakeup_and_deinit(&sip->si_waitq, NO_EVENT64, wr);
2166 }
2167
2168
2169 void
selwakeup(struct selinfo * sip)2170 selwakeup(struct selinfo *sip)
2171 {
2172 selwakeup_internal(sip, 0, THREAD_AWAKENED);
2173 }
2174
2175 void
selthreadclear(struct selinfo * sip)2176 selthreadclear(struct selinfo *sip)
2177 {
2178 selwakeup_internal(sip, NOTE_REVOKE, THREAD_RESTART);
2179 }
2180
2181
2182 /*
2183 * gethostuuid
2184 *
2185 * Description: Get the host UUID from IOKit and return it to user space.
2186 *
2187 * Parameters: uuid_buf Pointer to buffer to receive UUID
2188 * timeout Timespec for timout
2189 *
2190 * Returns: 0 Success
2191 * EWOULDBLOCK Timeout is too short
2192 * copyout:EFAULT Bad user buffer
2193 * mac_system_check_info:EPERM Client not allowed to perform this operation
2194 *
2195 * Notes: A timeout seems redundant, since if it's tolerable to not
2196 * have a system UUID in hand, then why ask for one?
2197 */
2198 int
gethostuuid(struct proc * p,struct gethostuuid_args * uap,__unused int32_t * retval)2199 gethostuuid(struct proc *p, struct gethostuuid_args *uap, __unused int32_t *retval)
2200 {
2201 kern_return_t kret;
2202 int error;
2203 mach_timespec_t mach_ts; /* for IOKit call */
2204 __darwin_uuid_t uuid_kern = {}; /* for IOKit call */
2205
2206 /* Check entitlement */
2207 if (!IOCurrentTaskHasEntitlement("com.apple.private.getprivatesysid")) {
2208 #if !defined(XNU_TARGET_OS_OSX)
2209 #if CONFIG_MACF
2210 if ((error = mac_system_check_info(kauth_cred_get(), "hw.uuid")) != 0) {
2211 /* EPERM invokes userspace upcall if present */
2212 return error;
2213 }
2214 #endif
2215 #endif
2216 }
2217
2218 /* Convert the 32/64 bit timespec into a mach_timespec_t */
2219 if (proc_is64bit(p)) {
2220 struct user64_timespec ts;
2221 error = copyin(uap->timeoutp, &ts, sizeof(ts));
2222 if (error) {
2223 return error;
2224 }
2225 mach_ts.tv_sec = (unsigned int)ts.tv_sec;
2226 mach_ts.tv_nsec = (clock_res_t)ts.tv_nsec;
2227 } else {
2228 struct user32_timespec ts;
2229 error = copyin(uap->timeoutp, &ts, sizeof(ts));
2230 if (error) {
2231 return error;
2232 }
2233 mach_ts.tv_sec = ts.tv_sec;
2234 mach_ts.tv_nsec = ts.tv_nsec;
2235 }
2236
2237 /* Call IOKit with the stack buffer to get the UUID */
2238 kret = IOBSDGetPlatformUUID(uuid_kern, mach_ts);
2239
2240 /*
2241 * If we get it, copy out the data to the user buffer; note that a
2242 * uuid_t is an array of characters, so this is size invariant for
2243 * 32 vs. 64 bit.
2244 */
2245 if (kret == KERN_SUCCESS) {
2246 error = copyout(uuid_kern, uap->uuid_buf, sizeof(uuid_kern));
2247 } else {
2248 error = EWOULDBLOCK;
2249 }
2250
2251 return error;
2252 }
2253
2254 /*
2255 * ledger
2256 *
2257 * Description: Omnibus system call for ledger operations
2258 */
2259 int
ledger(struct proc * p,struct ledger_args * args,__unused int32_t * retval)2260 ledger(struct proc *p, struct ledger_args *args, __unused int32_t *retval)
2261 {
2262 #if !CONFIG_MACF
2263 #pragma unused(p)
2264 #endif
2265 int rval, pid, len, error;
2266 #ifdef LEDGER_DEBUG
2267 struct ledger_limit_args lla;
2268 #endif
2269 task_t task;
2270 proc_t proc;
2271
2272 /* Finish copying in the necessary args before taking the proc lock */
2273 error = 0;
2274 len = 0;
2275 if (args->cmd == LEDGER_ENTRY_INFO) {
2276 error = copyin(args->arg3, (char *)&len, sizeof(len));
2277 } else if (args->cmd == LEDGER_TEMPLATE_INFO) {
2278 error = copyin(args->arg2, (char *)&len, sizeof(len));
2279 } else if (args->cmd == LEDGER_LIMIT)
2280 #ifdef LEDGER_DEBUG
2281 { error = copyin(args->arg2, (char *)&lla, sizeof(lla));}
2282 #else
2283 { return EINVAL; }
2284 #endif
2285 else if ((args->cmd < 0) || (args->cmd > LEDGER_MAX_CMD)) {
2286 return EINVAL;
2287 }
2288
2289 if (error) {
2290 return error;
2291 }
2292 if (len < 0) {
2293 return EINVAL;
2294 }
2295
2296 rval = 0;
2297 if (args->cmd != LEDGER_TEMPLATE_INFO) {
2298 pid = (int)args->arg1;
2299 proc = proc_find(pid);
2300 if (proc == NULL) {
2301 return ESRCH;
2302 }
2303
2304 #if CONFIG_MACF
2305 error = mac_proc_check_ledger(p, proc, args->cmd);
2306 if (error) {
2307 proc_rele(proc);
2308 return error;
2309 }
2310 #endif
2311
2312 task = proc_task(proc);
2313 }
2314
2315 switch (args->cmd) {
2316 #ifdef LEDGER_DEBUG
2317 case LEDGER_LIMIT: {
2318 if (!kauth_cred_issuser(kauth_cred_get())) {
2319 rval = EPERM;
2320 }
2321 rval = ledger_limit(task, &lla);
2322 proc_rele(proc);
2323 break;
2324 }
2325 #endif
2326 case LEDGER_INFO: {
2327 struct ledger_info info = {};
2328
2329 rval = ledger_info(task, &info);
2330 proc_rele(proc);
2331 if (rval == 0) {
2332 rval = copyout(&info, args->arg2,
2333 sizeof(info));
2334 }
2335 break;
2336 }
2337
2338 case LEDGER_ENTRY_INFO: {
2339 void *buf;
2340 int sz;
2341
2342 #if CONFIG_MEMORYSTATUS
2343 task_ledger_settle_dirty_time(task);
2344 #endif /* CONFIG_MEMORYSTATUS */
2345
2346 rval = ledger_get_task_entry_info_multiple(task, &buf, &len);
2347 proc_rele(proc);
2348 if ((rval == 0) && (len >= 0)) {
2349 sz = len * sizeof(struct ledger_entry_info);
2350 rval = copyout(buf, args->arg2, sz);
2351 kfree_data(buf, sz);
2352 }
2353 if (rval == 0) {
2354 rval = copyout(&len, args->arg3, sizeof(len));
2355 }
2356 break;
2357 }
2358
2359 case LEDGER_TEMPLATE_INFO: {
2360 void *buf;
2361 int sz;
2362
2363 rval = ledger_template_info(&buf, &len);
2364 if ((rval == 0) && (len >= 0)) {
2365 sz = len * sizeof(struct ledger_template_info);
2366 rval = copyout(buf, args->arg1, sz);
2367 kfree_data(buf, sz);
2368 }
2369 if (rval == 0) {
2370 rval = copyout(&len, args->arg2, sizeof(len));
2371 }
2372 break;
2373 }
2374
2375 default:
2376 panic("ledger syscall logic error -- command type %d", args->cmd);
2377 proc_rele(proc);
2378 rval = EINVAL;
2379 }
2380
2381 return rval;
2382 }
2383
2384 int
telemetry(__unused struct proc * p,struct telemetry_args * args,__unused int32_t * retval)2385 telemetry(__unused struct proc *p, struct telemetry_args *args, __unused int32_t *retval)
2386 {
2387 int error = 0;
2388
2389 switch (args->cmd) {
2390 #if CONFIG_TELEMETRY
2391 case TELEMETRY_CMD_TIMER_EVENT:
2392 error = telemetry_timer_event(args->deadline, args->interval, args->leeway);
2393 break;
2394 case TELEMETRY_CMD_PMI_SETUP:
2395 error = telemetry_pmi_setup((enum telemetry_pmi)args->deadline, args->interval);
2396 break;
2397 #endif /* CONFIG_TELEMETRY */
2398 case TELEMETRY_CMD_VOUCHER_NAME:
2399 if (thread_set_voucher_name((mach_port_name_t)args->deadline)) {
2400 error = EINVAL;
2401 }
2402 break;
2403
2404 default:
2405 error = EINVAL;
2406 break;
2407 }
2408
2409 return error;
2410 }
2411
2412 /*
2413 * Logging
2414 *
2415 * Description: syscall to access kernel logging from userspace
2416 *
2417 * Args:
2418 * tag - used for syncing with userspace on the version.
2419 * flags - flags used by the syscall.
2420 * buffer - userspace address of string to copy.
2421 * size - size of buffer.
2422 */
2423 int
log_data(__unused struct proc * p,struct log_data_args * args,int * retval)2424 log_data(__unused struct proc *p, struct log_data_args *args, int *retval)
2425 {
2426 unsigned int tag = args->tag;
2427 unsigned int flags = args->flags;
2428 user_addr_t buffer = args->buffer;
2429 unsigned int size = args->size;
2430 int ret = 0;
2431 *retval = 0;
2432
2433 /* Only DEXTs are suppose to use this syscall. */
2434 if (!task_is_driver(current_task())) {
2435 return EPERM;
2436 }
2437
2438 /*
2439 * Tag synchronize the syscall version with userspace.
2440 * Tag == 0 => flags == OS_LOG_TYPE
2441 */
2442 if (tag != 0) {
2443 return EINVAL;
2444 }
2445
2446 /*
2447 * OS_LOG_TYPE are defined in libkern/os/log.h
2448 * In userspace they are defined in libtrace/os/log.h
2449 */
2450 if (flags != OS_LOG_TYPE_DEFAULT &&
2451 flags != OS_LOG_TYPE_INFO &&
2452 flags != OS_LOG_TYPE_DEBUG &&
2453 flags != OS_LOG_TYPE_ERROR &&
2454 flags != OS_LOG_TYPE_FAULT) {
2455 return EINVAL;
2456 }
2457
2458 if (size == 0) {
2459 return EINVAL;
2460 }
2461
2462 /* truncate to OS_LOG_DATA_MAX_SIZE */
2463 if (size > OS_LOG_DATA_MAX_SIZE) {
2464 printf("%s: WARNING msg is going to be truncated from %u to %u\n",
2465 __func__, size, OS_LOG_DATA_MAX_SIZE);
2466 size = OS_LOG_DATA_MAX_SIZE;
2467 }
2468
2469 char *log_msg = (char *)kalloc_data(size, Z_WAITOK);
2470 if (!log_msg) {
2471 return ENOMEM;
2472 }
2473
2474 if (copyin(buffer, log_msg, size) != 0) {
2475 ret = EFAULT;
2476 goto out;
2477 }
2478 log_msg[size - 1] = '\0';
2479
2480 /*
2481 * This will log to dmesg and logd.
2482 * The call will fail if the current
2483 * process is not a driverKit process.
2484 */
2485 os_log_driverKit(&ret, OS_LOG_DEFAULT, (os_log_type_t)flags, "%s", log_msg);
2486
2487 out:
2488 if (log_msg != NULL) {
2489 kfree_data(log_msg, size);
2490 }
2491
2492 return ret;
2493 }
2494
2495 #if DEVELOPMENT || DEBUG
2496
2497 static int
2498 sysctl_mpsc_test_pingpong SYSCTL_HANDLER_ARGS
2499 {
2500 #pragma unused(oidp, arg1, arg2)
2501 uint64_t value = 0;
2502 int error;
2503
2504 error = SYSCTL_IN(req, &value, sizeof(value));
2505 if (error) {
2506 return error;
2507 }
2508
2509 if (error == 0 && req->newptr) {
2510 error = mpsc_test_pingpong(value, &value);
2511 if (error == 0) {
2512 error = SYSCTL_OUT(req, &value, sizeof(value));
2513 }
2514 }
2515
2516 return error;
2517 }
2518 SYSCTL_PROC(_kern, OID_AUTO, mpsc_test_pingpong, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
2519 0, 0, sysctl_mpsc_test_pingpong, "Q", "MPSC tests: pingpong");
2520
2521 #endif /* DEVELOPMENT || DEBUG */
2522
2523 /* Telemetry, microstackshots */
2524
2525 SYSCTL_NODE(_kern, OID_AUTO, microstackshot, CTLFLAG_RD | CTLFLAG_LOCKED, 0,
2526 "microstackshot info");
2527
2528 extern uint32_t telemetry_sample_rate;
2529 SYSCTL_UINT(_kern_microstackshot, OID_AUTO, interrupt_sample_rate,
2530 CTLFLAG_RD | CTLFLAG_LOCKED, &telemetry_sample_rate, 0,
2531 "interrupt-based sampling rate in Hz");
2532
2533 #if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES)
2534
2535 extern uint64_t mt_microstackshot_period;
2536 SYSCTL_QUAD(_kern_microstackshot, OID_AUTO, pmi_sample_period,
2537 CTLFLAG_RD | CTLFLAG_LOCKED, &mt_microstackshot_period,
2538 "PMI sampling rate");
2539 extern unsigned int mt_microstackshot_ctr;
2540 SYSCTL_UINT(_kern_microstackshot, OID_AUTO, pmi_sample_counter,
2541 CTLFLAG_RD | CTLFLAG_LOCKED, &mt_microstackshot_ctr, 0,
2542 "PMI counter");
2543
2544 #endif /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */
2545
2546 /*Remote Time api*/
2547 SYSCTL_NODE(_machdep, OID_AUTO, remotetime, CTLFLAG_RD | CTLFLAG_LOCKED, 0, "Remote time api");
2548
2549 #if DEVELOPMENT || DEBUG
2550 #if CONFIG_MACH_BRIDGE_SEND_TIME
2551 extern _Atomic uint32_t bt_init_flag;
2552 extern uint32_t mach_bridge_timer_enable(uint32_t, int);
2553
2554 SYSCTL_INT(_machdep_remotetime, OID_AUTO, bridge_timer_init_flag,
2555 CTLFLAG_RD | CTLFLAG_LOCKED, &bt_init_flag, 0, "");
2556
2557 static int sysctl_mach_bridge_timer_enable SYSCTL_HANDLER_ARGS
2558 {
2559 #pragma unused(oidp, arg1, arg2)
2560 uint32_t value = 0;
2561 int error = 0;
2562 /* User is querying buffer size */
2563 if (req->oldptr == USER_ADDR_NULL && req->newptr == USER_ADDR_NULL) {
2564 req->oldidx = sizeof(value);
2565 return 0;
2566 }
2567 if (os_atomic_load(&bt_init_flag, acquire)) {
2568 if (req->newptr) {
2569 int new_value = 0;
2570 error = SYSCTL_IN(req, &new_value, sizeof(new_value));
2571 if (error) {
2572 return error;
2573 }
2574 if (new_value == 0 || new_value == 1) {
2575 value = mach_bridge_timer_enable(new_value, 1);
2576 } else {
2577 return EPERM;
2578 }
2579 } else {
2580 value = mach_bridge_timer_enable(0, 0);
2581 }
2582 }
2583 error = SYSCTL_OUT(req, &value, sizeof(value));
2584 return error;
2585 }
2586
2587 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, bridge_timer_enable,
2588 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2589 0, 0, sysctl_mach_bridge_timer_enable, "I", "");
2590
2591 #endif /* CONFIG_MACH_BRIDGE_SEND_TIME */
2592
2593 static int sysctl_mach_bridge_remote_time SYSCTL_HANDLER_ARGS
2594 {
2595 #pragma unused(oidp, arg1, arg2)
2596 uint64_t ltime = 0, rtime = 0;
2597 if (req->oldptr == USER_ADDR_NULL) {
2598 req->oldidx = sizeof(rtime);
2599 return 0;
2600 }
2601 if (req->newptr) {
2602 int error = SYSCTL_IN(req, <ime, sizeof(ltime));
2603 if (error) {
2604 return error;
2605 }
2606 }
2607 rtime = mach_bridge_remote_time(ltime);
2608 return SYSCTL_OUT(req, &rtime, sizeof(rtime));
2609 }
2610 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, mach_bridge_remote_time,
2611 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
2612 0, 0, sysctl_mach_bridge_remote_time, "Q", "");
2613
2614 #endif /* DEVELOPMENT || DEBUG */
2615
2616 #if CONFIG_MACH_BRIDGE_RECV_TIME
2617 extern struct bt_params bt_params_get_latest(void);
2618
2619 static int sysctl_mach_bridge_conversion_params SYSCTL_HANDLER_ARGS
2620 {
2621 #pragma unused(oidp, arg1, arg2)
2622 struct bt_params params = {};
2623 if (req->oldptr == USER_ADDR_NULL) {
2624 req->oldidx = sizeof(struct bt_params);
2625 return 0;
2626 }
2627 if (req->newptr) {
2628 return EPERM;
2629 }
2630 params = bt_params_get_latest();
2631 return SYSCTL_OUT(req, ¶ms, MIN(sizeof(params), req->oldlen));
2632 }
2633
2634 SYSCTL_PROC(_machdep_remotetime, OID_AUTO, conversion_params,
2635 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0,
2636 0, sysctl_mach_bridge_conversion_params, "S,bt_params", "");
2637
2638 #endif /* CONFIG_MACH_BRIDGE_RECV_TIME */
2639
2640 #if DEVELOPMENT || DEBUG
2641
2642 #include <pexpert/pexpert.h>
2643 extern int32_t sysctl_get_bound_cpuid(void);
2644 extern kern_return_t sysctl_thread_bind_cpuid(int32_t cpuid);
2645 static int
2646 sysctl_kern_sched_thread_bind_cpu SYSCTL_HANDLER_ARGS
2647 {
2648 #pragma unused(oidp, arg1, arg2)
2649
2650 /*
2651 * DO NOT remove this bootarg guard or make this non-development.
2652 * This kind of binding should only be used for tests and
2653 * experiments in a custom configuration, never shipping code.
2654 */
2655
2656 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2657 return ENOENT;
2658 }
2659
2660 int32_t cpuid = sysctl_get_bound_cpuid();
2661
2662 int32_t new_value;
2663 int changed;
2664 int error = sysctl_io_number(req, cpuid, sizeof cpuid, &new_value, &changed);
2665 if (error) {
2666 return error;
2667 }
2668
2669 if (changed) {
2670 kern_return_t kr = sysctl_thread_bind_cpuid(new_value);
2671
2672 if (kr == KERN_NOT_SUPPORTED) {
2673 return ENOTSUP;
2674 }
2675
2676 if (kr == KERN_INVALID_VALUE) {
2677 return ERANGE;
2678 }
2679 }
2680
2681 return error;
2682 }
2683
2684 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cpu, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2685 0, 0, sysctl_kern_sched_thread_bind_cpu, "I", "");
2686
2687 #if __AMP__
2688 extern char sysctl_get_bound_cluster_type(void);
2689 extern void sysctl_thread_bind_cluster_type(char cluster_type);
2690 static int
2691 sysctl_kern_sched_thread_bind_cluster_type SYSCTL_HANDLER_ARGS
2692 {
2693 #pragma unused(oidp, arg1, arg2)
2694 char buff[4];
2695
2696 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2697 return ENOENT;
2698 }
2699
2700 int error = SYSCTL_IN(req, buff, 1);
2701 if (error) {
2702 return error;
2703 }
2704 char cluster_type = buff[0];
2705
2706 if (!req->newptr) {
2707 goto out;
2708 }
2709
2710 sysctl_thread_bind_cluster_type(cluster_type);
2711 out:
2712 cluster_type = sysctl_get_bound_cluster_type();
2713 buff[0] = cluster_type;
2714
2715 return SYSCTL_OUT(req, buff, 1);
2716 }
2717
2718 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cluster_type, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
2719 0, 0, sysctl_kern_sched_thread_bind_cluster_type, "A", "");
2720
2721 extern char sysctl_get_task_cluster_type(void);
2722 extern void sysctl_task_set_cluster_type(char cluster_type);
2723 static int
2724 sysctl_kern_sched_task_set_cluster_type SYSCTL_HANDLER_ARGS
2725 {
2726 #pragma unused(oidp, arg1, arg2)
2727 char buff[4];
2728
2729 if (!PE_parse_boot_argn("enable_skstsct", NULL, 0)) {
2730 return ENOENT;
2731 }
2732
2733 int error = SYSCTL_IN(req, buff, 1);
2734 if (error) {
2735 return error;
2736 }
2737 char cluster_type = buff[0];
2738
2739 if (!req->newptr) {
2740 goto out;
2741 }
2742
2743 sysctl_task_set_cluster_type(cluster_type);
2744 out:
2745 cluster_type = sysctl_get_task_cluster_type();
2746 buff[0] = cluster_type;
2747
2748 return SYSCTL_OUT(req, buff, 1);
2749 }
2750
2751 SYSCTL_PROC(_kern, OID_AUTO, sched_task_set_cluster_type, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
2752 0, 0, sysctl_kern_sched_task_set_cluster_type, "A", "");
2753
2754 extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options);
2755 extern uint32_t thread_bound_cluster_id(thread_t);
2756 static int
2757 sysctl_kern_sched_thread_bind_cluster_id SYSCTL_HANDLER_ARGS
2758 {
2759 #pragma unused(oidp, arg1, arg2)
2760 if (!PE_parse_boot_argn("enable_skstb", NULL, 0)) {
2761 return ENOENT;
2762 }
2763
2764 thread_t self = current_thread();
2765 uint32_t old_value = thread_bound_cluster_id(self);
2766 uint32_t new_value;
2767
2768 int error = SYSCTL_IN(req, &new_value, sizeof(new_value));
2769 if (error) {
2770 return error;
2771 }
2772 if (new_value != old_value) {
2773 /*
2774 * This sysctl binds the thread to the cluster without any flags,
2775 * which means it will be hard bound and not check eligibility.
2776 */
2777 thread_bind_cluster_id(self, new_value, 0);
2778 }
2779 return SYSCTL_OUT(req, &old_value, sizeof(old_value));
2780 }
2781
2782 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_bind_cluster_id, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2783 0, 0, sysctl_kern_sched_thread_bind_cluster_id, "I", "");
2784
2785 #if CONFIG_SCHED_EDGE
2786
2787 extern int sched_edge_restrict_ut;
2788 SYSCTL_INT(_kern, OID_AUTO, sched_edge_restrict_ut, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_restrict_ut, 0, "Edge Scheduler Restrict UT Threads");
2789 extern int sched_edge_restrict_bg;
2790 SYSCTL_INT(_kern, OID_AUTO, sched_edge_restrict_bg, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_restrict_ut, 0, "Edge Scheduler Restrict BG Threads");
2791 extern int sched_edge_migrate_ipi_immediate;
2792 SYSCTL_INT(_kern, OID_AUTO, sched_edge_migrate_ipi_immediate, CTLFLAG_RW | CTLFLAG_LOCKED, &sched_edge_migrate_ipi_immediate, 0, "Edge Scheduler uses immediate IPIs for migration event based on execution latency");
2793
2794 #endif /* CONFIG_SCHED_EDGE */
2795
2796 #endif /* __AMP__ */
2797
2798 #if SCHED_HYGIENE_DEBUG
2799
2800 SYSCTL_QUAD(_kern, OID_AUTO, interrupt_masked_threshold_mt, CTLFLAG_RW | CTLFLAG_LOCKED,
2801 &interrupt_masked_timeout,
2802 "Interrupt masked duration after which a tracepoint is emitted or the device panics (in mach timebase units)");
2803
2804 SYSCTL_INT(_kern, OID_AUTO, interrupt_masked_debug_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
2805 &interrupt_masked_debug_mode, 0,
2806 "Enable interrupt masked tracing or panic (0: off, 1: trace, 2: panic)");
2807
2808 SYSCTL_QUAD(_kern, OID_AUTO, sched_preemption_disable_threshold_mt, CTLFLAG_RW | CTLFLAG_LOCKED,
2809 &sched_preemption_disable_threshold_mt,
2810 "Preemption disablement duration after which a tracepoint is emitted or the device panics (in mach timebase units)");
2811
2812 SYSCTL_INT(_kern, OID_AUTO, sched_preemption_disable_debug_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
2813 &sched_preemption_disable_debug_mode, 0,
2814 "Enable preemption disablement tracing or panic (0: off, 1: trace, 2: panic)");
2815
2816 PERCPU_DECL(uint64_t _Atomic, preemption_disable_max_mt);
2817
2818 static int
sysctl_sched_preemption_disable_stats(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)2819 sysctl_sched_preemption_disable_stats(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
2820 {
2821 uint64_t stats[MAX_CPUS]; // maximum per CPU
2822
2823 /*
2824 * No synchronization here. The individual values are pretty much
2825 * independent, and reading/writing them is atomic.
2826 */
2827
2828 int cpu = 0;
2829 percpu_foreach(max_stat, preemption_disable_max_mt) {
2830 stats[cpu++] = os_atomic_load(max_stat, relaxed);
2831 }
2832
2833 if (req->newlen > 0) {
2834 // writing just resets all stats.
2835 percpu_foreach(max_stat, preemption_disable_max_mt) {
2836 os_atomic_store(max_stat, 0, relaxed);
2837 }
2838 }
2839
2840 return sysctl_io_opaque(req, stats, cpu * sizeof(uint64_t), NULL);
2841 }
2842
2843 SYSCTL_PROC(_kern, OID_AUTO, sched_preemption_disable_stats,
2844 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_LOCKED,
2845 0, 0, sysctl_sched_preemption_disable_stats, "I", "Preemption disablement statistics");
2846
2847 #endif /* SCHED_HYGIENE_DEBUG */
2848
2849 /* used for testing by exception_tests */
2850 extern uint32_t ipc_control_port_options;
2851 SYSCTL_INT(_kern, OID_AUTO, ipc_control_port_options,
2852 CTLFLAG_RD | CTLFLAG_LOCKED, &ipc_control_port_options, 0, "");
2853
2854 #endif /* DEVELOPMENT || DEBUG */
2855
2856 extern uint32_t task_exc_guard_default;
2857
2858 SYSCTL_INT(_kern, OID_AUTO, task_exc_guard_default,
2859 CTLFLAG_RD | CTLFLAG_LOCKED, &task_exc_guard_default, 0, "");
2860
2861
2862 static int
2863 sysctl_kern_tcsm_available SYSCTL_HANDLER_ARGS
2864 {
2865 #pragma unused(oidp, arg1, arg2)
2866 uint32_t value = machine_csv(CPUVN_CI) ? 1 : 0;
2867
2868 if (req->newptr) {
2869 return EINVAL;
2870 }
2871
2872 return SYSCTL_OUT(req, &value, sizeof(value));
2873 }
2874 SYSCTL_PROC(_kern, OID_AUTO, tcsm_available,
2875 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED | CTLFLAG_ANYBODY,
2876 0, 0, sysctl_kern_tcsm_available, "I", "");
2877
2878
2879 static int
2880 sysctl_kern_tcsm_enable SYSCTL_HANDLER_ARGS
2881 {
2882 #pragma unused(oidp, arg1, arg2)
2883 uint32_t soflags = 0;
2884 uint32_t old_value = thread_get_no_smt() ? 1 : 0;
2885
2886 int error = SYSCTL_IN(req, &soflags, sizeof(soflags));
2887 if (error) {
2888 return error;
2889 }
2890
2891 if (soflags && machine_csv(CPUVN_CI)) {
2892 thread_set_no_smt(true);
2893 machine_tecs(current_thread());
2894 }
2895
2896 return SYSCTL_OUT(req, &old_value, sizeof(old_value));
2897 }
2898 SYSCTL_PROC(_kern, OID_AUTO, tcsm_enable,
2899 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_MASKED | CTLFLAG_ANYBODY,
2900 0, 0, sysctl_kern_tcsm_enable, "I", "");
2901
2902 static int
2903 sysctl_kern_debug_get_preoslog SYSCTL_HANDLER_ARGS
2904 {
2905 #pragma unused(oidp, arg1, arg2)
2906 static bool oneshot_executed = false;
2907 size_t preoslog_size = 0;
2908 const char *preoslog = NULL;
2909 int ret = 0;
2910
2911 // DumpPanic passes a non-zero write value when it needs oneshot behaviour
2912 if (req->newptr != USER_ADDR_NULL) {
2913 uint8_t oneshot = 0;
2914 int error = SYSCTL_IN(req, &oneshot, sizeof(oneshot));
2915 if (error) {
2916 return error;
2917 }
2918
2919 if (oneshot) {
2920 if (!os_atomic_cmpxchg(&oneshot_executed, false, true, acq_rel)) {
2921 return EPERM;
2922 }
2923 }
2924 }
2925
2926 preoslog = sysctl_debug_get_preoslog(&preoslog_size);
2927 if (preoslog != NULL && preoslog_size == 0) {
2928 sysctl_debug_free_preoslog();
2929 return 0;
2930 }
2931
2932 if (preoslog == NULL || preoslog_size == 0) {
2933 return 0;
2934 }
2935
2936 if (req->oldptr == USER_ADDR_NULL) {
2937 req->oldidx = preoslog_size;
2938 return 0;
2939 }
2940
2941 ret = SYSCTL_OUT(req, preoslog, preoslog_size);
2942 sysctl_debug_free_preoslog();
2943 return ret;
2944 }
2945
2946 SYSCTL_PROC(_kern, OID_AUTO, preoslog, CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_LOCKED,
2947 0, 0, sysctl_kern_debug_get_preoslog, "-", "");
2948
2949 #if DEVELOPMENT || DEBUG
2950 extern void sysctl_task_set_no_smt(char no_smt);
2951 extern char sysctl_task_get_no_smt(void);
2952
2953 static int
2954 sysctl_kern_sched_task_set_no_smt SYSCTL_HANDLER_ARGS
2955 {
2956 #pragma unused(oidp, arg1, arg2)
2957 char buff[4];
2958
2959 int error = SYSCTL_IN(req, buff, 1);
2960 if (error) {
2961 return error;
2962 }
2963 char no_smt = buff[0];
2964
2965 if (!req->newptr) {
2966 goto out;
2967 }
2968
2969 sysctl_task_set_no_smt(no_smt);
2970 out:
2971 no_smt = sysctl_task_get_no_smt();
2972 buff[0] = no_smt;
2973
2974 return SYSCTL_OUT(req, buff, 1);
2975 }
2976
2977 SYSCTL_PROC(_kern, OID_AUTO, sched_task_set_no_smt, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY,
2978 0, 0, sysctl_kern_sched_task_set_no_smt, "A", "");
2979
2980 static int
sysctl_kern_sched_thread_set_no_smt(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)2981 sysctl_kern_sched_thread_set_no_smt(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
2982 {
2983 int new_value, changed;
2984 int old_value = thread_get_no_smt() ? 1 : 0;
2985 int error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
2986
2987 if (changed) {
2988 thread_set_no_smt(!!new_value);
2989 }
2990
2991 return error;
2992 }
2993
2994 SYSCTL_PROC(_kern, OID_AUTO, sched_thread_set_no_smt,
2995 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY,
2996 0, 0, sysctl_kern_sched_thread_set_no_smt, "I", "");
2997
2998
2999 static int
3000 sysctl_kern_task_set_filter_msg_flag SYSCTL_HANDLER_ARGS
3001 {
3002 #pragma unused(oidp, arg1, arg2)
3003 int new_value, changed;
3004 int old_value = task_get_filter_msg_flag(current_task()) ? 1 : 0;
3005 int error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
3006
3007 if (changed) {
3008 task_set_filter_msg_flag(current_task(), !!new_value);
3009 }
3010
3011 return error;
3012 }
3013
3014 SYSCTL_PROC(_kern, OID_AUTO, task_set_filter_msg_flag, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3015 0, 0, sysctl_kern_task_set_filter_msg_flag, "I", "");
3016
3017 #if CONFIG_PROC_RESOURCE_LIMITS
3018
3019 extern mach_port_name_t current_task_get_fatal_port_name(void);
3020
3021 static int
3022 sysctl_kern_task_get_fatal_port SYSCTL_HANDLER_ARGS
3023 {
3024 #pragma unused(oidp, arg1, arg2)
3025 int port = 0;
3026 int flag = 0;
3027
3028 if (req->oldptr == USER_ADDR_NULL) {
3029 req->oldidx = sizeof(mach_port_t);
3030 return 0;
3031 }
3032
3033 int error = SYSCTL_IN(req, &flag, sizeof(flag));
3034 if (error) {
3035 return error;
3036 }
3037
3038 if (flag == 1) {
3039 port = (int)current_task_get_fatal_port_name();
3040 }
3041 return SYSCTL_OUT(req, &port, sizeof(port));
3042 }
3043
3044 SYSCTL_PROC(_machdep, OID_AUTO, task_get_fatal_port, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3045 0, 0, sysctl_kern_task_get_fatal_port, "I", "");
3046
3047 #endif /* CONFIG_PROC_RESOURCE_LIMITS */
3048
3049 extern unsigned int ipc_entry_table_count_max(void);
3050
3051 static int
3052 sysctl_mach_max_port_table_size SYSCTL_HANDLER_ARGS
3053 {
3054 #pragma unused(oidp, arg1, arg2)
3055 int old_value = ipc_entry_table_count_max();
3056 int error = sysctl_io_number(req, old_value, sizeof(int), NULL, NULL);
3057
3058 return error;
3059 }
3060
3061 SYSCTL_PROC(_machdep, OID_AUTO, max_port_table_size, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3062 0, 0, sysctl_mach_max_port_table_size, "I", "");
3063
3064 #endif /* DEVELOPMENT || DEBUG */
3065
3066 #if defined(CONFIG_KDP_INTERACTIVE_DEBUGGING) && defined(CONFIG_KDP_COREDUMP_ENCRYPTION)
3067
3068 #define COREDUMP_ENCRYPTION_KEY_ENTITLEMENT "com.apple.private.coredump-encryption-key"
3069
3070 static int
3071 sysctl_coredump_encryption_key_update SYSCTL_HANDLER_ARGS
3072 {
3073 kern_return_t ret = KERN_SUCCESS;
3074 int error = 0;
3075 struct kdp_core_encryption_key_descriptor key_descriptor = {
3076 .kcekd_format = MACH_CORE_FILEHEADER_V2_FLAG_NEXT_COREFILE_KEY_FORMAT_NIST_P256,
3077 };
3078
3079 /* Need to be root and have entitlement */
3080 if (!kauth_cred_issuser(kauth_cred_get()) && !IOCurrentTaskHasEntitlement(COREDUMP_ENCRYPTION_KEY_ENTITLEMENT)) {
3081 return EPERM;
3082 }
3083
3084 // Sanity-check the given key length
3085 if (req->newlen > UINT16_MAX) {
3086 return EINVAL;
3087 }
3088
3089 // It is allowed for the caller to pass in a NULL buffer.
3090 // This indicates that they want us to forget about any public key we might have.
3091 if (req->newptr) {
3092 key_descriptor.kcekd_size = (uint16_t) req->newlen;
3093 key_descriptor.kcekd_key = kalloc_data(key_descriptor.kcekd_size, Z_WAITOK);
3094
3095 if (key_descriptor.kcekd_key == NULL) {
3096 return ENOMEM;
3097 }
3098
3099 error = SYSCTL_IN(req, key_descriptor.kcekd_key, key_descriptor.kcekd_size);
3100 if (error) {
3101 goto out;
3102 }
3103 }
3104
3105 ret = IOProvideCoreFileAccess(kdp_core_handle_new_encryption_key, (void *)&key_descriptor);
3106 if (KERN_SUCCESS != ret) {
3107 printf("Failed to handle the new encryption key. Error 0x%x", ret);
3108 error = EFAULT;
3109 }
3110
3111 out:
3112 kfree_data(key_descriptor.kcekd_key, key_descriptor.kcekd_size);
3113 return 0;
3114 }
3115
3116 SYSCTL_PROC(_kern, OID_AUTO, coredump_encryption_key, CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3117 0, 0, &sysctl_coredump_encryption_key_update, "-", "Set a new encryption key for coredumps");
3118
3119 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING && CONFIG_KDP_COREDUMP_ENCRYPTION*/
3120