1 /*
2 * Copyright (c) 2000-2019 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1989, 1993, 1995
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/kauth.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/conf.h>
70 #include <sys/buf_internal.h>
71 #include <sys/mount_internal.h>
72 #include <sys/vnode_internal.h>
73 #include <sys/file_internal.h>
74 #include <sys/namei.h>
75 #include <sys/stat.h>
76 #include <sys/errno.h>
77 #include <sys/ioctl.h>
78 #include <sys/file.h>
79 #include <sys/user.h>
80 #include <sys/malloc.h>
81 #include <sys/disk.h>
82 #include <sys/uio_internal.h>
83 #include <sys/resource.h>
84 #include <machine/machine_routines.h>
85 #include <miscfs/specfs/specdev.h>
86 #include <vfs/vfs_support.h>
87 #include <vfs/vfs_disk_conditioner.h>
88
89 #include <kern/assert.h>
90 #include <kern/task.h>
91 #include <kern/sched_prim.h>
92 #include <kern/thread.h>
93 #include <kern/policy_internal.h>
94 #include <kern/timer_call.h>
95 #include <kern/waitq.h>
96
97 #include <pexpert/pexpert.h>
98
99 #include <sys/kdebug.h>
100 #include <libkern/section_keywords.h>
101
102 #if CONFIG_IO_COMPRESSION_STATS
103 #include <vfs/vfs_io_compression_stats.h>
104 #endif /* CONFIG_IO_COMPRESSION_STATS */
105
106 #if CONFIG_IOSCHED
107 #include <vm/vm_pageout_xnu.h>
108 #include <vm/vm_object_xnu.h>
109 #endif /* CONFIG_IOSCHED */
110
111 /* XXX following three prototypes should be in a header file somewhere */
112 extern dev_t chrtoblk(dev_t dev);
113 extern boolean_t iskmemdev(dev_t dev);
114 extern int bpfkqfilter(dev_t dev, struct knote *kn);
115 extern int ptsd_kqfilter(dev_t, struct knote *);
116 extern int ptmx_kqfilter(dev_t, struct knote *);
117 #if CONFIG_PHYS_WRITE_ACCT
118 uint64_t kernel_pm_writes; // to track the sync writes occurring during power management transitions
119 #endif /* CONFIG_PHYS_WRITE_ACCT */
120
121
122 struct vnode *speclisth[SPECHSZ];
123
124 /* symbolic sleep message strings for devices */
125 char devopn[] = "devopn";
126 char devio[] = "devio";
127 char devwait[] = "devwait";
128 char devin[] = "devin";
129 char devout[] = "devout";
130 char devioc[] = "devioc";
131 char devcls[] = "devcls";
132
133 #define VOPFUNC int (*)(void *)
134
135 int(**spec_vnodeop_p)(void *);
136 const struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
137 { .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC)(void (*)(void))vn_default_error },
138 { .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC)spec_lookup }, /* lookup */
139 { .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC)err_create }, /* create */
140 { .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC)err_mknod }, /* mknod */
141 { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC)spec_open }, /* open */
142 { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC)spec_close }, /* close */
143 { .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC)spec_access }, /* access */
144 { .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC)spec_getattr }, /* getattr */
145 { .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC)spec_setattr }, /* setattr */
146 { .opve_op = &vnop_read_desc, .opve_impl = (VOPFUNC)spec_read }, /* read */
147 { .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC)spec_write }, /* write */
148 { .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC)spec_ioctl }, /* ioctl */
149 { .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC)spec_select }, /* select */
150 { .opve_op = &vnop_revoke_desc, .opve_impl = (VOPFUNC)nop_revoke }, /* revoke */
151 { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC)err_mmap }, /* mmap */
152 { .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC)spec_fsync }, /* fsync */
153 { .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC)err_remove }, /* remove */
154 { .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC)err_link }, /* link */
155 { .opve_op = &vnop_rename_desc, .opve_impl = (VOPFUNC)err_rename }, /* rename */
156 { .opve_op = &vnop_mkdir_desc, .opve_impl = (VOPFUNC)err_mkdir }, /* mkdir */
157 { .opve_op = &vnop_rmdir_desc, .opve_impl = (VOPFUNC)err_rmdir }, /* rmdir */
158 { .opve_op = &vnop_symlink_desc, .opve_impl = (VOPFUNC)err_symlink }, /* symlink */
159 { .opve_op = &vnop_readdir_desc, .opve_impl = (VOPFUNC)err_readdir }, /* readdir */
160 { .opve_op = &vnop_readlink_desc, .opve_impl = (VOPFUNC)err_readlink }, /* readlink */
161 { .opve_op = &vnop_inactive_desc, .opve_impl = (VOPFUNC)nop_inactive }, /* inactive */
162 { .opve_op = &vnop_reclaim_desc, .opve_impl = (VOPFUNC)nop_reclaim }, /* reclaim */
163 { .opve_op = &vnop_strategy_desc, .opve_impl = (VOPFUNC)spec_strategy }, /* strategy */
164 { .opve_op = &vnop_pathconf_desc, .opve_impl = (VOPFUNC)spec_pathconf }, /* pathconf */
165 { .opve_op = &vnop_advlock_desc, .opve_impl = (VOPFUNC)err_advlock }, /* advlock */
166 { .opve_op = &vnop_bwrite_desc, .opve_impl = (VOPFUNC)spec_bwrite }, /* bwrite */
167 { .opve_op = &vnop_pagein_desc, .opve_impl = (VOPFUNC)err_pagein }, /* Pagein */
168 { .opve_op = &vnop_pageout_desc, .opve_impl = (VOPFUNC)err_pageout }, /* Pageout */
169 { .opve_op = &vnop_copyfile_desc, .opve_impl = (VOPFUNC)err_copyfile }, /* Copyfile */
170 { .opve_op = &vnop_blktooff_desc, .opve_impl = (VOPFUNC)spec_blktooff }, /* blktooff */
171 { .opve_op = &vnop_offtoblk_desc, .opve_impl = (VOPFUNC)spec_offtoblk }, /* offtoblk */
172 { .opve_op = &vnop_blockmap_desc, .opve_impl = (VOPFUNC)spec_blockmap }, /* blockmap */
173 { .opve_op = (struct vnodeop_desc*)NULL, .opve_impl = (int (*)(void *))NULL }
174 };
175 const struct vnodeopv_desc spec_vnodeop_opv_desc =
176 { .opv_desc_vector_p = &spec_vnodeop_p, .opv_desc_ops = spec_vnodeop_entries };
177
178
179 static void set_blocksize(vnode_t, dev_t);
180
181 #define LOWPRI_TIER1_WINDOW_MSECS 25
182 #define LOWPRI_TIER2_WINDOW_MSECS 100
183 #define LOWPRI_TIER3_WINDOW_MSECS 500
184
185 #define LOWPRI_TIER1_IO_PERIOD_MSECS 40
186 #define LOWPRI_TIER2_IO_PERIOD_MSECS 85
187 #define LOWPRI_TIER3_IO_PERIOD_MSECS 200
188
189 #define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5
190 #define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15
191 #define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25
192
193
194 int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = {
195 0,
196 LOWPRI_TIER1_WINDOW_MSECS,
197 LOWPRI_TIER2_WINDOW_MSECS,
198 LOWPRI_TIER3_WINDOW_MSECS,
199 };
200
201 int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = {
202 0,
203 LOWPRI_TIER1_IO_PERIOD_MSECS,
204 LOWPRI_TIER2_IO_PERIOD_MSECS,
205 LOWPRI_TIER3_IO_PERIOD_MSECS,
206 };
207
208 int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = {
209 0,
210 LOWPRI_TIER1_IO_PERIOD_SSD_MSECS,
211 LOWPRI_TIER2_IO_PERIOD_SSD_MSECS,
212 LOWPRI_TIER3_IO_PERIOD_SSD_MSECS,
213 };
214
215
216 int throttled_count[THROTTLE_LEVEL_END + 1];
217
218 struct _throttle_io_info_t {
219 lck_mtx_t throttle_lock;
220
221 struct timeval throttle_last_write_timestamp;
222 struct timeval throttle_min_timer_deadline;
223 struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */
224 struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1];
225 pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1];
226 struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1];
227 int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1];
228
229 TAILQ_HEAD(, uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */
230 int throttle_next_wake_level;
231
232 thread_call_t throttle_timer_call;
233 int32_t throttle_timer_ref;
234 int32_t throttle_timer_active;
235
236 int32_t throttle_io_count;
237 int32_t throttle_io_count_begin;
238 int *throttle_io_periods;
239 uint32_t throttle_io_period_num;
240
241 int32_t throttle_refcnt;
242 int32_t throttle_alloc;
243 int32_t throttle_disabled;
244 int32_t throttle_is_fusion_with_priority;
245 };
246
247 struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV];
248
249
250 int lowpri_throttle_enabled = 1;
251
252
253 static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level);
254 static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap);
255 static int throttle_get_thread_throttle_level(uthread_t ut);
256 static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier);
257 void throttle_info_mount_reset_period(mount_t mp, int isssd);
258
259 /*
260 * Trivial lookup routine that always fails.
261 */
262 int
spec_lookup(struct vnop_lookup_args * ap)263 spec_lookup(struct vnop_lookup_args *ap)
264 {
265 *ap->a_vpp = NULL;
266 return ENOTDIR;
267 }
268
269 static void
set_blocksize(struct vnode * vp,dev_t dev)270 set_blocksize(struct vnode *vp, dev_t dev)
271 {
272 int (*size)(dev_t);
273 int rsize;
274
275 if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) {
276 rsize = (*size)(dev);
277 if (rsize <= 0) { /* did size fail? */
278 vp->v_specsize = DEV_BSIZE;
279 } else {
280 vp->v_specsize = rsize;
281 }
282 } else {
283 vp->v_specsize = DEV_BSIZE;
284 }
285 }
286
287 void
set_fsblocksize(struct vnode * vp)288 set_fsblocksize(struct vnode *vp)
289 {
290 if (vp->v_type == VBLK) {
291 dev_t dev = (dev_t)vp->v_rdev;
292 int maj = major(dev);
293
294 if ((u_int)maj >= (u_int)nblkdev) {
295 return;
296 }
297
298 vnode_lock(vp);
299 set_blocksize(vp, dev);
300 vnode_unlock(vp);
301 }
302 }
303
304 static void
spec_init_bsdunit(vnode_t vp,vfs_context_t ctx,const char * caller)305 spec_init_bsdunit(vnode_t vp, vfs_context_t ctx, const char* caller)
306 {
307 int isssd = 0;
308 uint64_t throttle_mask = 0;
309 uint32_t devbsdunit = 0;
310
311 if (VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ctx)) {
312 isssd = 0;
313 }
314 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL)) {
315 throttle_mask = 0;
316 }
317
318 if (throttle_mask != 0) {
319 /*
320 * as a reasonable approximation, only use the lowest bit of the mask
321 * to generate a disk unit number
322 */
323 devbsdunit = num_trailing_0(throttle_mask);
324 } else {
325 devbsdunit = 0;
326 }
327
328 if (vp->v_un.vu_specinfo->si_initted == 0) {
329 vnode_lock(vp);
330 if (vp->v_un.vu_specinfo->si_initted == 0) {
331 vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
332 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
333 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
334 vp->v_un.vu_specinfo->si_throttleable = 1;
335 vp->v_un.vu_specinfo->si_initted = 1;
336 }
337 vnode_unlock(vp);
338 printf("%s : si_devbsdunit initialized to (%d), throttle_mask is (0x%llx), isssd is (%d)\n",
339 caller, vp->v_un.vu_specinfo->si_devbsdunit,
340 vp->v_un.vu_specinfo->si_throttle_mask,
341 vp->v_un.vu_specinfo->si_isssd);
342 }
343 }
344
345 #define SPEC_INIT_BSDUNIT(vp, ctx) spec_init_bsdunit((vp), (ctx), __FUNCTION__)
346
347 /*
348 * Open a special file.
349 */
350 int
spec_open(struct vnop_open_args * ap)351 spec_open(struct vnop_open_args *ap)
352 {
353 struct proc *p = vfs_context_proc(ap->a_context);
354 kauth_cred_t cred = vfs_context_ucred(ap->a_context);
355 struct vnode *vp = ap->a_vp;
356 dev_t bdev, dev = (dev_t)vp->v_rdev;
357 int maj = major(dev);
358 int error;
359
360 /*
361 * Don't allow open if fs is mounted -nodev.
362 */
363 if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV)) {
364 return ENXIO;
365 }
366
367 switch (vp->v_type) {
368 case VCHR:
369 if ((u_int)maj >= (u_int)nchrdev) {
370 return ENXIO;
371 }
372 if (cred != FSCRED && (ap->a_mode & FWRITE)) {
373 #if 0
374 /*
375 * When running in very secure mode, do not allow
376 * opens for writing of any disk character devices.
377 */
378 if (securelevel >= 2 && isdisk(dev, VCHR)) {
379 return EPERM;
380 }
381 #endif
382
383 /* Never allow writing to /dev/mem or /dev/kmem */
384 if (iskmemdev(dev)) {
385 return EPERM;
386 }
387 /*
388 * When running in secure mode, do not allow opens for
389 * writing of character devices whose corresponding block
390 * devices are currently mounted.
391 */
392 if (securelevel >= 1) {
393 if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(bdev, VBLK, &error)) {
394 return error;
395 }
396 }
397 }
398
399 devsw_lock(dev, S_IFCHR);
400 error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p);
401
402 if (error == 0) {
403 vp->v_specinfo->si_opencount++;
404 }
405
406 devsw_unlock(dev, S_IFCHR);
407
408 if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
409 int isssd = 0;
410 uint64_t throttle_mask = 0;
411 uint32_t devbsdunit = 0;
412
413 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL) == 0) {
414 if (throttle_mask != 0 &&
415 VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ap->a_context) == 0) {
416 /*
417 * as a reasonable approximation, only use the lowest bit of the mask
418 * to generate a disk unit number
419 */
420 devbsdunit = num_trailing_0(throttle_mask);
421
422 vnode_lock(vp);
423
424 vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
425 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
426 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
427 vp->v_un.vu_specinfo->si_throttleable = 1;
428 vp->v_un.vu_specinfo->si_initted = 1;
429
430 vnode_unlock(vp);
431 }
432 }
433 if (vp->v_un.vu_specinfo->si_initted == 0) {
434 vnode_lock(vp);
435 vp->v_un.vu_specinfo->si_initted = 1;
436 vnode_unlock(vp);
437 }
438 }
439 return error;
440
441 case VBLK:
442 if ((u_int)maj >= (u_int)nblkdev) {
443 return ENXIO;
444 }
445 /*
446 * When running in very secure mode, do not allow
447 * opens for writing of any disk block devices.
448 */
449 if (securelevel >= 2 && cred != FSCRED &&
450 (ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK) {
451 return EPERM;
452 }
453 /*
454 * Do not allow opens of block devices that are
455 * currently mounted.
456 */
457 if ((error = vfs_mountedon(vp))) {
458 return error;
459 }
460
461 devsw_lock(dev, S_IFBLK);
462 error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p);
463 if (!error) {
464 vp->v_specinfo->si_opencount++;
465 }
466 devsw_unlock(dev, S_IFBLK);
467
468 if (!error) {
469 u_int64_t blkcnt;
470 u_int32_t blksize;
471 int setsize = 0;
472 u_int32_t size512 = 512;
473
474 if (bdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
475 SPEC_INIT_BSDUNIT(vp, ap->a_context);
476 }
477
478 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, (caddr_t)&blksize, 0, ap->a_context)) {
479 /* Switch to 512 byte sectors (temporarily) */
480
481 if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&size512, FWRITE, ap->a_context)) {
482 /* Get the number of 512 byte physical blocks. */
483 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, ap->a_context)) {
484 setsize = 1;
485 }
486 }
487 /* If it doesn't set back, we can't recover */
488 if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, ap->a_context)) {
489 error = ENXIO;
490 }
491 }
492
493
494 vnode_lock(vp);
495 set_blocksize(vp, dev);
496
497 /*
498 * Cache the size in bytes of the block device for later
499 * use by spec_write().
500 */
501 if (setsize) {
502 vp->v_specdevsize = blkcnt * (u_int64_t)size512;
503 } else {
504 vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */
505 }
506 vnode_unlock(vp);
507 }
508 return error;
509 default:
510 panic("spec_open type");
511 }
512 return 0;
513 }
514
515 /*
516 * Vnode op for read
517 */
518 int
spec_read(struct vnop_read_args * ap)519 spec_read(struct vnop_read_args *ap)
520 {
521 struct vnode *vp = ap->a_vp;
522 struct uio *uio = ap->a_uio;
523 struct buf *bp;
524 daddr64_t bn, nextbn;
525 long bscale;
526 int devBlockSize = 0;
527 size_t bsize, n, on;
528 int error = 0;
529 dev_t dev;
530
531 #if DIAGNOSTIC
532 if (uio->uio_rw != UIO_READ) {
533 panic("spec_read mode");
534 }
535 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
536 panic("spec_read proc");
537 }
538 #endif
539 if (uio_resid(uio) == 0) {
540 return 0;
541 }
542
543 switch (vp->v_type) {
544 case VCHR:
545 {
546 struct _throttle_io_info_t *throttle_info = NULL;
547 int thread_throttle_level;
548 uint64_t blkno = 0;
549 uint32_t iolen = 0;
550 int ddisk = 0;
551 int ktrace_code = DKIO_READ;
552 devBlockSize = vp->v_specsize;
553 uintptr_t our_id = 0;
554
555 if (cdevsw[major(vp->v_rdev)].d_type == D_DISK) {
556 ddisk = 1;
557 }
558
559 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
560 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
561 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
562 }
563
564 if (kdebug_enable && ddisk) {
565 if (devBlockSize == 0) {
566 devBlockSize = 512; // default sector size
567 }
568
569 if (uio_offset(uio) && devBlockSize) {
570 blkno = ((uint64_t) uio_offset(uio) / ((uint64_t)devBlockSize));
571 }
572 iolen = (int) uio_resid(uio);
573 our_id = (uintptr_t)thread_tid(current_thread());
574 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
575 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
576 vp->v_rdev, blkno, iolen, 0);
577 }
578
579 error = (*cdevsw[major(vp->v_rdev)].d_read)
580 (vp->v_rdev, uio, ap->a_ioflag);
581
582
583 if (kdebug_enable && ddisk) {
584 uint32_t residual = (uint32_t)uio_resid(uio);
585 ktrace_code |= DKIO_DONE;
586 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
587 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
588 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
589 }
590
591 if (throttle_info) {
592 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
593 }
594
595 return error;
596 }
597
598 case VBLK:
599 if (uio->uio_offset < 0) {
600 return EINVAL;
601 }
602
603 dev = vp->v_rdev;
604
605 devBlockSize = vp->v_specsize;
606
607 if (devBlockSize > PAGE_SIZE) {
608 return EINVAL;
609 }
610
611 bscale = PAGE_SIZE / devBlockSize;
612 bsize = bscale * devBlockSize;
613
614 do {
615 on = uio->uio_offset % bsize;
616
617 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~(bscale - 1));
618
619 if (vp->v_speclastr + bscale == bn) {
620 nextbn = bn + bscale;
621 error = buf_breadn(vp, bn, (int)bsize, &nextbn,
622 (int *)&bsize, 1, NOCRED, &bp);
623 } else {
624 error = buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
625 }
626
627 vnode_lock(vp);
628 vp->v_speclastr = bn;
629 vnode_unlock(vp);
630
631 n = bsize - buf_resid(bp);
632 if ((on > n) || error) {
633 if (!error) {
634 error = EINVAL;
635 }
636 buf_brelse(bp);
637 return error;
638 }
639 n = MIN((n - on), (size_t)uio_resid(uio));
640
641 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
642 if (n + on == bsize) {
643 buf_markaged(bp);
644 }
645 buf_brelse(bp);
646 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
647 return error;
648
649 default:
650 panic("spec_read type");
651 }
652 /* NOTREACHED */
653
654 return 0;
655 }
656
657 /*
658 * Vnode op for write
659 */
660 int
spec_write(struct vnop_write_args * ap)661 spec_write(struct vnop_write_args *ap)
662 {
663 struct vnode *vp = ap->a_vp;
664 struct uio *uio = ap->a_uio;
665 struct buf *bp;
666 daddr64_t bn;
667 int blkmask, bscale;
668 int io_sync;
669 int devBlockSize = 0;
670 size_t bsize, n, on;
671 int error = 0;
672 dev_t dev;
673
674 #if DIAGNOSTIC
675 if (uio->uio_rw != UIO_WRITE) {
676 panic("spec_write mode");
677 }
678 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
679 panic("spec_write proc");
680 }
681 #endif
682
683 switch (vp->v_type) {
684 case VCHR:
685 {
686 struct _throttle_io_info_t *throttle_info = NULL;
687 int thread_throttle_level;
688 dev = vp->v_rdev;
689 devBlockSize = vp->v_specsize;
690 uint32_t iolen = 0;
691 uint64_t blkno = 0;
692 int ddisk = 0;
693 int ktrace_code = 0; // write is implied; read must be OR'd in.
694 uintptr_t our_id = 0;
695
696 if (cdevsw[major(dev)].d_type == D_DISK) {
697 ddisk = 1;
698 }
699
700 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
701 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
702
703 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
704
705 microuptime(&throttle_info->throttle_last_write_timestamp);
706 }
707
708 if (kdebug_enable && ddisk) {
709 if (devBlockSize == 0) {
710 devBlockSize = 512; // default sector size
711 }
712 if ((uio_offset(uio) != 0) && devBlockSize) {
713 blkno = ((uint64_t)uio_offset(uio)) / ((uint64_t)devBlockSize);
714 }
715 iolen = (int)uio_resid(uio);
716 our_id = (uintptr_t)thread_tid(current_thread());
717 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
718 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
719 vp->v_rdev, blkno, iolen, 0);
720 }
721 error = (*cdevsw[major(vp->v_rdev)].d_write)
722 (vp->v_rdev, uio, ap->a_ioflag);
723
724 if (kdebug_enable && ddisk) {
725 //emit the I/O completion
726 uint32_t residual = (uint32_t)uio_resid(uio);
727 ktrace_code |= DKIO_DONE;
728 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
729 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
730 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
731 }
732
733 if (throttle_info) {
734 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
735 }
736
737 return error;
738 }
739
740 case VBLK:
741 if (uio_resid(uio) == 0) {
742 return 0;
743 }
744 if (uio->uio_offset < 0) {
745 return EINVAL;
746 }
747
748 io_sync = (ap->a_ioflag & IO_SYNC);
749
750 dev = (vp->v_rdev);
751
752 devBlockSize = vp->v_specsize;
753 if (devBlockSize > PAGE_SIZE) {
754 return EINVAL;
755 }
756
757 bscale = PAGE_SIZE / devBlockSize;
758 blkmask = bscale - 1;
759 bsize = bscale * devBlockSize;
760
761
762 do {
763 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~blkmask);
764 on = uio->uio_offset % bsize;
765
766 n = MIN((bsize - on), (size_t)uio_resid(uio));
767
768 /*
769 * Use buf_getblk() as an optimization IFF:
770 *
771 * 1) We are reading exactly a block on a block
772 * aligned boundary
773 * 2) We know the size of the device from spec_open
774 * 3) The read doesn't span the end of the device
775 *
776 * Otherwise, we fall back on buf_bread().
777 */
778 if (n == bsize &&
779 vp->v_specdevsize != (u_int64_t)0 &&
780 (uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) {
781 /* reduce the size of the read to what is there */
782 n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize;
783 }
784
785 if (n == bsize) {
786 bp = buf_getblk(vp, bn, (int)bsize, 0, 0, BLK_WRITE);
787 } else {
788 error = (int)buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
789 }
790
791 /* Translate downstream error for upstream, if needed */
792 if (!error) {
793 error = (int)buf_error(bp);
794 }
795 if (error) {
796 buf_brelse(bp);
797 return error;
798 }
799 n = MIN(n, bsize - buf_resid(bp));
800
801 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
802 if (error) {
803 buf_brelse(bp);
804 return error;
805 }
806 buf_markaged(bp);
807
808 if (io_sync) {
809 error = buf_bwrite(bp);
810 } else {
811 if ((n + on) == bsize) {
812 error = buf_bawrite(bp);
813 } else {
814 error = buf_bdwrite(bp);
815 }
816 }
817 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
818 return error;
819
820 default:
821 panic("spec_write type");
822 }
823 /* NOTREACHED */
824
825 return 0;
826 }
827
828 /*
829 * Device ioctl operation.
830 */
831 int
spec_ioctl(struct vnop_ioctl_args * ap)832 spec_ioctl(struct vnop_ioctl_args *ap)
833 {
834 proc_t p = vfs_context_proc(ap->a_context);
835 dev_t dev = ap->a_vp->v_rdev;
836 int retval = 0;
837
838 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START,
839 dev, ap->a_command, ap->a_fflag, ap->a_vp->v_type, 0);
840
841 switch (ap->a_vp->v_type) {
842 case VCHR:
843 retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data,
844 ap->a_fflag, p);
845 break;
846
847 case VBLK:
848 retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p);
849 if (!retval && ap->a_command == DKIOCSETBLOCKSIZE) {
850 ap->a_vp->v_specsize = *(uint32_t *)ap->a_data;
851 }
852 break;
853
854 default:
855 panic("spec_ioctl");
856 /* NOTREACHED */
857 }
858 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END,
859 dev, ap->a_command, ap->a_fflag, retval, 0);
860
861 return retval;
862 }
863
864 int
spec_select(struct vnop_select_args * ap)865 spec_select(struct vnop_select_args *ap)
866 {
867 proc_t p = vfs_context_proc(ap->a_context);
868 dev_t dev;
869
870 switch (ap->a_vp->v_type) {
871 default:
872 return 1; /* XXX */
873
874 case VCHR:
875 dev = ap->a_vp->v_rdev;
876 return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p);
877 }
878 }
879
880 int
spec_kqfilter(vnode_t vp,struct knote * kn,struct kevent_qos_s * kev)881 spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_qos_s *kev)
882 {
883 dev_t dev;
884
885 assert(vnode_ischr(vp));
886
887 dev = vnode_specrdev(vp);
888
889 #if NETWORKING
890 /*
891 * Try a bpf device, as defined in bsd/net/bpf.c
892 * If it doesn't error out the attach, then it
893 * claimed it. Otherwise, fall through and try
894 * other attaches.
895 */
896 int32_t tmp_flags = kn->kn_flags;
897 int64_t tmp_sdata = kn->kn_sdata;
898 int res;
899
900 res = bpfkqfilter(dev, kn);
901 if ((kn->kn_flags & EV_ERROR) == 0) {
902 return res;
903 }
904 kn->kn_flags = tmp_flags;
905 kn->kn_sdata = tmp_sdata;
906 #endif
907
908 if (major(dev) >= nchrdev) {
909 knote_set_error(kn, ENXIO);
910 return 0;
911 }
912
913 kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE);
914 kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET);
915
916 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
917 kn->kn_filtid = EVFILTID_PTSD;
918 return ptsd_kqfilter(dev, kn);
919 } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
920 kn->kn_filtid = EVFILTID_PTMX;
921 return ptmx_kqfilter(dev, kn);
922 } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) {
923 /*
924 * TTYs from drivers that use struct ttys use their own filter
925 * routines. The PTC driver doesn't use the tty for character
926 * counts, so it must go through the select fallback.
927 */
928 kn->kn_filtid = EVFILTID_TTY;
929 } else {
930 /* Try to attach to other char special devices */
931 kn->kn_filtid = EVFILTID_SPEC;
932 }
933
934 return knote_fops(kn)->f_attach(kn, kev);
935 }
936
937 /*
938 * Synch buffers associated with a block device
939 */
940 int
spec_fsync_internal(vnode_t vp,int waitfor,__unused vfs_context_t context)941 spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
942 {
943 if (vp->v_type == VCHR) {
944 return 0;
945 }
946 /*
947 * Flush all dirty buffers associated with a block device.
948 */
949 buf_flushdirtyblks(vp, (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), 0, "spec_fsync");
950
951 return 0;
952 }
953
954 int
spec_fsync(struct vnop_fsync_args * ap)955 spec_fsync(struct vnop_fsync_args *ap)
956 {
957 return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
958 }
959
960
961 /*
962 * Just call the device strategy routine
963 */
964 void throttle_init(void);
965
966
967 #if 0
968 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \
969 do { \
970 if ((debug_info)->alloc) \
971 printf("%s: "format, __FUNCTION__, ## args); \
972 } while(0)
973
974 #else
975 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...)
976 #endif
977
978
979 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "");
980 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "");
981 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "");
982
983 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "");
984 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "");
985 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "");
986
987 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "");
988 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "");
989 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "");
990
991 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "");
992
993
994 static LCK_GRP_DECLARE(throttle_lock_grp, "throttle I/O");
995
996
997 /*
998 * throttled I/O helper function
999 * convert the index of the lowest set bit to a device index
1000 */
1001 int
num_trailing_0(uint64_t n)1002 num_trailing_0(uint64_t n)
1003 {
1004 /*
1005 * since in most cases the number of trailing 0s is very small,
1006 * we simply counting sequentially from the lowest bit
1007 */
1008 if (n == 0) {
1009 return sizeof(n) * 8;
1010 }
1011 int count = 0;
1012 while (!ISSET(n, 1)) {
1013 n >>= 1;
1014 ++count;
1015 }
1016 return count;
1017 }
1018
1019
1020 /*
1021 * Release the reference and if the item was allocated and this is the last
1022 * reference then free it.
1023 *
1024 * This routine always returns the old value.
1025 */
1026 static int
throttle_info_rel(struct _throttle_io_info_t * info)1027 throttle_info_rel(struct _throttle_io_info_t *info)
1028 {
1029 SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt);
1030
1031 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
1032 info, (int)(oldValue - 1), info );
1033
1034 /* The reference count just went negative, very bad */
1035 if (oldValue == 0) {
1036 panic("throttle info ref cnt went negative!");
1037 }
1038
1039 /*
1040 * Once reference count is zero, no one else should be able to take a
1041 * reference
1042 */
1043 if ((oldValue == 1) && (info->throttle_alloc)) {
1044 DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n", info);
1045
1046 lck_mtx_destroy(&info->throttle_lock, &throttle_lock_grp);
1047 kfree_type(struct _throttle_io_info_t, info);
1048 }
1049 return oldValue;
1050 }
1051
1052
1053 /*
1054 * Just take a reference on the throttle info structure.
1055 *
1056 * This routine always returns the old value.
1057 */
1058 static SInt32
throttle_info_ref(struct _throttle_io_info_t * info)1059 throttle_info_ref(struct _throttle_io_info_t *info)
1060 {
1061 SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt);
1062
1063 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
1064 info, (int)(oldValue - 1), info );
1065 /* Allocated items should never have a reference of zero */
1066 if (info->throttle_alloc && (oldValue == 0)) {
1067 panic("Taking a reference without calling create throttle info!");
1068 }
1069
1070 return oldValue;
1071 }
1072
1073 /*
1074 * on entry the throttle_lock is held...
1075 * this function is responsible for taking
1076 * and dropping the reference on the info
1077 * structure which will keep it from going
1078 * away while the timer is running if it
1079 * happens to have been dynamically allocated by
1080 * a network fileystem kext which is now trying
1081 * to free it
1082 */
1083 static uint32_t
throttle_timer_start(struct _throttle_io_info_t * info,boolean_t update_io_count,int wakelevel)1084 throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel)
1085 {
1086 struct timeval elapsed;
1087 struct timeval now;
1088 struct timeval period;
1089 uint64_t elapsed_msecs;
1090 int throttle_level;
1091 int level;
1092 int msecs;
1093 boolean_t throttled = FALSE;
1094 boolean_t need_timer = FALSE;
1095
1096 microuptime(&now);
1097
1098 if (update_io_count == TRUE) {
1099 info->throttle_io_count_begin = info->throttle_io_count;
1100 info->throttle_io_period_num++;
1101
1102 while (wakelevel >= THROTTLE_LEVEL_THROTTLED) {
1103 info->throttle_start_IO_period_timestamp[wakelevel--] = now;
1104 }
1105
1106 info->throttle_min_timer_deadline = now;
1107
1108 msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED];
1109 period.tv_sec = msecs / 1000;
1110 period.tv_usec = (msecs % 1000) * 1000;
1111
1112 timevaladd(&info->throttle_min_timer_deadline, &period);
1113 }
1114 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) {
1115 elapsed = now;
1116 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1117 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1118
1119 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1120 if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1121 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) {
1122 /*
1123 * we had an I/O occur at a higher priority tier within
1124 * this tier's throttle window
1125 */
1126 throttled = TRUE;
1127 }
1128 /*
1129 * we assume that the windows are the same or longer
1130 * as we drop through the throttling tiers... thus
1131 * we can stop looking once we run into a tier with
1132 * threads to schedule regardless of whether it's
1133 * still in its throttling window or not
1134 */
1135 break;
1136 }
1137 }
1138 if (throttled == TRUE) {
1139 break;
1140 }
1141 }
1142 if (throttled == TRUE) {
1143 uint64_t deadline = 0;
1144 struct timeval target;
1145 struct timeval min_target;
1146
1147 /*
1148 * we've got at least one tier still in a throttled window
1149 * so we need a timer running... compute the next deadline
1150 * and schedule it
1151 */
1152 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1153 if (TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1154 continue;
1155 }
1156
1157 target = info->throttle_start_IO_period_timestamp[level];
1158
1159 msecs = info->throttle_io_periods[level];
1160 period.tv_sec = msecs / 1000;
1161 period.tv_usec = (msecs % 1000) * 1000;
1162
1163 timevaladd(&target, &period);
1164
1165 if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) {
1166 min_target = target;
1167 need_timer = TRUE;
1168 }
1169 }
1170 if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) {
1171 if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >)) {
1172 min_target = info->throttle_min_timer_deadline;
1173 }
1174 }
1175
1176 if (info->throttle_timer_active) {
1177 if (thread_call_cancel(info->throttle_timer_call) == FALSE) {
1178 /*
1179 * couldn't kill the timer because it's already
1180 * been dispatched, so don't try to start a new
1181 * one... once we drop the lock, the timer will
1182 * proceed and eventually re-run this function
1183 */
1184 need_timer = FALSE;
1185 } else {
1186 info->throttle_timer_active = 0;
1187 }
1188 }
1189 if (need_timer == TRUE) {
1190 /*
1191 * This is defined as an int (32-bit) rather than a 64-bit
1192 * value because it would need a really big period in the
1193 * order of ~500 days to overflow this. So, we let this be
1194 * 32-bit which allows us to use the clock_interval_to_deadline()
1195 * routine.
1196 */
1197 int target_msecs;
1198
1199 if (info->throttle_timer_ref == 0) {
1200 /*
1201 * take a reference for the timer
1202 */
1203 throttle_info_ref(info);
1204
1205 info->throttle_timer_ref = 1;
1206 }
1207 elapsed = min_target;
1208 timevalsub(&elapsed, &now);
1209 target_msecs = (int)(elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000);
1210
1211 if (target_msecs <= 0) {
1212 /*
1213 * we may have computed a deadline slightly in the past
1214 * due to various factors... if so, just set the timer
1215 * to go off in the near future (we don't need to be precise)
1216 */
1217 target_msecs = 1;
1218 }
1219 clock_interval_to_deadline(target_msecs, 1000000, &deadline);
1220
1221 thread_call_enter_delayed(info->throttle_timer_call, deadline);
1222 info->throttle_timer_active = 1;
1223 }
1224 }
1225 return throttle_level;
1226 }
1227
1228
1229 static void
throttle_timer(struct _throttle_io_info_t * info,__unused thread_call_param_t p)1230 throttle_timer(struct _throttle_io_info_t *info, __unused thread_call_param_t p)
1231 {
1232 uthread_t ut, utlist;
1233 struct timeval elapsed;
1234 struct timeval now;
1235 uint64_t elapsed_msecs;
1236 int throttle_level;
1237 int level;
1238 int wake_level;
1239 caddr_t wake_address = NULL;
1240 boolean_t update_io_count = FALSE;
1241 boolean_t need_wakeup = FALSE;
1242 boolean_t need_release = FALSE;
1243
1244 ut = NULL;
1245 lck_mtx_lock(&info->throttle_lock);
1246
1247 info->throttle_timer_active = 0;
1248 microuptime(&now);
1249
1250 elapsed = now;
1251 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]);
1252 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1253
1254 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) {
1255 wake_level = info->throttle_next_wake_level;
1256
1257 for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) {
1258 elapsed = now;
1259 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[wake_level]);
1260 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1261
1262 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1263 /*
1264 * we're closing out the current IO period...
1265 * if we have a waiting thread, wake it up
1266 * after we have reset the I/O window info
1267 */
1268 need_wakeup = TRUE;
1269 update_io_count = TRUE;
1270
1271 info->throttle_next_wake_level = wake_level - 1;
1272
1273 if (info->throttle_next_wake_level == THROTTLE_LEVEL_START) {
1274 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1275 }
1276
1277 break;
1278 }
1279 wake_level--;
1280
1281 if (wake_level == THROTTLE_LEVEL_START) {
1282 wake_level = THROTTLE_LEVEL_END;
1283 }
1284 }
1285 }
1286 if (need_wakeup == TRUE) {
1287 if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1288 ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]);
1289 TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist);
1290 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1291 ut->uu_is_throttled = false;
1292
1293 wake_address = (caddr_t)&ut->uu_on_throttlelist;
1294 }
1295 } else {
1296 wake_level = THROTTLE_LEVEL_START;
1297 }
1298
1299 throttle_level = throttle_timer_start(info, update_io_count, wake_level);
1300
1301 if (wake_address != NULL) {
1302 wakeup(wake_address);
1303 }
1304
1305 for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) {
1306 TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) {
1307 TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist);
1308 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1309 ut->uu_is_throttled = false;
1310
1311 wakeup(&ut->uu_on_throttlelist);
1312 }
1313 }
1314 if (info->throttle_timer_active == 0 && info->throttle_timer_ref) {
1315 info->throttle_timer_ref = 0;
1316 need_release = TRUE;
1317 }
1318 lck_mtx_unlock(&info->throttle_lock);
1319
1320 if (need_release == TRUE) {
1321 throttle_info_rel(info);
1322 }
1323 }
1324
1325
1326 static int
throttle_add_to_list(struct _throttle_io_info_t * info,uthread_t ut,int mylevel,boolean_t insert_tail)1327 throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail)
1328 {
1329 boolean_t start_timer = FALSE;
1330 int level = THROTTLE_LEVEL_START;
1331
1332 if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) {
1333 info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel];
1334 start_timer = TRUE;
1335 }
1336
1337 if (insert_tail == TRUE) {
1338 TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1339 } else {
1340 TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1341 }
1342
1343 ut->uu_on_throttlelist = (int8_t)mylevel;
1344
1345 if (start_timer == TRUE) {
1346 /* we may need to start or rearm the timer */
1347 level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START);
1348
1349 if (level == THROTTLE_LEVEL_END) {
1350 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
1351 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1352
1353 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1354 }
1355 }
1356 }
1357 return level;
1358 }
1359
1360 static void
throttle_init_throttle_window(void)1361 throttle_init_throttle_window(void)
1362 {
1363 int throttle_window_size;
1364
1365 /*
1366 * The hierarchy of throttle window values is as follows:
1367 * - Global defaults
1368 * - Device tree properties
1369 * - Boot-args
1370 * All values are specified in msecs.
1371 */
1372
1373 #if (XNU_TARGET_OS_OSX && __arm64__)
1374 /*
1375 * IO Tier EDT overrides are meant for
1376 * some arm platforms but not for
1377 * macs.
1378 */
1379 #else /* (XNU_TARGET_OS_OSX && __arm64__) */
1380 /* Override global values with device-tree properties */
1381 if (PE_get_default("kern.io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1382 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1383 }
1384
1385 if (PE_get_default("kern.io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1386 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1387 }
1388
1389 if (PE_get_default("kern.io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1390 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1391 }
1392 #endif /* (XNU_TARGET_OS_OSX && __arm64__) */
1393
1394 /* Override with boot-args */
1395 if (PE_parse_boot_argn("io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1396 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1397 }
1398
1399 if (PE_parse_boot_argn("io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1400 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1401 }
1402
1403 if (PE_parse_boot_argn("io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1404 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1405 }
1406 }
1407
1408 static void
throttle_init_throttle_period(struct _throttle_io_info_t * info,boolean_t isssd)1409 throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd)
1410 {
1411 int throttle_period_size;
1412
1413 /*
1414 * The hierarchy of throttle period values is as follows:
1415 * - Global defaults
1416 * - Device tree properties
1417 * - Boot-args
1418 * All values are specified in msecs.
1419 */
1420
1421 /* Assign global defaults */
1422 if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0)) {
1423 info->throttle_io_periods = &throttle_io_period_ssd_msecs[0];
1424 } else {
1425 info->throttle_io_periods = &throttle_io_period_msecs[0];
1426 }
1427
1428 #if (XNU_TARGET_OS_OSX && __arm64__)
1429 /*
1430 * IO Tier EDT overrides are meant for
1431 * some arm platforms but not for
1432 * macs.
1433 */
1434 #else /* (XNU_TARGET_OS_OSX && __arm64__) */
1435 /* Override global values with device-tree properties */
1436 if (PE_get_default("kern.io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1437 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1438 }
1439
1440 if (PE_get_default("kern.io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1441 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1442 }
1443
1444 if (PE_get_default("kern.io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1445 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1446 }
1447 #endif /* (XNU_TARGET_OS_OSX && __arm64__) */
1448
1449 /* Override with boot-args */
1450 if (PE_parse_boot_argn("io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1451 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1452 }
1453
1454 if (PE_parse_boot_argn("io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1455 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1456 }
1457
1458 if (PE_parse_boot_argn("io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1459 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1460 }
1461 }
1462
1463 #if CONFIG_IOSCHED
1464 int iosched_enabled = 1;
1465 #endif
1466
1467 void
throttle_init(void)1468 throttle_init(void)
1469 {
1470 struct _throttle_io_info_t *info;
1471 int i;
1472 int level;
1473 #if CONFIG_IOSCHED
1474 int iosched;
1475 #endif
1476
1477 /* Update throttle parameters based on device tree configuration */
1478 throttle_init_throttle_window();
1479
1480 for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) {
1481 info = &_throttle_io_info[i];
1482
1483 lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
1484 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1485
1486 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1487 TAILQ_INIT(&info->throttle_uthlist[level]);
1488 info->throttle_last_IO_pid[level] = 0;
1489 info->throttle_inflight_count[level] = 0;
1490 }
1491 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1492 info->throttle_disabled = 0;
1493 info->throttle_is_fusion_with_priority = 0;
1494 }
1495 #if CONFIG_IOSCHED
1496 if (PE_parse_boot_argn("iosched", &iosched, sizeof(iosched))) {
1497 iosched_enabled = iosched;
1498 }
1499 if (iosched_enabled) {
1500 /* Initialize I/O Reprioritization mechanism */
1501 vm_io_reprioritize_init();
1502 }
1503 #endif
1504 }
1505
1506 void
sys_override_io_throttle(boolean_t enable_override)1507 sys_override_io_throttle(boolean_t enable_override)
1508 {
1509 if (enable_override) {
1510 lowpri_throttle_enabled = 0;
1511 } else {
1512 lowpri_throttle_enabled = 1;
1513 }
1514 }
1515
1516 int rethrottle_wakeups = 0;
1517
1518 /*
1519 * the uu_rethrottle_lock is used to synchronize this function
1520 * with "throttle_lowpri_io" which is where a throttled thread
1521 * will block... that function will grab this lock before beginning
1522 * it's decision making process concerning the need to block, and
1523 * hold it through the assert_wait. When that thread is awakened
1524 * for any reason (timer or rethrottle), it will reacquire the
1525 * uu_rethrottle_lock before determining if it really is ok for
1526 * it to now run. This is the point at which the thread could
1527 * enter a different throttling queue and reblock or return from
1528 * the throttle w/o having waited out it's entire throttle if
1529 * the rethrottle has now moved it out of any currently
1530 * active throttle window.
1531 *
1532 *
1533 * NOTES:
1534 * 1 - This may be called with the task lock held.
1535 * 2 - This may be called with preemption and interrupts disabled
1536 * in the kqueue wakeup path so we can't take the throttle_lock which is a mutex
1537 * 3 - This cannot safely dereference uu_throttle_info, as it may
1538 * get deallocated out from under us
1539 */
1540
1541 void
rethrottle_thread(uthread_t ut)1542 rethrottle_thread(uthread_t ut)
1543 {
1544 /*
1545 * If uthread doesn't have throttle state, then there's no chance
1546 * of it needing a rethrottle.
1547 */
1548 if (ut->uu_throttle_info == NULL) {
1549 return;
1550 }
1551
1552 boolean_t s = ml_set_interrupts_enabled(FALSE);
1553 lck_spin_lock(&ut->uu_rethrottle_lock);
1554
1555 if (!ut->uu_is_throttled) {
1556 ut->uu_was_rethrottled = true;
1557 } else {
1558 int my_new_level = throttle_get_thread_throttle_level(ut);
1559
1560 if (my_new_level != ut->uu_on_throttlelist) {
1561 /*
1562 * ut is currently blocked (as indicated by
1563 * ut->uu_is_throttled == true)
1564 * and we're changing it's throttle level, so
1565 * we need to wake it up.
1566 */
1567 ut->uu_is_throttled = false;
1568 wakeup(&ut->uu_on_throttlelist);
1569
1570 rethrottle_wakeups++;
1571 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)),
1572 uthread_tid(ut), ut->uu_on_throttlelist, my_new_level, 0, 0);
1573 }
1574 }
1575 lck_spin_unlock(&ut->uu_rethrottle_lock);
1576 ml_set_interrupts_enabled(s);
1577 }
1578
1579
1580 /*
1581 * KPI routine
1582 *
1583 * Create and take a reference on a throttle info structure and return a
1584 * pointer for the file system to use when calling throttle_info_update.
1585 * Calling file system must have a matching release for every create.
1586 */
1587 void *
throttle_info_create(void)1588 throttle_info_create(void)
1589 {
1590 struct _throttle_io_info_t *info;
1591 int level;
1592
1593 info = kalloc_type(struct _throttle_io_info_t,
1594 Z_ZERO | Z_WAITOK | Z_NOFAIL);
1595 /* Mark that this one was allocated and needs to be freed */
1596 DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n", info, info );
1597 info->throttle_alloc = TRUE;
1598
1599 lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
1600 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1601
1602 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1603 TAILQ_INIT(&info->throttle_uthlist[level]);
1604 }
1605 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1606
1607 /* Take a reference */
1608 OSIncrementAtomic(&info->throttle_refcnt);
1609 return info;
1610 }
1611
1612 /*
1613 * KPI routine
1614 *
1615 * Release the throttle info pointer if all the reference are gone. Should be
1616 * called to release reference taken by throttle_info_create
1617 */
1618 void
throttle_info_release(void * throttle_info)1619 throttle_info_release(void *throttle_info)
1620 {
1621 DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n",
1622 (struct _throttle_io_info_t *)throttle_info,
1623 (struct _throttle_io_info_t *)throttle_info);
1624 if (throttle_info) { /* Just to be careful */
1625 throttle_info_rel(throttle_info);
1626 }
1627 }
1628
1629 /*
1630 * KPI routine
1631 *
1632 * File Systems that create an info structure, need to call this routine in
1633 * their mount routine (used by cluster code). File Systems that call this in
1634 * their mount routines must call throttle_info_mount_rel in their unmount
1635 * routines.
1636 */
1637 void
throttle_info_mount_ref(mount_t mp,void * throttle_info)1638 throttle_info_mount_ref(mount_t mp, void *throttle_info)
1639 {
1640 if ((throttle_info == NULL) || (mp == NULL)) {
1641 return;
1642 }
1643 throttle_info_ref(throttle_info);
1644
1645 /*
1646 * We already have a reference release it before adding the new one
1647 */
1648 if (mp->mnt_throttle_info) {
1649 throttle_info_rel(mp->mnt_throttle_info);
1650 }
1651 mp->mnt_throttle_info = throttle_info;
1652 }
1653
1654 /*
1655 * Private KPI routine
1656 *
1657 * return a handle for accessing throttle_info given a throttle_mask. The
1658 * handle must be released by throttle_info_rel_by_mask
1659 */
1660 int
throttle_info_ref_by_mask(uint64_t throttle_mask,throttle_info_handle_t * throttle_info_handle)1661 throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle)
1662 {
1663 int dev_index;
1664 struct _throttle_io_info_t *info;
1665
1666 /*
1667 * The 'throttle_mask' is not expected to be 0 otherwise num_trailing_0()
1668 * would return value of 64 and this will cause '_throttle_io_info' to
1669 * go out of bounds as '_throttle_io_info' is only LOWPRI_MAX_NUM_DEV (64)
1670 * elements long.
1671 */
1672 if (throttle_info_handle == NULL || throttle_mask == 0) {
1673 return EINVAL;
1674 }
1675
1676 dev_index = num_trailing_0(throttle_mask);
1677 info = &_throttle_io_info[dev_index];
1678 throttle_info_ref(info);
1679 *(struct _throttle_io_info_t**)throttle_info_handle = info;
1680
1681 return 0;
1682 }
1683
1684 /*
1685 * Private KPI routine
1686 *
1687 * release the handle obtained by throttle_info_ref_by_mask
1688 */
1689 void
throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)1690 throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)
1691 {
1692 /*
1693 * for now the handle is just a pointer to _throttle_io_info_t
1694 */
1695 throttle_info_rel((struct _throttle_io_info_t*)throttle_info_handle);
1696 }
1697
1698 /*
1699 * KPI routine
1700 *
1701 * File Systems that throttle_info_mount_ref, must call this routine in their
1702 * umount routine.
1703 */
1704 void
throttle_info_mount_rel(mount_t mp)1705 throttle_info_mount_rel(mount_t mp)
1706 {
1707 if (mp->mnt_throttle_info) {
1708 throttle_info_rel(mp->mnt_throttle_info);
1709 }
1710 mp->mnt_throttle_info = NULL;
1711 }
1712
1713 /*
1714 * Reset throttling periods for the given mount point
1715 *
1716 * private interface used by disk conditioner to reset
1717 * throttling periods when 'is_ssd' status changes
1718 */
1719 void
throttle_info_mount_reset_period(mount_t mp,int isssd)1720 throttle_info_mount_reset_period(mount_t mp, int isssd)
1721 {
1722 struct _throttle_io_info_t *info;
1723
1724 if (mp == NULL) {
1725 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1726 } else if (mp->mnt_throttle_info == NULL) {
1727 info = &_throttle_io_info[mp->mnt_devbsdunit];
1728 } else {
1729 info = mp->mnt_throttle_info;
1730 }
1731
1732 throttle_init_throttle_period(info, isssd);
1733 }
1734
1735 void
throttle_info_get_last_io_time(mount_t mp,struct timeval * tv)1736 throttle_info_get_last_io_time(mount_t mp, struct timeval *tv)
1737 {
1738 struct _throttle_io_info_t *info;
1739
1740 if (mp == NULL) {
1741 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1742 } else if (mp->mnt_throttle_info == NULL) {
1743 info = &_throttle_io_info[mp->mnt_devbsdunit];
1744 } else {
1745 info = mp->mnt_throttle_info;
1746 }
1747
1748 *tv = info->throttle_last_write_timestamp;
1749 }
1750
1751 void
update_last_io_time(mount_t mp)1752 update_last_io_time(mount_t mp)
1753 {
1754 struct _throttle_io_info_t *info;
1755
1756 if (mp == NULL) {
1757 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1758 } else if (mp->mnt_throttle_info == NULL) {
1759 info = &_throttle_io_info[mp->mnt_devbsdunit];
1760 } else {
1761 info = mp->mnt_throttle_info;
1762 }
1763
1764 microuptime(&info->throttle_last_write_timestamp);
1765 if (mp != NULL) {
1766 mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp;
1767 }
1768 }
1769
1770 int
throttle_get_io_policy(uthread_t * ut)1771 throttle_get_io_policy(uthread_t *ut)
1772 {
1773 if (ut != NULL) {
1774 *ut = current_uthread();
1775 }
1776
1777 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
1778 }
1779
1780 int
throttle_get_passive_io_policy(uthread_t * ut)1781 throttle_get_passive_io_policy(uthread_t *ut)
1782 {
1783 if (ut != NULL) {
1784 *ut = current_uthread();
1785 }
1786
1787 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_PASSIVE_IO);
1788 }
1789
1790
1791 static int
throttle_get_thread_throttle_level(uthread_t ut)1792 throttle_get_thread_throttle_level(uthread_t ut)
1793 {
1794 uthread_t *ut_p = (ut == NULL) ? &ut : NULL;
1795 int io_tier = throttle_get_io_policy(ut_p);
1796
1797 return throttle_get_thread_throttle_level_internal(ut, io_tier);
1798 }
1799
1800 /*
1801 * Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy)
1802 */
1803 static int
throttle_get_thread_throttle_level_internal(uthread_t ut,int io_tier)1804 throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier)
1805 {
1806 int thread_throttle_level = io_tier;
1807 int user_idle_level;
1808
1809 assert(ut != NULL);
1810
1811 /* Bootcache misses should always be throttled */
1812 if (ut->uu_throttle_bc) {
1813 thread_throttle_level = THROTTLE_LEVEL_TIER3;
1814 }
1815
1816 /*
1817 * Issue tier3 I/O as tier2 when the user is idle
1818 * to allow maintenance tasks to make more progress.
1819 *
1820 * Assume any positive idle level is enough... for now it's
1821 * only ever 0 or 128 but this is not defined anywhere.
1822 */
1823 if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) {
1824 user_idle_level = timer_get_user_idle_level();
1825 if (user_idle_level > 0) {
1826 thread_throttle_level--;
1827 }
1828 }
1829
1830 return thread_throttle_level;
1831 }
1832
1833 /*
1834 * I/O will be throttled if either of the following are true:
1835 * - Higher tiers have in-flight I/O
1836 * - The time delta since the last start/completion of a higher tier is within the throttle window interval
1837 *
1838 * In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal
1839 */
1840 static int
throttle_io_will_be_throttled_internal(void * throttle_info,int * mylevel,int * throttling_level)1841 throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level)
1842 {
1843 struct _throttle_io_info_t *info = throttle_info;
1844 struct timeval elapsed;
1845 struct timeval now;
1846 uint64_t elapsed_msecs;
1847 int thread_throttle_level;
1848 int throttle_level;
1849
1850 if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED) {
1851 return THROTTLE_DISENGAGED;
1852 }
1853
1854 microuptime(&now);
1855
1856 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
1857 if (info->throttle_inflight_count[throttle_level]) {
1858 break;
1859 }
1860 elapsed = now;
1861 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1862 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1863
1864 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
1865 break;
1866 }
1867 }
1868 if (throttle_level >= thread_throttle_level) {
1869 /*
1870 * we're beyond all of the throttle windows
1871 * that affect the throttle level of this thread,
1872 * so go ahead and treat as normal I/O
1873 */
1874 return THROTTLE_DISENGAGED;
1875 }
1876 if (mylevel) {
1877 *mylevel = thread_throttle_level;
1878 }
1879 if (throttling_level) {
1880 *throttling_level = throttle_level;
1881 }
1882
1883 if (info->throttle_io_count != info->throttle_io_count_begin) {
1884 /*
1885 * we've already issued at least one throttleable I/O
1886 * in the current I/O window, so avoid issuing another one
1887 */
1888 return THROTTLE_NOW;
1889 }
1890 /*
1891 * we're in the throttle window, so
1892 * cut the I/O size back
1893 */
1894 return THROTTLE_ENGAGED;
1895 }
1896
1897 /*
1898 * If we have a mount point and it has a throttle info pointer then
1899 * use it to do the check, otherwise use the device unit number to find
1900 * the correct throttle info array element.
1901 */
1902 int
throttle_io_will_be_throttled(__unused int lowpri_window_msecs,mount_t mp)1903 throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp)
1904 {
1905 struct _throttle_io_info_t *info;
1906
1907 /*
1908 * Should we just return zero if no mount point
1909 */
1910 if (mp == NULL) {
1911 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1912 } else if (mp->mnt_throttle_info == NULL) {
1913 info = &_throttle_io_info[mp->mnt_devbsdunit];
1914 } else {
1915 info = mp->mnt_throttle_info;
1916 }
1917
1918 if (info->throttle_is_fusion_with_priority) {
1919 uthread_t ut = current_uthread();
1920 if (ut->uu_lowpri_window == 0) {
1921 return THROTTLE_DISENGAGED;
1922 }
1923 }
1924
1925 if (info->throttle_disabled) {
1926 return THROTTLE_DISENGAGED;
1927 } else {
1928 return throttle_io_will_be_throttled_internal(info, NULL, NULL);
1929 }
1930 }
1931
1932 /*
1933 * Routine to increment I/O throttling counters maintained in the proc
1934 */
1935
1936 static void
throttle_update_proc_stats(pid_t throttling_pid,int count)1937 throttle_update_proc_stats(pid_t throttling_pid, int count)
1938 {
1939 proc_t throttling_proc;
1940 proc_t throttled_proc = current_proc();
1941
1942 /* The throttled_proc is always the current proc; so we are not concerned with refs */
1943 OSAddAtomic64(count, &(throttled_proc->was_throttled));
1944
1945 /* The throttling pid might have exited by now */
1946 throttling_proc = proc_find(throttling_pid);
1947 if (throttling_proc != PROC_NULL) {
1948 OSAddAtomic64(count, &(throttling_proc->did_throttle));
1949 proc_rele(throttling_proc);
1950 }
1951 }
1952
1953 /*
1954 * Block until woken up by the throttle timer or by a rethrottle call.
1955 * As long as we hold the throttle_lock while querying the throttle tier, we're
1956 * safe against seeing an old throttle tier after a rethrottle.
1957 */
1958 uint32_t
throttle_lowpri_io(int sleep_amount)1959 throttle_lowpri_io(int sleep_amount)
1960 {
1961 uthread_t ut;
1962 struct _throttle_io_info_t *info;
1963 int throttle_type = 0;
1964 int mylevel = 0;
1965 int throttling_level = THROTTLE_LEVEL_NONE;
1966 int sleep_cnt = 0;
1967 uint32_t throttle_io_period_num = 0;
1968 boolean_t insert_tail = TRUE;
1969 boolean_t s;
1970
1971 ut = current_uthread();
1972
1973 if (ut->uu_lowpri_window == 0) {
1974 return 0;
1975 }
1976 if (current_thread_in_kernel_fault()) {
1977 /* do not throttle kernel faults */
1978 return 0;
1979 }
1980
1981 info = ut->uu_throttle_info;
1982
1983 if (info == NULL) {
1984 ut->uu_throttle_bc = false;
1985 ut->uu_lowpri_window = 0;
1986 return 0;
1987 }
1988 lck_mtx_lock(&info->throttle_lock);
1989 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
1990
1991 if (sleep_amount == 0) {
1992 goto done;
1993 }
1994
1995 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
1996 sleep_amount = 0;
1997 }
1998
1999 throttle_io_period_num = info->throttle_io_period_num;
2000
2001 ut->uu_was_rethrottled = false;
2002
2003 while ((throttle_type = throttle_io_will_be_throttled_internal(info, &mylevel, &throttling_level))) {
2004 if (throttle_type == THROTTLE_ENGAGED) {
2005 if (sleep_amount == 0) {
2006 break;
2007 }
2008 if (info->throttle_io_period_num < throttle_io_period_num) {
2009 break;
2010 }
2011 if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
2012 break;
2013 }
2014 }
2015 /*
2016 * keep the same position in the list if "rethrottle_thread" changes our throttle level and
2017 * then puts us back to the original level before we get a chance to run
2018 */
2019 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) {
2020 /*
2021 * must have been awakened via "rethrottle_thread" (the timer pulls us off the list)
2022 * and we've changed our throttling level, so pull ourselves off of the appropriate list
2023 * and make sure we get put on the tail of the new list since we're starting anew w/r to
2024 * the throttling engine
2025 */
2026 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
2027 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
2028 insert_tail = TRUE;
2029 }
2030 if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) {
2031 if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END) {
2032 goto done;
2033 }
2034 }
2035 assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END);
2036
2037 s = ml_set_interrupts_enabled(FALSE);
2038 lck_spin_lock(&ut->uu_rethrottle_lock);
2039
2040 /*
2041 * this is the critical section w/r to our interaction
2042 * with "rethrottle_thread"
2043 */
2044 if (ut->uu_was_rethrottled) {
2045 lck_spin_unlock(&ut->uu_rethrottle_lock);
2046 ml_set_interrupts_enabled(s);
2047 lck_mtx_yield(&info->throttle_lock);
2048
2049 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)),
2050 uthread_tid(ut), ut->uu_on_throttlelist, 0, 0, 0);
2051
2052 ut->uu_was_rethrottled = false;
2053 continue;
2054 }
2055 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE,
2056 info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0);
2057
2058 if (sleep_cnt == 0) {
2059 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
2060 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2061 throttled_count[mylevel]++;
2062 }
2063 ut->uu_wmesg = "throttle_lowpri_io";
2064
2065 assert_wait((caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT);
2066
2067 ut->uu_is_throttled = true;
2068 lck_spin_unlock(&ut->uu_rethrottle_lock);
2069 ml_set_interrupts_enabled(s);
2070
2071 lck_mtx_unlock(&info->throttle_lock);
2072
2073 thread_block(THREAD_CONTINUE_NULL);
2074
2075 ut->uu_wmesg = NULL;
2076
2077 ut->uu_is_throttled = false;
2078 ut->uu_was_rethrottled = false;
2079
2080 lck_mtx_lock(&info->throttle_lock);
2081
2082 sleep_cnt++;
2083
2084 if (sleep_amount == 0) {
2085 insert_tail = FALSE;
2086 } else if (info->throttle_io_period_num < throttle_io_period_num ||
2087 (info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
2088 insert_tail = FALSE;
2089 sleep_amount = 0;
2090 }
2091 }
2092 done:
2093 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
2094 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
2095 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
2096 }
2097 lck_mtx_unlock(&info->throttle_lock);
2098
2099 if (sleep_cnt) {
2100 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
2101 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2102 /*
2103 * We update the stats for the last pid which opened a throttle window for the throttled thread.
2104 * This might not be completely accurate since the multiple throttles seen by the lower tier pid
2105 * might have been caused by various higher prio pids. However, updating these stats accurately
2106 * means doing a proc_find while holding the throttle lock which leads to deadlock.
2107 */
2108 throttle_update_proc_stats(info->throttle_last_IO_pid[throttling_level], sleep_cnt);
2109 }
2110
2111 ut->uu_throttle_info = NULL;
2112 ut->uu_throttle_bc = false;
2113 ut->uu_lowpri_window = 0;
2114
2115 throttle_info_rel(info);
2116
2117 return sleep_cnt;
2118 }
2119
2120 /*
2121 * returns TRUE if the throttle_lowpri_io called with the same sleep_amount would've slept
2122 * This function mimics the most of the throttle_lowpri_io checks but without actual sleeping
2123 */
2124 int
throttle_lowpri_io_will_be_throttled(int sleep_amount)2125 throttle_lowpri_io_will_be_throttled(int sleep_amount)
2126 {
2127 if (sleep_amount == 0) {
2128 return FALSE;
2129 }
2130
2131 uthread_t ut = current_uthread();
2132 if (ut->uu_lowpri_window == 0) {
2133 return FALSE;
2134 }
2135
2136 struct _throttle_io_info_t *info = ut->uu_throttle_info;
2137 if (info == NULL) {
2138 return FALSE;
2139 }
2140
2141 lck_mtx_lock(&info->throttle_lock);
2142 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
2143
2144 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
2145 sleep_amount = 0;
2146 }
2147
2148 int result = FALSE;
2149
2150 int throttle_type = throttle_io_will_be_throttled_internal(info, NULL, NULL);
2151 if (throttle_type > THROTTLE_DISENGAGED) {
2152 result = TRUE;
2153 if ((throttle_type == THROTTLE_ENGAGED) && (sleep_amount == 0)) {
2154 result = FALSE;
2155 }
2156 }
2157
2158 lck_mtx_unlock(&info->throttle_lock);
2159
2160 return result;
2161 }
2162
2163
2164 /*
2165 * KPI routine
2166 *
2167 * set a kernel thread's IO policy. policy can be:
2168 * IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD
2169 *
2170 * explanations about these policies are in the man page of setiopolicy_np
2171 */
2172 void
throttle_set_thread_io_policy(int policy)2173 throttle_set_thread_io_policy(int policy)
2174 {
2175 proc_set_thread_policy(current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, policy);
2176 }
2177
2178 int
throttle_get_thread_effective_io_policy()2179 throttle_get_thread_effective_io_policy()
2180 {
2181 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
2182 }
2183
2184 int
throttle_thread_io_tier_above_metadata(void)2185 throttle_thread_io_tier_above_metadata(void)
2186 {
2187 return throttle_get_thread_effective_io_policy() < IOSCHED_METADATA_TIER;
2188 }
2189
2190 void
throttle_info_reset_window(uthread_t ut)2191 throttle_info_reset_window(uthread_t ut)
2192 {
2193 struct _throttle_io_info_t *info;
2194
2195 if (ut == NULL) {
2196 ut = current_uthread();
2197 }
2198
2199 if ((info = ut->uu_throttle_info)) {
2200 throttle_info_rel(info);
2201
2202 ut->uu_throttle_info = NULL;
2203 ut->uu_lowpri_window = 0;
2204 ut->uu_throttle_bc = false;
2205 }
2206 }
2207
2208 static
2209 void
throttle_info_set_initial_window(uthread_t ut,struct _throttle_io_info_t * info,boolean_t BC_throttle,boolean_t isssd)2210 throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd)
2211 {
2212 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2213 return;
2214 }
2215
2216 if (info->throttle_io_periods == 0) {
2217 throttle_init_throttle_period(info, isssd);
2218 }
2219 if (ut->uu_throttle_info == NULL) {
2220 ut->uu_throttle_info = info;
2221 throttle_info_ref(info);
2222 DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n", info, info );
2223
2224 ut->uu_lowpri_window = 1;
2225 ut->uu_throttle_bc = BC_throttle;
2226 }
2227 }
2228
2229 /*
2230 * Update inflight IO count and throttling window
2231 * Should be called when an IO is done
2232 *
2233 * Only affects IO that was sent through spec_strategy
2234 */
2235 void
throttle_info_end_io(buf_t bp)2236 throttle_info_end_io(buf_t bp)
2237 {
2238 vnode_t vp;
2239 mount_t mp;
2240 struct bufattr *bap;
2241 struct _throttle_io_info_t *info;
2242 int io_tier;
2243
2244 bap = &bp->b_attr;
2245 if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) {
2246 return;
2247 }
2248 CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2249
2250 vp = buf_vnode(bp);
2251 mp = vp->v_mount;
2252
2253 if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
2254 info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
2255 } else if (mp != NULL) {
2256 info = &_throttle_io_info[mp->mnt_devbsdunit];
2257 } else {
2258 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2259 }
2260
2261 io_tier = GET_BUFATTR_IO_TIER(bap);
2262 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2263 io_tier--;
2264 }
2265
2266 throttle_info_end_io_internal(info, io_tier);
2267 }
2268
2269 /*
2270 * Decrement inflight count initially incremented by throttle_info_update_internal
2271 */
2272 static
2273 void
throttle_info_end_io_internal(struct _throttle_io_info_t * info,int throttle_level)2274 throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level)
2275 {
2276 if (throttle_level == THROTTLE_LEVEL_NONE) {
2277 return;
2278 }
2279
2280 microuptime(&info->throttle_window_start_timestamp[throttle_level]);
2281 OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]);
2282 assert(info->throttle_inflight_count[throttle_level] >= 0);
2283 }
2284
2285 /*
2286 * If inflight is TRUE and bap is NULL then the caller is responsible for calling
2287 * throttle_info_end_io_internal to avoid leaking in-flight I/O.
2288 */
2289 static
2290 int
throttle_info_update_internal(struct _throttle_io_info_t * info,uthread_t ut,int flags,boolean_t isssd,boolean_t inflight,struct bufattr * bap)2291 throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap)
2292 {
2293 int thread_throttle_level;
2294
2295 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2296 return THROTTLE_LEVEL_NONE;
2297 }
2298
2299 if (ut == NULL) {
2300 ut = current_uthread();
2301 }
2302
2303 if (bap && inflight && !ut->uu_throttle_bc) {
2304 thread_throttle_level = GET_BUFATTR_IO_TIER(bap);
2305 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2306 thread_throttle_level--;
2307 }
2308 } else {
2309 thread_throttle_level = throttle_get_thread_throttle_level(ut);
2310 }
2311
2312 if (thread_throttle_level != THROTTLE_LEVEL_NONE) {
2313 if (!ISSET(flags, B_PASSIVE)) {
2314 info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid();
2315 if (inflight && !ut->uu_throttle_bc) {
2316 if (NULL != bap) {
2317 SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2318 }
2319 OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]);
2320 } else {
2321 microuptime(&info->throttle_window_start_timestamp[thread_throttle_level]);
2322 }
2323 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE,
2324 proc_getpid(current_proc()), thread_throttle_level, 0, 0, 0);
2325 }
2326 microuptime(&info->throttle_last_IO_timestamp[thread_throttle_level]);
2327 }
2328
2329
2330 if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) {
2331 /*
2332 * I'd really like to do the IOSleep here, but
2333 * we may be holding all kinds of filesystem related locks
2334 * and the pages for this I/O marked 'busy'...
2335 * we don't want to cause a normal task to block on
2336 * one of these locks while we're throttling a task marked
2337 * for low priority I/O... we'll mark the uthread and
2338 * do the delay just before we return from the system
2339 * call that triggered this I/O or from vnode_pagein
2340 */
2341 OSAddAtomic(1, &info->throttle_io_count);
2342
2343 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2344 }
2345
2346 return thread_throttle_level;
2347 }
2348
2349 void *
throttle_info_update_by_mount(mount_t mp)2350 throttle_info_update_by_mount(mount_t mp)
2351 {
2352 struct _throttle_io_info_t *info;
2353 uthread_t ut;
2354 boolean_t isssd = FALSE;
2355
2356 ut = current_uthread();
2357
2358 if (mp != NULL) {
2359 if (disk_conditioner_mount_is_ssd(mp)) {
2360 isssd = TRUE;
2361 }
2362 info = &_throttle_io_info[mp->mnt_devbsdunit];
2363 } else {
2364 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2365 }
2366
2367 if (!ut->uu_lowpri_window) {
2368 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2369 }
2370
2371 return info;
2372 }
2373
2374
2375 /*
2376 * KPI routine
2377 *
2378 * this is usually called before every I/O, used for throttled I/O
2379 * book keeping. This routine has low overhead and does not sleep
2380 */
2381 void
throttle_info_update(void * throttle_info,int flags)2382 throttle_info_update(void *throttle_info, int flags)
2383 {
2384 if (throttle_info) {
2385 throttle_info_update_internal(throttle_info, NULL, flags, FALSE, FALSE, NULL);
2386 }
2387 }
2388
2389 /*
2390 * KPI routine (private)
2391 *
2392 * similar to throttle_info_update() but takes an additional argument to
2393 * indicate if the backing device type is SSD or not.
2394 */
2395 void
throttle_info_update_with_type(void * throttle_info,int flags,boolean_t isssd)2396 throttle_info_update_with_type(void *throttle_info, int flags, boolean_t isssd)
2397 {
2398 if (throttle_info) {
2399 throttle_info_update_internal(throttle_info, NULL, flags, isssd, FALSE, NULL);
2400 }
2401 }
2402
2403 /*
2404 * KPI routine
2405 *
2406 * this is usually called before every I/O, used for throttled I/O
2407 * book keeping. This routine has low overhead and does not sleep
2408 */
2409 void
throttle_info_update_by_mask(void * throttle_info_handle,int flags)2410 throttle_info_update_by_mask(void *throttle_info_handle, int flags)
2411 {
2412 void *throttle_info = throttle_info_handle;
2413
2414 /*
2415 * for now we only use the lowest bit of the throttle mask, so the
2416 * handle is the same as the throttle_info. Later if we store a
2417 * set of throttle infos in the handle, we will want to loop through
2418 * them and call throttle_info_update in a loop
2419 */
2420 throttle_info_update(throttle_info, flags);
2421 }
2422 /*
2423 * KPI routine
2424 *
2425 * This routine marks the throttle info as disabled. Used for mount points which
2426 * support I/O scheduling.
2427 */
2428
2429 void
throttle_info_disable_throttle(int devno,boolean_t isfusion)2430 throttle_info_disable_throttle(int devno, boolean_t isfusion)
2431 {
2432 struct _throttle_io_info_t *info;
2433
2434 if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV) {
2435 panic("Illegal devno (%d) passed into throttle_info_disable_throttle()", devno);
2436 }
2437
2438 info = &_throttle_io_info[devno];
2439 // don't disable software throttling on devices that are part of a fusion device
2440 // and override the software throttle periods to use HDD periods
2441 if (isfusion) {
2442 info->throttle_is_fusion_with_priority = isfusion;
2443 throttle_init_throttle_period(info, FALSE);
2444 }
2445 info->throttle_disabled = !info->throttle_is_fusion_with_priority;
2446 return;
2447 }
2448
2449
2450 /*
2451 * KPI routine (private)
2452 * Called to determine if this IO is being throttled to this level so that it can be treated specially
2453 */
2454 int
throttle_info_io_will_be_throttled(void * throttle_info,int policy)2455 throttle_info_io_will_be_throttled(void * throttle_info, int policy)
2456 {
2457 struct _throttle_io_info_t *info = throttle_info;
2458 struct timeval elapsed;
2459 uint64_t elapsed_msecs;
2460 int throttle_level;
2461 int thread_throttle_level;
2462
2463 switch (policy) {
2464 case IOPOL_THROTTLE:
2465 thread_throttle_level = THROTTLE_LEVEL_TIER3;
2466 break;
2467 case IOPOL_UTILITY:
2468 thread_throttle_level = THROTTLE_LEVEL_TIER2;
2469 break;
2470 case IOPOL_STANDARD:
2471 thread_throttle_level = THROTTLE_LEVEL_TIER1;
2472 break;
2473 default:
2474 thread_throttle_level = THROTTLE_LEVEL_TIER0;
2475 break;
2476 }
2477 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
2478 if (info->throttle_inflight_count[throttle_level]) {
2479 break;
2480 }
2481
2482 microuptime(&elapsed);
2483 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
2484 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
2485
2486 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
2487 break;
2488 }
2489 }
2490 if (throttle_level >= thread_throttle_level) {
2491 /*
2492 * we're beyond all of the throttle windows
2493 * so go ahead and treat as normal I/O
2494 */
2495 return THROTTLE_DISENGAGED;
2496 }
2497 /*
2498 * we're in the throttle window
2499 */
2500 return THROTTLE_ENGAGED;
2501 }
2502
2503 int
throttle_lowpri_window(void)2504 throttle_lowpri_window(void)
2505 {
2506 return current_uthread()->uu_lowpri_window;
2507 }
2508
2509 #if CONFIG_PHYS_WRITE_ACCT
2510 extern thread_t pm_sync_thread;
2511 #endif /* CONFIG_PHYS_WRITE_ACCT */
2512
2513 int
spec_strategy(struct vnop_strategy_args * ap)2514 spec_strategy(struct vnop_strategy_args *ap)
2515 {
2516 buf_t bp;
2517 int bflags;
2518 int io_tier;
2519 int passive;
2520 dev_t bdev;
2521 uthread_t ut;
2522 vnode_t vp;
2523 mount_t mp;
2524 struct bufattr *bap;
2525 int strategy_ret;
2526 struct _throttle_io_info_t *throttle_info;
2527 boolean_t isssd = FALSE;
2528 boolean_t inflight = FALSE;
2529 boolean_t upgrade = FALSE;
2530 int code = 0;
2531
2532 #if CONFIG_DELAY_IDLE_SLEEP
2533 proc_t curproc = current_proc();
2534 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2535
2536 bp = ap->a_bp;
2537 bdev = buf_device(bp);
2538 vp = buf_vnode(bp);
2539 mp = vp ? vp->v_mount : NULL;
2540 bap = &bp->b_attr;
2541
2542 #if CONFIG_PHYS_WRITE_ACCT
2543 if (current_thread() == pm_sync_thread) {
2544 OSAddAtomic64(buf_count(bp), (SInt64 *)&(kernel_pm_writes));
2545 }
2546 #endif /* CONFIG_PHYS_WRITE_ACCT */
2547
2548 #if CONFIG_IOSCHED
2549 if (bp->b_flags & B_CLUSTER) {
2550 io_tier = upl_get_cached_tier(bp->b_upl);
2551
2552 if (io_tier == -1) {
2553 io_tier = throttle_get_io_policy(&ut);
2554 }
2555 #if DEVELOPMENT || DEBUG
2556 else {
2557 int my_io_tier = throttle_get_io_policy(&ut);
2558
2559 if (io_tier != my_io_tier) {
2560 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0);
2561 }
2562 }
2563 #endif
2564 } else {
2565 io_tier = throttle_get_io_policy(&ut);
2566 }
2567 #else
2568 io_tier = throttle_get_io_policy(&ut);
2569 #endif
2570 passive = throttle_get_passive_io_policy(&ut);
2571
2572 /*
2573 * Mark if the I/O was upgraded by throttle_get_thread_throttle_level
2574 * while preserving the original issued tier (throttle_get_io_policy
2575 * does not return upgraded tiers)
2576 */
2577 if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) {
2578 #if CONFIG_IOSCHED
2579 if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
2580 upgrade = TRUE;
2581 }
2582 #else /* CONFIG_IOSCHED */
2583 upgrade = TRUE;
2584 #endif /* CONFIG_IOSCHED */
2585 }
2586
2587 if (bp->b_flags & B_META) {
2588 bap->ba_flags |= BA_META;
2589 }
2590
2591 #if CONFIG_IOSCHED
2592 /*
2593 * For metadata reads, ceil the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited, otherwise
2594 * ceil it to IOSCHED_METADATA_TIER. Mark them passive if the I/O tier was upgraded.
2595 * For metadata writes, set the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited. Otherwise
2596 * set it to IOSCHED_METADATA_TIER. In addition, mark them as passive.
2597 */
2598 if (bap->ba_flags & BA_META) {
2599 if ((mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) || (bap->ba_flags & BA_IO_SCHEDULED)) {
2600 if (bp->b_flags & B_READ) {
2601 if ((bap->ba_flags & BA_EXPEDITED_META_IO) && (io_tier > IOSCHED_METADATA_EXPEDITED_TIER)) {
2602 io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
2603 passive = 1;
2604 } else if (io_tier > IOSCHED_METADATA_TIER) {
2605 io_tier = IOSCHED_METADATA_TIER;
2606 passive = 1;
2607 }
2608 } else {
2609 if (bap->ba_flags & BA_EXPEDITED_META_IO) {
2610 io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
2611 } else {
2612 io_tier = IOSCHED_METADATA_TIER;
2613 }
2614 passive = 1;
2615 }
2616 }
2617 }
2618 #endif /* CONFIG_IOSCHED */
2619
2620 SET_BUFATTR_IO_TIER(bap, io_tier);
2621
2622 if (passive) {
2623 bp->b_flags |= B_PASSIVE;
2624 bap->ba_flags |= BA_PASSIVE;
2625 }
2626
2627 #if CONFIG_DELAY_IDLE_SLEEP
2628 if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) {
2629 bap->ba_flags |= BA_DELAYIDLESLEEP;
2630 }
2631 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2632
2633 bflags = bp->b_flags;
2634
2635 if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0)) {
2636 bufattr_markquickcomplete(bap);
2637 }
2638
2639 if (bflags & B_READ) {
2640 code |= DKIO_READ;
2641 }
2642 if (bflags & B_ASYNC) {
2643 code |= DKIO_ASYNC;
2644 }
2645
2646 if (bap->ba_flags & BA_META) {
2647 code |= DKIO_META;
2648 } else if (bflags & B_PAGEIO) {
2649 code |= DKIO_PAGING;
2650 }
2651
2652 if (io_tier != 0) {
2653 code |= DKIO_THROTTLE;
2654 }
2655
2656 code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK);
2657
2658 if (bflags & B_PASSIVE) {
2659 code |= DKIO_PASSIVE;
2660 }
2661
2662 if (bap->ba_flags & BA_NOCACHE) {
2663 code |= DKIO_NOCACHE;
2664 }
2665
2666 if (upgrade) {
2667 code |= DKIO_TIER_UPGRADE;
2668 SET(bap->ba_flags, BA_IO_TIER_UPGRADE);
2669 }
2670
2671 if (kdebug_enable) {
2672 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
2673 buf_kernel_addrperm_addr(bp), bdev, buf_blkno(bp), buf_count(bp), 0);
2674 }
2675
2676 #if CONFIG_IO_COMPRESSION_STATS
2677 // Do not run IO Compression Stats when a privilege thread is active
2678 if (!is_vm_privileged() && !is_external_pageout_thread()) {
2679 io_compression_stats(bp);
2680 }
2681 #endif /* CONFIG_IO_COMPRESSION_STATS */
2682 thread_update_io_stats(current_thread(), buf_count(bp), code);
2683
2684 if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
2685 if (!vp->v_un.vu_specinfo->si_initted) {
2686 SPEC_INIT_BSDUNIT(vp, vfs_context_current());
2687 }
2688 if (vp->v_un.vu_specinfo->si_devbsdunit > (LOWPRI_MAX_NUM_DEV - 1)) {
2689 panic("Invalid value (%d) for si_devbsdunit for vnode %p",
2690 vp->v_un.vu_specinfo->si_devbsdunit, vp);
2691 }
2692 if (vp->v_un.vu_specinfo->si_isssd > 1) {
2693 panic("Invalid value (%d) for si_isssd for vnode %p",
2694 vp->v_un.vu_specinfo->si_isssd, vp);
2695 }
2696 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
2697 isssd = vp->v_un.vu_specinfo->si_isssd;
2698 } else if (mp != NULL) {
2699 if (disk_conditioner_mount_is_ssd(mp)) {
2700 isssd = TRUE;
2701 }
2702 /*
2703 * Partially initialized mounts don't have a final devbsdunit and should not be tracked.
2704 * Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value
2705 * (mnt_throttle_mask is initialized and num_trailing_0 would be 0)
2706 */
2707 if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) {
2708 inflight = TRUE;
2709 }
2710 throttle_info = &_throttle_io_info[mp->mnt_devbsdunit];
2711 } else {
2712 throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2713 }
2714
2715 throttle_info_update_internal(throttle_info, ut, bflags, isssd, inflight, bap);
2716
2717 if ((bflags & B_READ) == 0) {
2718 microuptime(&throttle_info->throttle_last_write_timestamp);
2719
2720 if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
2721 mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp;
2722 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size);
2723 }
2724 } else if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
2725 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size);
2726 }
2727 /*
2728 * The BootCache may give us special information about
2729 * the IO, so it returns special values that we check
2730 * for here.
2731 *
2732 * IO_SATISFIED_BY_CACHE
2733 * The read has been satisfied by the boot cache. Don't
2734 * throttle the thread unnecessarily.
2735 *
2736 * IO_SHOULD_BE_THROTTLED
2737 * The boot cache is playing back a playlist and this IO
2738 * cut through. Throttle it so we're not cutting through
2739 * the boot cache too often.
2740 *
2741 * Note that typical strategy routines are defined with
2742 * a void return so we'll get garbage here. In the
2743 * unlikely case the garbage matches our special return
2744 * value, it's not a big deal since we're only adjusting
2745 * the throttling delay.
2746 */
2747 #define IO_SATISFIED_BY_CACHE ((int)0xcafefeed)
2748 #define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef)
2749 #pragma clang diagnostic push
2750 #pragma clang diagnostic ignored "-Wcast-function-type"
2751
2752 typedef int strategy_fcn_ret_t(struct buf *bp);
2753
2754 strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp);
2755
2756 #pragma clang diagnostic pop
2757
2758 // disk conditioner needs to track when this I/O actually starts
2759 // which means track it after `strategy` which may include delays
2760 // from inflight I/Os
2761 microuptime(&bp->b_timestamp_tv);
2762
2763 if (IO_SATISFIED_BY_CACHE == strategy_ret) {
2764 /*
2765 * If this was a throttled IO satisfied by the boot cache,
2766 * don't delay the thread.
2767 */
2768 throttle_info_reset_window(ut);
2769 } else if (IO_SHOULD_BE_THROTTLED == strategy_ret) {
2770 /*
2771 * If the boot cache indicates this IO should be throttled,
2772 * delay the thread.
2773 */
2774 throttle_info_set_initial_window(ut, throttle_info, TRUE, isssd);
2775 }
2776 return 0;
2777 }
2778
2779
2780 /*
2781 * This is a noop, simply returning what one has been given.
2782 */
2783 int
spec_blockmap(__unused struct vnop_blockmap_args * ap)2784 spec_blockmap(__unused struct vnop_blockmap_args *ap)
2785 {
2786 return ENOTSUP;
2787 }
2788
2789
2790 /*
2791 * Device close routine
2792 */
2793 int
spec_close(struct vnop_close_args * ap)2794 spec_close(struct vnop_close_args *ap)
2795 {
2796 struct vnode *vp = ap->a_vp;
2797 dev_t dev = vp->v_rdev;
2798 int error = 0;
2799 int flags = ap->a_fflag;
2800 struct proc *p = vfs_context_proc(ap->a_context);
2801 struct session *sessp;
2802 struct pgrp *pg;
2803
2804 switch (vp->v_type) {
2805 case VCHR:
2806 /*
2807 * Hack: a tty device that is a controlling terminal
2808 * has a reference from the session structure.
2809 * We cannot easily tell that a character device is
2810 * a controlling terminal, unless it is the closing
2811 * process' controlling terminal. In that case,
2812 * if the reference count is 1 (this is the very
2813 * last close)
2814 */
2815 pg = proc_pgrp(p, &sessp);
2816 devsw_lock(dev, S_IFCHR);
2817 if (sessp != SESSION_NULL) {
2818 if (vp == sessp->s_ttyvp && vcount(vp) == 1) {
2819 struct tty *tp = TTY_NULL;
2820
2821 devsw_unlock(dev, S_IFCHR);
2822 session_lock(sessp);
2823 if (vp == sessp->s_ttyvp) {
2824 tp = session_clear_tty_locked(sessp);
2825 }
2826 session_unlock(sessp);
2827
2828 if (tp != TTY_NULL) {
2829 ttyfree(tp);
2830 }
2831 devsw_lock(dev, S_IFCHR);
2832 }
2833 }
2834 pgrp_rele(pg);
2835
2836 if (--vp->v_specinfo->si_opencount < 0) {
2837 panic("negative open count (c, %u, %u)", major(dev), minor(dev));
2838 }
2839
2840 /*
2841 * close on last reference or on vnode revoke call
2842 */
2843 if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0) {
2844 error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p);
2845 }
2846
2847 devsw_unlock(dev, S_IFCHR);
2848 break;
2849
2850 case VBLK:
2851 /*
2852 * If there is more than one outstanding open, don't
2853 * send the close to the device.
2854 */
2855 devsw_lock(dev, S_IFBLK);
2856 if (vcount(vp) > 1) {
2857 vp->v_specinfo->si_opencount--;
2858 devsw_unlock(dev, S_IFBLK);
2859 return 0;
2860 }
2861 devsw_unlock(dev, S_IFBLK);
2862
2863 /*
2864 * On last close of a block device (that isn't mounted)
2865 * we must invalidate any in core blocks, so that
2866 * we can, for instance, change floppy disks.
2867 */
2868 if ((error = spec_fsync_internal(vp, MNT_WAIT, ap->a_context))) {
2869 return error;
2870 }
2871
2872 error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
2873 if (error) {
2874 return error;
2875 }
2876
2877 devsw_lock(dev, S_IFBLK);
2878
2879 if (--vp->v_specinfo->si_opencount < 0) {
2880 panic("negative open count (b, %u, %u)", major(dev), minor(dev));
2881 }
2882
2883 if (vcount(vp) == 0) {
2884 error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p);
2885 }
2886
2887 devsw_unlock(dev, S_IFBLK);
2888 break;
2889
2890 default:
2891 panic("spec_close: not special");
2892 return EBADF;
2893 }
2894
2895 return error;
2896 }
2897
2898 /*
2899 * Return POSIX pathconf information applicable to special devices.
2900 */
2901 int
spec_pathconf(struct vnop_pathconf_args * ap)2902 spec_pathconf(struct vnop_pathconf_args *ap)
2903 {
2904 switch (ap->a_name) {
2905 case _PC_LINK_MAX:
2906 *ap->a_retval = LINK_MAX;
2907 return 0;
2908 case _PC_MAX_CANON:
2909 *ap->a_retval = MAX_CANON;
2910 return 0;
2911 case _PC_MAX_INPUT:
2912 *ap->a_retval = MAX_INPUT;
2913 return 0;
2914 case _PC_PIPE_BUF:
2915 *ap->a_retval = PIPE_BUF;
2916 return 0;
2917 case _PC_CHOWN_RESTRICTED:
2918 *ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */
2919 return 0;
2920 case _PC_VDISABLE:
2921 *ap->a_retval = _POSIX_VDISABLE;
2922 return 0;
2923 default:
2924 return EINVAL;
2925 }
2926 /* NOTREACHED */
2927 }
2928
2929 /*
2930 * Special device failed operation
2931 */
2932 int
spec_ebadf(__unused void * dummy)2933 spec_ebadf(__unused void *dummy)
2934 {
2935 return EBADF;
2936 }
2937
2938 /* Blktooff derives file offset from logical block number */
2939 int
spec_blktooff(struct vnop_blktooff_args * ap)2940 spec_blktooff(struct vnop_blktooff_args *ap)
2941 {
2942 struct vnode *vp = ap->a_vp;
2943
2944 switch (vp->v_type) {
2945 case VCHR:
2946 *ap->a_offset = (off_t)-1; /* failure */
2947 return ENOTSUP;
2948
2949 case VBLK:
2950 printf("spec_blktooff: not implemented for VBLK\n");
2951 *ap->a_offset = (off_t)-1; /* failure */
2952 return ENOTSUP;
2953
2954 default:
2955 panic("spec_blktooff type");
2956 }
2957 /* NOTREACHED */
2958
2959 return 0;
2960 }
2961
2962 /* Offtoblk derives logical block number from file offset */
2963 int
spec_offtoblk(struct vnop_offtoblk_args * ap)2964 spec_offtoblk(struct vnop_offtoblk_args *ap)
2965 {
2966 struct vnode *vp = ap->a_vp;
2967
2968 switch (vp->v_type) {
2969 case VCHR:
2970 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2971 return ENOTSUP;
2972
2973 case VBLK:
2974 printf("spec_offtoblk: not implemented for VBLK\n");
2975 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2976 return ENOTSUP;
2977
2978 default:
2979 panic("spec_offtoblk type");
2980 }
2981 /* NOTREACHED */
2982
2983 return 0;
2984 }
2985
2986 static int filt_specattach(struct knote *kn, struct kevent_qos_s *kev);
2987 static void filt_specdetach(struct knote *kn);
2988 static int filt_specevent(struct knote *kn, long hint);
2989 static int filt_spectouch(struct knote *kn, struct kevent_qos_s *kev);
2990 static int filt_specprocess(struct knote *kn, struct kevent_qos_s *kev);
2991
2992 SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = {
2993 .f_isfd = 1,
2994 .f_attach = filt_specattach,
2995 .f_detach = filt_specdetach,
2996 .f_event = filt_specevent,
2997 .f_touch = filt_spectouch,
2998 .f_process = filt_specprocess,
2999 };
3000
3001 static void
filt_spec_make_eof(struct knote * kn)3002 filt_spec_make_eof(struct knote *kn)
3003 {
3004 /*
3005 * The spec filter might touch kn_flags from f_event
3006 * without holding "the primitive lock", so make it atomic.
3007 */
3008 os_atomic_or(&kn->kn_flags, EV_EOF | EV_ONESHOT, relaxed);
3009 }
3010
3011 static int
filt_spec_common(struct knote * kn,struct kevent_qos_s * kev,bool attach)3012 filt_spec_common(struct knote *kn, struct kevent_qos_s *kev, bool attach)
3013 {
3014 uthread_t uth = current_uthread();
3015 vfs_context_t ctx = vfs_context_current();
3016 vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp);
3017 __block bool selrecorded = false;
3018 struct select_set *old_wqs;
3019 int64_t data = 0;
3020 int ret, selret;
3021
3022 if (kn->kn_flags & EV_EOF) {
3023 ret = FILTER_ACTIVE;
3024 goto out;
3025 }
3026
3027 if (!attach && vnode_getwithvid(vp, vnode_vid(vp)) != 0) {
3028 filt_spec_make_eof(kn);
3029 ret = FILTER_ACTIVE;
3030 goto out;
3031 }
3032
3033 selspec_record_hook_t cb = ^(struct selinfo *si) {
3034 selspec_attach(kn, si);
3035 selrecorded = true;
3036 };
3037
3038 old_wqs = uth->uu_selset;
3039 uth->uu_selset = SELSPEC_RECORD_MARKER;
3040 selret = VNOP_SELECT(vp, knote_get_seltype(kn), 0, cb, ctx);
3041 uth->uu_selset = old_wqs;
3042
3043 if (!attach) {
3044 vnode_put(vp);
3045 }
3046
3047 if (!selrecorded && selret == 0) {
3048 /*
3049 * The device indicated that there's no data to read,
3050 * but didn't call `selrecord`.
3051 *
3052 * Nothing will be notified of changes to this vnode,
3053 * so return an error back to user space on attach,
3054 * or pretend the knote disappeared for other cases,
3055 * to make it clear that the knote is not attached.
3056 */
3057 if (attach) {
3058 knote_set_error(kn, ENODEV);
3059 return 0;
3060 }
3061
3062 filt_spec_make_eof(kn);
3063 ret = FILTER_ACTIVE;
3064 goto out;
3065 }
3066
3067 if (kn->kn_vnode_use_ofst) {
3068 if (kn->kn_fp->fp_glob->fg_offset >= (uint32_t)selret) {
3069 data = 0;
3070 } else {
3071 data = ((uint32_t)selret) - kn->kn_fp->fp_glob->fg_offset;
3072 }
3073 } else {
3074 data = selret;
3075 }
3076
3077 if (data >= knote_low_watermark(kn)) {
3078 ret = FILTER_ACTIVE;
3079 } else {
3080 ret = 0;
3081 }
3082 out:
3083 if (ret) {
3084 knote_fill_kevent(kn, kev, data);
3085 }
3086 return ret;
3087 }
3088
3089 static int
filt_specattach(struct knote * kn,__unused struct kevent_qos_s * kev)3090 filt_specattach(struct knote *kn, __unused struct kevent_qos_s *kev)
3091 {
3092 vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp); /* Already have iocount, and vnode is alive */
3093 dev_t dev;
3094
3095 assert(vnode_ischr(vp));
3096
3097 dev = vnode_specrdev(vp);
3098
3099 /*
3100 * For a few special kinds of devices, we can attach knotes with
3101 * no restrictions because their "select" vectors return the amount
3102 * of data available. Others require an explicit NOTE_LOWAT with
3103 * data of 1, indicating that the caller doesn't care about actual
3104 * data counts, just an indication that the device has data.
3105 */
3106 if (!kn->kn_vnode_kqok &&
3107 ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) {
3108 knote_set_error(kn, EINVAL);
3109 return 0;
3110 }
3111
3112 return filt_spec_common(kn, kev, true);
3113 }
3114
3115 static void
filt_specdetach(struct knote * kn)3116 filt_specdetach(struct knote *kn)
3117 {
3118 selspec_detach(kn);
3119 }
3120
3121 static int
filt_specevent(struct knote * kn,long hint)3122 filt_specevent(struct knote *kn, long hint)
3123 {
3124 /* Due to selwakeup_internal() on SI_SELSPEC */
3125 assert(KNOTE_IS_AUTODETACHED(kn));
3126 knote_kn_hook_set_raw(kn, NULL);
3127
3128 /* called by selwakeup with the selspec_lock lock held */
3129 if (hint & NOTE_REVOKE) {
3130 filt_spec_make_eof(kn);
3131 }
3132 return FILTER_ACTIVE;
3133 }
3134
3135 static int
filt_spectouch(struct knote * kn,struct kevent_qos_s * kev)3136 filt_spectouch(struct knote *kn, struct kevent_qos_s *kev)
3137 {
3138 kn->kn_sdata = kev->data;
3139 kn->kn_sfflags = kev->fflags;
3140
3141 return filt_spec_common(kn, kev, false);
3142 }
3143
3144 static int
filt_specprocess(struct knote * kn,struct kevent_qos_s * kev)3145 filt_specprocess(struct knote *kn, struct kevent_qos_s *kev)
3146 {
3147 return filt_spec_common(kn, kev, false);
3148 }
3149