1 /*
2 * Copyright (c) 2000-2019 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1989, 1993, 1995
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/kauth.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/conf.h>
70 #include <sys/buf_internal.h>
71 #include <sys/mount_internal.h>
72 #include <sys/vnode_internal.h>
73 #include <sys/file_internal.h>
74 #include <sys/namei.h>
75 #include <sys/stat.h>
76 #include <sys/errno.h>
77 #include <sys/ioctl.h>
78 #include <sys/file.h>
79 #include <sys/user.h>
80 #include <sys/malloc.h>
81 #include <sys/disk.h>
82 #include <sys/uio_internal.h>
83 #include <sys/resource.h>
84 #include <machine/machine_routines.h>
85 #include <miscfs/specfs/specdev.h>
86 #include <vfs/vfs_support.h>
87 #include <vfs/vfs_disk_conditioner.h>
88
89 #include <kern/assert.h>
90 #include <kern/task.h>
91 #include <kern/sched_prim.h>
92 #include <kern/thread.h>
93 #include <kern/policy_internal.h>
94 #include <kern/timer_call.h>
95 #include <kern/waitq.h>
96
97 #include <pexpert/pexpert.h>
98
99 #include <sys/kdebug.h>
100 #include <libkern/section_keywords.h>
101
102 #if CONFIG_IO_COMPRESSION_STATS
103 #include <vfs/vfs_io_compression_stats.h>
104 #endif /* CONFIG_IO_COMPRESSION_STATS */
105
106 #if CONFIG_IOSCHED
107 #include <vm/vm_pageout_xnu.h>
108 #include <vm/vm_object_xnu.h>
109 #endif /* CONFIG_IOSCHED */
110
111 /* XXX following three prototypes should be in a header file somewhere */
112 extern dev_t chrtoblk(dev_t dev);
113 extern boolean_t iskmemdev(dev_t dev);
114 extern int bpfkqfilter(dev_t dev, struct knote *kn);
115 extern int ptsd_kqfilter(dev_t, struct knote *);
116 extern int ptmx_kqfilter(dev_t, struct knote *);
117 #if CONFIG_PHYS_WRITE_ACCT
118 uint64_t kernel_pm_writes; // to track the sync writes occurring during power management transitions
119 #endif /* CONFIG_PHYS_WRITE_ACCT */
120
121
122 struct vnode *speclisth[SPECHSZ];
123
124 /* symbolic sleep message strings for devices */
125 char devopn[] = "devopn";
126 char devio[] = "devio";
127 char devwait[] = "devwait";
128 char devin[] = "devin";
129 char devout[] = "devout";
130 char devioc[] = "devioc";
131 char devcls[] = "devcls";
132
133 #define VOPFUNC int (*)(void *)
134
135 int(**spec_vnodeop_p)(void *);
136 const struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
137 { .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC)(void (*)(void))vn_default_error },
138 { .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC)spec_lookup }, /* lookup */
139 { .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC)err_create }, /* create */
140 { .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC)err_mknod }, /* mknod */
141 { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC)spec_open }, /* open */
142 { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC)spec_close }, /* close */
143 { .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC)spec_access }, /* access */
144 { .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC)spec_getattr }, /* getattr */
145 { .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC)spec_setattr }, /* setattr */
146 { .opve_op = &vnop_read_desc, .opve_impl = (VOPFUNC)spec_read }, /* read */
147 { .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC)spec_write }, /* write */
148 { .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC)spec_ioctl }, /* ioctl */
149 { .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC)spec_select }, /* select */
150 { .opve_op = &vnop_revoke_desc, .opve_impl = (VOPFUNC)nop_revoke }, /* revoke */
151 { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC)err_mmap }, /* mmap */
152 { .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC)spec_fsync }, /* fsync */
153 { .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC)err_remove }, /* remove */
154 { .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC)err_link }, /* link */
155 { .opve_op = &vnop_rename_desc, .opve_impl = (VOPFUNC)err_rename }, /* rename */
156 { .opve_op = &vnop_mkdir_desc, .opve_impl = (VOPFUNC)err_mkdir }, /* mkdir */
157 { .opve_op = &vnop_rmdir_desc, .opve_impl = (VOPFUNC)err_rmdir }, /* rmdir */
158 { .opve_op = &vnop_symlink_desc, .opve_impl = (VOPFUNC)err_symlink }, /* symlink */
159 { .opve_op = &vnop_readdir_desc, .opve_impl = (VOPFUNC)err_readdir }, /* readdir */
160 { .opve_op = &vnop_readlink_desc, .opve_impl = (VOPFUNC)err_readlink }, /* readlink */
161 { .opve_op = &vnop_inactive_desc, .opve_impl = (VOPFUNC)nop_inactive }, /* inactive */
162 { .opve_op = &vnop_reclaim_desc, .opve_impl = (VOPFUNC)nop_reclaim }, /* reclaim */
163 { .opve_op = &vnop_strategy_desc, .opve_impl = (VOPFUNC)spec_strategy }, /* strategy */
164 { .opve_op = &vnop_pathconf_desc, .opve_impl = (VOPFUNC)spec_pathconf }, /* pathconf */
165 { .opve_op = &vnop_advlock_desc, .opve_impl = (VOPFUNC)err_advlock }, /* advlock */
166 { .opve_op = &vnop_bwrite_desc, .opve_impl = (VOPFUNC)spec_bwrite }, /* bwrite */
167 { .opve_op = &vnop_pagein_desc, .opve_impl = (VOPFUNC)err_pagein }, /* Pagein */
168 { .opve_op = &vnop_pageout_desc, .opve_impl = (VOPFUNC)err_pageout }, /* Pageout */
169 { .opve_op = &vnop_copyfile_desc, .opve_impl = (VOPFUNC)err_copyfile }, /* Copyfile */
170 { .opve_op = &vnop_blktooff_desc, .opve_impl = (VOPFUNC)spec_blktooff }, /* blktooff */
171 { .opve_op = &vnop_offtoblk_desc, .opve_impl = (VOPFUNC)spec_offtoblk }, /* offtoblk */
172 { .opve_op = &vnop_blockmap_desc, .opve_impl = (VOPFUNC)spec_blockmap }, /* blockmap */
173 { .opve_op = (struct vnodeop_desc*)NULL, .opve_impl = (int (*)(void *))NULL }
174 };
175 const struct vnodeopv_desc spec_vnodeop_opv_desc =
176 { .opv_desc_vector_p = &spec_vnodeop_p, .opv_desc_ops = spec_vnodeop_entries };
177
178
179 static void set_blocksize(vnode_t, dev_t);
180
181 #define LOWPRI_TIER1_WINDOW_MSECS 25
182 #define LOWPRI_TIER2_WINDOW_MSECS 100
183 #define LOWPRI_TIER3_WINDOW_MSECS 500
184
185 #define LOWPRI_TIER1_IO_PERIOD_MSECS 40
186 #define LOWPRI_TIER2_IO_PERIOD_MSECS 85
187 #define LOWPRI_TIER3_IO_PERIOD_MSECS 200
188
189 #define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5
190 #define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15
191 #define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25
192
193
194 int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = {
195 0,
196 LOWPRI_TIER1_WINDOW_MSECS,
197 LOWPRI_TIER2_WINDOW_MSECS,
198 LOWPRI_TIER3_WINDOW_MSECS,
199 };
200
201 int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = {
202 0,
203 LOWPRI_TIER1_IO_PERIOD_MSECS,
204 LOWPRI_TIER2_IO_PERIOD_MSECS,
205 LOWPRI_TIER3_IO_PERIOD_MSECS,
206 };
207
208 int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = {
209 0,
210 LOWPRI_TIER1_IO_PERIOD_SSD_MSECS,
211 LOWPRI_TIER2_IO_PERIOD_SSD_MSECS,
212 LOWPRI_TIER3_IO_PERIOD_SSD_MSECS,
213 };
214
215
216 int throttled_count[THROTTLE_LEVEL_END + 1];
217
218 struct _throttle_io_info_t {
219 lck_mtx_t throttle_lock;
220
221 struct timeval throttle_last_write_timestamp;
222 struct timeval throttle_min_timer_deadline;
223 struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */
224 struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1];
225 pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1];
226 struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1];
227 int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1];
228
229 TAILQ_HEAD(, uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */
230 int throttle_next_wake_level;
231
232 thread_call_t throttle_timer_call;
233 int32_t throttle_timer_ref;
234 int32_t throttle_timer_active;
235
236 int32_t throttle_io_count;
237 int32_t throttle_io_count_begin;
238 int *throttle_io_periods;
239 uint32_t throttle_io_period_num;
240
241 int32_t throttle_refcnt;
242 int32_t throttle_alloc;
243 int32_t throttle_disabled;
244 int32_t throttle_is_fusion_with_priority;
245 };
246
247 struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV];
248
249
250 int lowpri_throttle_enabled = 1;
251
252 static int spec_close_internal(struct vnode *vp, dev_t dev, int flags, vfs_context_t ctx);
253
254 static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level);
255 static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap);
256 static int throttle_get_thread_throttle_level(uthread_t ut);
257 static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier);
258 void throttle_info_mount_reset_period(mount_t mp, int isssd);
259
260 /*
261 * Trivial lookup routine that always fails.
262 */
263 int
spec_lookup(struct vnop_lookup_args * ap)264 spec_lookup(struct vnop_lookup_args *ap)
265 {
266 *ap->a_vpp = NULL;
267 return ENOTDIR;
268 }
269
270 static void
set_blocksize(struct vnode * vp,dev_t dev)271 set_blocksize(struct vnode *vp, dev_t dev)
272 {
273 int (*size)(dev_t);
274 int rsize;
275
276 if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) {
277 rsize = (*size)(dev);
278 if (rsize <= 0) { /* did size fail? */
279 vp->v_specsize = DEV_BSIZE;
280 } else {
281 vp->v_specsize = rsize;
282 }
283 } else {
284 vp->v_specsize = DEV_BSIZE;
285 }
286 }
287
288 void
set_fsblocksize(struct vnode * vp)289 set_fsblocksize(struct vnode *vp)
290 {
291 if (vp->v_type == VBLK) {
292 dev_t dev = (dev_t)vp->v_rdev;
293 int maj = major(dev);
294
295 if ((u_int)maj >= (u_int)nblkdev) {
296 return;
297 }
298
299 vnode_lock(vp);
300 set_blocksize(vp, dev);
301 vnode_unlock(vp);
302 }
303 }
304
305 static void
spec_init_bsdunit(vnode_t vp,vfs_context_t ctx,const char * caller)306 spec_init_bsdunit(vnode_t vp, vfs_context_t ctx, const char* caller)
307 {
308 int isssd = 0;
309 uint64_t throttle_mask = 0;
310 uint32_t devbsdunit = 0;
311
312 if (VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ctx)) {
313 isssd = 0;
314 }
315 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL)) {
316 throttle_mask = 0;
317 }
318
319 if (throttle_mask != 0) {
320 /*
321 * as a reasonable approximation, only use the lowest bit of the mask
322 * to generate a disk unit number
323 */
324 devbsdunit = num_trailing_0(throttle_mask);
325 } else {
326 devbsdunit = 0;
327 }
328
329 if (vp->v_un.vu_specinfo->si_initted == 0) {
330 vnode_lock(vp);
331 if (vp->v_un.vu_specinfo->si_initted == 0) {
332 vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
333 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
334 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
335 vp->v_un.vu_specinfo->si_throttleable = 1;
336 vp->v_un.vu_specinfo->si_initted = 1;
337 }
338 vnode_unlock(vp);
339 printf("%s : si_devbsdunit initialized to (%d), throttle_mask is (0x%llx), isssd is (%d)\n",
340 caller, vp->v_un.vu_specinfo->si_devbsdunit,
341 vp->v_un.vu_specinfo->si_throttle_mask,
342 vp->v_un.vu_specinfo->si_isssd);
343 }
344 }
345
346 #define SPEC_INIT_BSDUNIT(vp, ctx) spec_init_bsdunit((vp), (ctx), __FUNCTION__)
347
348 /*
349 * Open a special file.
350 */
351 int
spec_open(struct vnop_open_args * ap)352 spec_open(struct vnop_open_args *ap)
353 {
354 struct proc *p = vfs_context_proc(ap->a_context);
355 kauth_cred_t cred = vfs_context_ucred(ap->a_context);
356 struct vnode *vp = ap->a_vp;
357 dev_t bdev, dev = (dev_t)vp->v_rdev;
358 int maj = major(dev);
359 int error;
360
361 /*
362 * Don't allow open if fs is mounted -nodev.
363 */
364 if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV)) {
365 return ENXIO;
366 }
367
368 switch (vp->v_type) {
369 case VCHR:
370 if ((u_int)maj >= (u_int)nchrdev) {
371 return ENXIO;
372 }
373 if (cred != FSCRED && (ap->a_mode & FWRITE)) {
374 #if 0
375 /*
376 * When running in very secure mode, do not allow
377 * opens for writing of any disk character devices.
378 */
379 if (securelevel >= 2 && isdisk(dev, VCHR)) {
380 return EPERM;
381 }
382 #endif
383
384 /* Never allow writing to /dev/mem or /dev/kmem */
385 if (iskmemdev(dev)) {
386 return EPERM;
387 }
388 /*
389 * When running in secure mode, do not allow opens for
390 * writing of character devices whose corresponding block
391 * devices are currently mounted.
392 */
393 if (securelevel >= 1) {
394 if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(bdev, VBLK, &error)) {
395 return error;
396 }
397 }
398 }
399
400 devsw_lock(dev, S_IFCHR);
401 error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p);
402
403 if (error == 0) {
404 vp->v_specinfo->si_opencount++;
405 }
406
407 devsw_unlock(dev, S_IFCHR);
408
409 if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
410 int isssd = 0;
411 uint64_t throttle_mask = 0;
412 uint32_t devbsdunit = 0;
413
414 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL) == 0) {
415 if (throttle_mask != 0 &&
416 VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ap->a_context) == 0) {
417 /*
418 * as a reasonable approximation, only use the lowest bit of the mask
419 * to generate a disk unit number
420 */
421 devbsdunit = num_trailing_0(throttle_mask);
422
423 vnode_lock(vp);
424
425 vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
426 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
427 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
428 vp->v_un.vu_specinfo->si_throttleable = 1;
429 vp->v_un.vu_specinfo->si_initted = 1;
430
431 vnode_unlock(vp);
432 }
433 }
434 if (vp->v_un.vu_specinfo->si_initted == 0) {
435 vnode_lock(vp);
436 vp->v_un.vu_specinfo->si_initted = 1;
437 vnode_unlock(vp);
438 }
439 }
440 return error;
441
442 case VBLK:
443 if ((u_int)maj >= (u_int)nblkdev) {
444 return ENXIO;
445 }
446 /*
447 * When running in very secure mode, do not allow
448 * opens for writing of any disk block devices.
449 */
450 if (securelevel >= 2 && cred != FSCRED &&
451 (ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK) {
452 return EPERM;
453 }
454 /*
455 * Do not allow opens of block devices that are
456 * currently mounted.
457 */
458 if ((error = vfs_mountedon(vp))) {
459 return error;
460 }
461
462 devsw_lock(dev, S_IFBLK);
463 error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p);
464 if (!error) {
465 vp->v_specinfo->si_opencount++;
466 }
467 devsw_unlock(dev, S_IFBLK);
468
469 if (!error) {
470 u_int64_t blkcnt;
471 u_int32_t blksize;
472 int setsize = 0;
473 u_int32_t size512 = 512;
474
475 if (bdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
476 SPEC_INIT_BSDUNIT(vp, ap->a_context);
477 }
478
479 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, (caddr_t)&blksize, 0, ap->a_context)) {
480 /* Switch to 512 byte sectors (temporarily) */
481
482 if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&size512, FWRITE, ap->a_context)) {
483 /* Get the number of 512 byte physical blocks. */
484 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, ap->a_context)) {
485 setsize = 1;
486 }
487 }
488 /* If it doesn't set back, we can't recover */
489 if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, ap->a_context)) {
490 /* Perform an explicit close on the block device, as the device is already open */
491 spec_close_internal(vp, dev, ap->a_mode, ap->a_context);
492
493 error = ENXIO;
494 }
495 }
496
497
498 vnode_lock(vp);
499 set_blocksize(vp, dev);
500
501 /*
502 * Cache the size in bytes of the block device for later
503 * use by spec_write().
504 */
505 if (setsize) {
506 vp->v_specdevsize = blkcnt * (u_int64_t)size512;
507 } else {
508 vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */
509 }
510 vnode_unlock(vp);
511 }
512 return error;
513 default:
514 panic("spec_open type");
515 }
516 return 0;
517 }
518
519 /*
520 * Vnode op for read
521 */
522 int
spec_read(struct vnop_read_args * ap)523 spec_read(struct vnop_read_args *ap)
524 {
525 struct vnode *vp = ap->a_vp;
526 struct uio *uio = ap->a_uio;
527 struct buf *bp;
528 daddr64_t bn, nextbn;
529 long bscale;
530 int devBlockSize = 0;
531 size_t bsize, n, on;
532 int error = 0;
533 dev_t dev;
534
535 #if DIAGNOSTIC
536 if (uio->uio_rw != UIO_READ) {
537 panic("spec_read mode");
538 }
539 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
540 panic("spec_read proc");
541 }
542 #endif
543 if (uio_resid(uio) == 0) {
544 return 0;
545 }
546
547 switch (vp->v_type) {
548 case VCHR:
549 {
550 struct _throttle_io_info_t *throttle_info = NULL;
551 int thread_throttle_level;
552 uint64_t blkno = 0;
553 uint32_t iolen = 0;
554 int ddisk = 0;
555 int ktrace_code = DKIO_READ;
556 devBlockSize = vp->v_specsize;
557 uintptr_t our_id = 0;
558
559 if (cdevsw[major(vp->v_rdev)].d_type == D_DISK) {
560 ddisk = 1;
561 }
562
563 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
564 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
565 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
566 }
567
568 if (kdebug_enable && ddisk) {
569 if (devBlockSize == 0) {
570 devBlockSize = 512; // default sector size
571 }
572
573 if (uio_offset(uio) && devBlockSize) {
574 blkno = ((uint64_t) uio_offset(uio) / ((uint64_t)devBlockSize));
575 }
576 iolen = (int) uio_resid(uio);
577 our_id = (uintptr_t)thread_tid(current_thread());
578 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
579 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
580 vp->v_rdev, blkno, iolen, 0);
581 }
582
583 error = (*cdevsw[major(vp->v_rdev)].d_read)
584 (vp->v_rdev, uio, ap->a_ioflag);
585
586
587 if (kdebug_enable && ddisk) {
588 uint32_t residual = (uint32_t)uio_resid(uio);
589 ktrace_code |= DKIO_DONE;
590 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
591 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
592 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
593 }
594
595 if (throttle_info) {
596 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
597 }
598
599 return error;
600 }
601
602 case VBLK:
603 if (uio->uio_offset < 0) {
604 return EINVAL;
605 }
606
607 dev = vp->v_rdev;
608
609 devBlockSize = vp->v_specsize;
610
611 if (devBlockSize > PAGE_SIZE) {
612 return EINVAL;
613 }
614
615 bscale = PAGE_SIZE / devBlockSize;
616 bsize = bscale * devBlockSize;
617
618 do {
619 on = uio->uio_offset % bsize;
620
621 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~(bscale - 1));
622
623 if (vp->v_speclastr + bscale == bn) {
624 nextbn = bn + bscale;
625 error = buf_breadn(vp, bn, (int)bsize, &nextbn,
626 (int *)&bsize, 1, NOCRED, &bp);
627 } else {
628 error = buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
629 }
630
631 vnode_lock(vp);
632 vp->v_speclastr = bn;
633 vnode_unlock(vp);
634
635 n = bsize - buf_resid(bp);
636 if ((on > n) || error) {
637 if (!error) {
638 error = EINVAL;
639 }
640 buf_brelse(bp);
641 return error;
642 }
643 n = MIN((n - on), (size_t)uio_resid(uio));
644
645 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
646 if (n + on == bsize) {
647 buf_markaged(bp);
648 }
649 buf_brelse(bp);
650 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
651 return error;
652
653 default:
654 panic("spec_read type");
655 }
656 /* NOTREACHED */
657
658 return 0;
659 }
660
661 /*
662 * Vnode op for write
663 */
664 int
spec_write(struct vnop_write_args * ap)665 spec_write(struct vnop_write_args *ap)
666 {
667 struct vnode *vp = ap->a_vp;
668 struct uio *uio = ap->a_uio;
669 struct buf *bp;
670 daddr64_t bn;
671 int blkmask, bscale;
672 int io_sync;
673 int devBlockSize = 0;
674 size_t bsize, n, on;
675 int error = 0;
676 dev_t dev;
677
678 #if DIAGNOSTIC
679 if (uio->uio_rw != UIO_WRITE) {
680 panic("spec_write mode");
681 }
682 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
683 panic("spec_write proc");
684 }
685 #endif
686
687 switch (vp->v_type) {
688 case VCHR:
689 {
690 struct _throttle_io_info_t *throttle_info = NULL;
691 int thread_throttle_level;
692 dev = vp->v_rdev;
693 devBlockSize = vp->v_specsize;
694 uint32_t iolen = 0;
695 uint64_t blkno = 0;
696 int ddisk = 0;
697 int ktrace_code = 0; // write is implied; read must be OR'd in.
698 uintptr_t our_id = 0;
699
700 if (cdevsw[major(dev)].d_type == D_DISK) {
701 ddisk = 1;
702 }
703
704 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
705 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
706
707 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
708
709 microuptime(&throttle_info->throttle_last_write_timestamp);
710 }
711
712 if (kdebug_enable && ddisk) {
713 if (devBlockSize == 0) {
714 devBlockSize = 512; // default sector size
715 }
716 if ((uio_offset(uio) != 0) && devBlockSize) {
717 blkno = ((uint64_t)uio_offset(uio)) / ((uint64_t)devBlockSize);
718 }
719 iolen = (int)uio_resid(uio);
720 our_id = (uintptr_t)thread_tid(current_thread());
721 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
722 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
723 vp->v_rdev, blkno, iolen, 0);
724 }
725 error = (*cdevsw[major(vp->v_rdev)].d_write)
726 (vp->v_rdev, uio, ap->a_ioflag);
727
728 if (kdebug_enable && ddisk) {
729 //emit the I/O completion
730 uint32_t residual = (uint32_t)uio_resid(uio);
731 ktrace_code |= DKIO_DONE;
732 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
733 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
734 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
735 }
736
737 if (throttle_info) {
738 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
739 }
740
741 return error;
742 }
743
744 case VBLK:
745 if (uio_resid(uio) == 0) {
746 return 0;
747 }
748 if (uio->uio_offset < 0) {
749 return EINVAL;
750 }
751
752 io_sync = (ap->a_ioflag & IO_SYNC);
753
754 dev = (vp->v_rdev);
755
756 devBlockSize = vp->v_specsize;
757 if (devBlockSize > PAGE_SIZE) {
758 return EINVAL;
759 }
760
761 bscale = PAGE_SIZE / devBlockSize;
762 blkmask = bscale - 1;
763 bsize = bscale * devBlockSize;
764
765
766 do {
767 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~blkmask);
768 on = uio->uio_offset % bsize;
769
770 n = MIN((bsize - on), (size_t)uio_resid(uio));
771
772 /*
773 * Use buf_getblk() as an optimization IFF:
774 *
775 * 1) We are reading exactly a block on a block
776 * aligned boundary
777 * 2) We know the size of the device from spec_open
778 * 3) The read doesn't span the end of the device
779 *
780 * Otherwise, we fall back on buf_bread().
781 */
782 if (n == bsize &&
783 vp->v_specdevsize != (u_int64_t)0 &&
784 (uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) {
785 /* reduce the size of the read to what is there */
786 n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize;
787 }
788
789 if (n == bsize) {
790 bp = buf_getblk(vp, bn, (int)bsize, 0, 0, BLK_WRITE);
791 } else {
792 error = (int)buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
793 }
794
795 /* Translate downstream error for upstream, if needed */
796 if (!error) {
797 error = (int)buf_error(bp);
798 }
799 if (error) {
800 buf_brelse(bp);
801 return error;
802 }
803 n = MIN(n, bsize - buf_resid(bp));
804
805 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
806 if (error) {
807 buf_brelse(bp);
808 return error;
809 }
810 buf_markaged(bp);
811
812 if (io_sync) {
813 error = buf_bwrite(bp);
814 } else {
815 if ((n + on) == bsize) {
816 error = buf_bawrite(bp);
817 } else {
818 error = buf_bdwrite(bp);
819 }
820 }
821 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
822 return error;
823
824 default:
825 panic("spec_write type");
826 }
827 /* NOTREACHED */
828
829 return 0;
830 }
831
832 static void
vnode_spec_revoke_wait(vnode_t vp)833 vnode_spec_revoke_wait(vnode_t vp)
834 {
835 vnode_lock_spin(vp);
836 while (vp->v_specinfo->si_flags & SI_REVOKEWAIT) {
837 msleep(&vp->v_specinfo->si_flags, &vp->v_lock, PVFS | PSPIN,
838 "vnode_spec_revokewait", 0);
839 }
840 vp->v_specinfo->si_flags |= SI_REVOKEWAIT;
841 vnode_unlock(vp);
842 }
843
844 static void
vnode_spec_revoke_done(vnode_t vp)845 vnode_spec_revoke_done(vnode_t vp)
846 {
847 vnode_lock_spin(vp);
848 vp->v_specinfo->si_flags &= ~SI_REVOKEWAIT;
849 vnode_unlock(vp);
850 wakeup(&vp->v_specinfo->si_flags);
851 }
852
853 /*
854 * Device ioctl operation.
855 */
856 int
spec_ioctl(struct vnop_ioctl_args * ap)857 spec_ioctl(struct vnop_ioctl_args *ap)
858 {
859 proc_t p = vfs_context_proc(ap->a_context);
860 vnode_t vp = ap->a_vp;
861 dev_t dev = vp->v_rdev;
862 int retval = 0;
863 bool revoke_done_needed = false;
864
865 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START,
866 dev, ap->a_command, ap->a_fflag, vp->v_type, 0);
867
868 switch (vp->v_type) {
869 case VCHR:
870 if (vnode_istty(vp) && (ap->a_command == TIOCREVOKE)) {
871 /*
872 * TIOCREVOKE is called while revoking a tty vnode.
873 * However, at the point where it is called, it is not
874 * possible to detect whether the device is still open
875 * or not to be able to make the call to the device
876 * safely. We detect that here and serialize with a
877 * close on the device through the revoke wait
878 * flag (if needed).
879 */
880 if (vp->v_specinfo->si_opencount == 0) {
881 goto out;
882 }
883 devsw_lock(dev, S_IFCHR);
884 if (vp->v_specinfo->si_opencount == 0) {
885 devsw_unlock(dev, S_IFCHR);
886 goto out;
887 }
888 vnode_spec_revoke_wait(vp);
889 devsw_unlock(dev, S_IFCHR);
890 revoke_done_needed = true;
891 }
892
893 retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data,
894 ap->a_fflag, p);
895
896 if (revoke_done_needed) {
897 vnode_spec_revoke_done(vp);
898 }
899 break;
900
901 case VBLK:
902 retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p);
903 if (!retval && ap->a_command == DKIOCSETBLOCKSIZE) {
904 vp->v_specsize = *(uint32_t *)ap->a_data;
905 }
906 break;
907
908 default:
909 panic("spec_ioctl");
910 /* NOTREACHED */
911 }
912 out:
913 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END,
914 dev, ap->a_command, ap->a_fflag, retval, 0);
915
916 return retval;
917 }
918
919 int
spec_select(struct vnop_select_args * ap)920 spec_select(struct vnop_select_args *ap)
921 {
922 proc_t p = vfs_context_proc(ap->a_context);
923 dev_t dev;
924
925 switch (ap->a_vp->v_type) {
926 default:
927 return 1; /* XXX */
928
929 case VCHR:
930 dev = ap->a_vp->v_rdev;
931 return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p);
932 }
933 }
934
935 int
spec_kqfilter(vnode_t vp,struct knote * kn,struct kevent_qos_s * kev)936 spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_qos_s *kev)
937 {
938 dev_t dev;
939
940 assert(vnode_ischr(vp));
941
942 dev = vnode_specrdev(vp);
943
944 #if NETWORKING
945 /*
946 * Try a bpf device, as defined in bsd/net/bpf.c
947 * If it doesn't error out the attach, then it
948 * claimed it. Otherwise, fall through and try
949 * other attaches.
950 */
951 int32_t tmp_flags = kn->kn_flags;
952 int64_t tmp_sdata = kn->kn_sdata;
953 int res;
954
955 res = bpfkqfilter(dev, kn);
956 if ((kn->kn_flags & EV_ERROR) == 0) {
957 return res;
958 }
959 kn->kn_flags = tmp_flags;
960 kn->kn_sdata = tmp_sdata;
961 #endif
962
963 if (major(dev) >= nchrdev) {
964 knote_set_error(kn, ENXIO);
965 return 0;
966 }
967
968 kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE);
969 kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET);
970
971 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
972 kn->kn_filtid = EVFILTID_PTSD;
973 return ptsd_kqfilter(dev, kn);
974 } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
975 kn->kn_filtid = EVFILTID_PTMX;
976 return ptmx_kqfilter(dev, kn);
977 } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) {
978 /*
979 * TTYs from drivers that use struct ttys use their own filter
980 * routines. The PTC driver doesn't use the tty for character
981 * counts, so it must go through the select fallback.
982 */
983 kn->kn_filtid = EVFILTID_TTY;
984 } else {
985 /* Try to attach to other char special devices */
986 kn->kn_filtid = EVFILTID_SPEC;
987 }
988
989 return knote_fops(kn)->f_attach(kn, kev);
990 }
991
992 /*
993 * Synch buffers associated with a block device
994 */
995 int
spec_fsync_internal(vnode_t vp,int waitfor,__unused vfs_context_t context)996 spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
997 {
998 if (vp->v_type == VCHR) {
999 return 0;
1000 }
1001 /*
1002 * Flush all dirty buffers associated with a block device.
1003 */
1004 buf_flushdirtyblks(vp, (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), 0, "spec_fsync");
1005
1006 return 0;
1007 }
1008
1009 int
spec_fsync(struct vnop_fsync_args * ap)1010 spec_fsync(struct vnop_fsync_args *ap)
1011 {
1012 return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
1013 }
1014
1015
1016 /*
1017 * Just call the device strategy routine
1018 */
1019 void throttle_init(void);
1020
1021
1022 #if 0
1023 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \
1024 do { \
1025 if ((debug_info)->alloc) \
1026 printf("%s: "format, __FUNCTION__, ## args); \
1027 } while(0)
1028
1029 #else
1030 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...)
1031 #endif
1032
1033
1034 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "");
1035 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "");
1036 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "");
1037
1038 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "");
1039 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "");
1040 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "");
1041
1042 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "");
1043 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "");
1044 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "");
1045
1046 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "");
1047
1048
1049 static LCK_GRP_DECLARE(throttle_lock_grp, "throttle I/O");
1050
1051
1052 /*
1053 * throttled I/O helper function
1054 * convert the index of the lowest set bit to a device index
1055 */
1056 int
num_trailing_0(uint64_t n)1057 num_trailing_0(uint64_t n)
1058 {
1059 /*
1060 * since in most cases the number of trailing 0s is very small,
1061 * we simply counting sequentially from the lowest bit
1062 */
1063 if (n == 0) {
1064 return sizeof(n) * 8;
1065 }
1066 int count = 0;
1067 while (!ISSET(n, 1)) {
1068 n >>= 1;
1069 ++count;
1070 }
1071 return count;
1072 }
1073
1074
1075 /*
1076 * Release the reference and if the item was allocated and this is the last
1077 * reference then free it.
1078 *
1079 * This routine always returns the old value.
1080 */
1081 static int
throttle_info_rel(struct _throttle_io_info_t * info)1082 throttle_info_rel(struct _throttle_io_info_t *info)
1083 {
1084 SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt);
1085
1086 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
1087 info, (int)(oldValue - 1), info );
1088
1089 /* The reference count just went negative, very bad */
1090 if (oldValue == 0) {
1091 panic("throttle info ref cnt went negative!");
1092 }
1093
1094 /*
1095 * Once reference count is zero, no one else should be able to take a
1096 * reference
1097 */
1098 if ((oldValue == 1) && (info->throttle_alloc)) {
1099 DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n", info);
1100
1101 lck_mtx_destroy(&info->throttle_lock, &throttle_lock_grp);
1102 kfree_type(struct _throttle_io_info_t, info);
1103 }
1104 return oldValue;
1105 }
1106
1107
1108 /*
1109 * Just take a reference on the throttle info structure.
1110 *
1111 * This routine always returns the old value.
1112 */
1113 static SInt32
throttle_info_ref(struct _throttle_io_info_t * info)1114 throttle_info_ref(struct _throttle_io_info_t *info)
1115 {
1116 SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt);
1117
1118 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
1119 info, (int)(oldValue - 1), info );
1120 /* Allocated items should never have a reference of zero */
1121 if (info->throttle_alloc && (oldValue == 0)) {
1122 panic("Taking a reference without calling create throttle info!");
1123 }
1124
1125 return oldValue;
1126 }
1127
1128 /*
1129 * on entry the throttle_lock is held...
1130 * this function is responsible for taking
1131 * and dropping the reference on the info
1132 * structure which will keep it from going
1133 * away while the timer is running if it
1134 * happens to have been dynamically allocated by
1135 * a network fileystem kext which is now trying
1136 * to free it
1137 */
1138 static uint32_t
throttle_timer_start(struct _throttle_io_info_t * info,boolean_t update_io_count,int wakelevel)1139 throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel)
1140 {
1141 struct timeval elapsed;
1142 struct timeval now;
1143 struct timeval period;
1144 uint64_t elapsed_msecs;
1145 int throttle_level;
1146 int level;
1147 int msecs;
1148 boolean_t throttled = FALSE;
1149 boolean_t need_timer = FALSE;
1150
1151 microuptime(&now);
1152
1153 if (update_io_count == TRUE) {
1154 info->throttle_io_count_begin = info->throttle_io_count;
1155 info->throttle_io_period_num++;
1156
1157 while (wakelevel >= THROTTLE_LEVEL_THROTTLED) {
1158 info->throttle_start_IO_period_timestamp[wakelevel--] = now;
1159 }
1160
1161 info->throttle_min_timer_deadline = now;
1162
1163 msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED];
1164 period.tv_sec = msecs / 1000;
1165 period.tv_usec = (msecs % 1000) * 1000;
1166
1167 timevaladd(&info->throttle_min_timer_deadline, &period);
1168 }
1169 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) {
1170 elapsed = now;
1171 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1172 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1173
1174 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1175 if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1176 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) {
1177 /*
1178 * we had an I/O occur at a higher priority tier within
1179 * this tier's throttle window
1180 */
1181 throttled = TRUE;
1182 }
1183 /*
1184 * we assume that the windows are the same or longer
1185 * as we drop through the throttling tiers... thus
1186 * we can stop looking once we run into a tier with
1187 * threads to schedule regardless of whether it's
1188 * still in its throttling window or not
1189 */
1190 break;
1191 }
1192 }
1193 if (throttled == TRUE) {
1194 break;
1195 }
1196 }
1197 if (throttled == TRUE) {
1198 uint64_t deadline = 0;
1199 struct timeval target;
1200 struct timeval min_target;
1201
1202 /*
1203 * we've got at least one tier still in a throttled window
1204 * so we need a timer running... compute the next deadline
1205 * and schedule it
1206 */
1207 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1208 if (TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1209 continue;
1210 }
1211
1212 target = info->throttle_start_IO_period_timestamp[level];
1213
1214 msecs = info->throttle_io_periods[level];
1215 period.tv_sec = msecs / 1000;
1216 period.tv_usec = (msecs % 1000) * 1000;
1217
1218 timevaladd(&target, &period);
1219
1220 if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) {
1221 min_target = target;
1222 need_timer = TRUE;
1223 }
1224 }
1225 if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) {
1226 if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >)) {
1227 min_target = info->throttle_min_timer_deadline;
1228 }
1229 }
1230
1231 if (info->throttle_timer_active) {
1232 if (thread_call_cancel(info->throttle_timer_call) == FALSE) {
1233 /*
1234 * couldn't kill the timer because it's already
1235 * been dispatched, so don't try to start a new
1236 * one... once we drop the lock, the timer will
1237 * proceed and eventually re-run this function
1238 */
1239 need_timer = FALSE;
1240 } else {
1241 info->throttle_timer_active = 0;
1242 }
1243 }
1244 if (need_timer == TRUE) {
1245 /*
1246 * This is defined as an int (32-bit) rather than a 64-bit
1247 * value because it would need a really big period in the
1248 * order of ~500 days to overflow this. So, we let this be
1249 * 32-bit which allows us to use the clock_interval_to_deadline()
1250 * routine.
1251 */
1252 int target_msecs;
1253
1254 if (info->throttle_timer_ref == 0) {
1255 /*
1256 * take a reference for the timer
1257 */
1258 throttle_info_ref(info);
1259
1260 info->throttle_timer_ref = 1;
1261 }
1262 elapsed = min_target;
1263 timevalsub(&elapsed, &now);
1264 target_msecs = (int)(elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000);
1265
1266 if (target_msecs <= 0) {
1267 /*
1268 * we may have computed a deadline slightly in the past
1269 * due to various factors... if so, just set the timer
1270 * to go off in the near future (we don't need to be precise)
1271 */
1272 target_msecs = 1;
1273 }
1274 clock_interval_to_deadline(target_msecs, 1000000, &deadline);
1275
1276 thread_call_enter_delayed(info->throttle_timer_call, deadline);
1277 info->throttle_timer_active = 1;
1278 }
1279 }
1280 return throttle_level;
1281 }
1282
1283
1284 static void
throttle_timer(struct _throttle_io_info_t * info,__unused thread_call_param_t p)1285 throttle_timer(struct _throttle_io_info_t *info, __unused thread_call_param_t p)
1286 {
1287 uthread_t ut, utlist;
1288 struct timeval elapsed;
1289 struct timeval now;
1290 uint64_t elapsed_msecs;
1291 int throttle_level;
1292 int level;
1293 int wake_level;
1294 caddr_t wake_address = NULL;
1295 boolean_t update_io_count = FALSE;
1296 boolean_t need_wakeup = FALSE;
1297 boolean_t need_release = FALSE;
1298
1299 ut = NULL;
1300 lck_mtx_lock(&info->throttle_lock);
1301
1302 info->throttle_timer_active = 0;
1303 microuptime(&now);
1304
1305 elapsed = now;
1306 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]);
1307 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1308
1309 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) {
1310 wake_level = info->throttle_next_wake_level;
1311
1312 for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) {
1313 elapsed = now;
1314 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[wake_level]);
1315 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1316
1317 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1318 /*
1319 * we're closing out the current IO period...
1320 * if we have a waiting thread, wake it up
1321 * after we have reset the I/O window info
1322 */
1323 need_wakeup = TRUE;
1324 update_io_count = TRUE;
1325
1326 info->throttle_next_wake_level = wake_level - 1;
1327
1328 if (info->throttle_next_wake_level == THROTTLE_LEVEL_START) {
1329 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1330 }
1331
1332 break;
1333 }
1334 wake_level--;
1335
1336 if (wake_level == THROTTLE_LEVEL_START) {
1337 wake_level = THROTTLE_LEVEL_END;
1338 }
1339 }
1340 }
1341 if (need_wakeup == TRUE) {
1342 if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1343 ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]);
1344 TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist);
1345 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1346 ut->uu_is_throttled = false;
1347
1348 wake_address = (caddr_t)&ut->uu_on_throttlelist;
1349 }
1350 } else {
1351 wake_level = THROTTLE_LEVEL_START;
1352 }
1353
1354 throttle_level = throttle_timer_start(info, update_io_count, wake_level);
1355
1356 if (wake_address != NULL) {
1357 wakeup(wake_address);
1358 }
1359
1360 for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) {
1361 TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) {
1362 TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist);
1363 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1364 ut->uu_is_throttled = false;
1365
1366 wakeup(&ut->uu_on_throttlelist);
1367 }
1368 }
1369 if (info->throttle_timer_active == 0 && info->throttle_timer_ref) {
1370 info->throttle_timer_ref = 0;
1371 need_release = TRUE;
1372 }
1373 lck_mtx_unlock(&info->throttle_lock);
1374
1375 if (need_release == TRUE) {
1376 throttle_info_rel(info);
1377 }
1378 }
1379
1380
1381 static int
throttle_add_to_list(struct _throttle_io_info_t * info,uthread_t ut,int mylevel,boolean_t insert_tail)1382 throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail)
1383 {
1384 boolean_t start_timer = FALSE;
1385 int level = THROTTLE_LEVEL_START;
1386
1387 if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) {
1388 info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel];
1389 start_timer = TRUE;
1390 }
1391
1392 if (insert_tail == TRUE) {
1393 TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1394 } else {
1395 TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1396 }
1397
1398 ut->uu_on_throttlelist = (int8_t)mylevel;
1399
1400 if (start_timer == TRUE) {
1401 /* we may need to start or rearm the timer */
1402 level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START);
1403
1404 if (level == THROTTLE_LEVEL_END) {
1405 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
1406 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1407
1408 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1409 }
1410 }
1411 }
1412 return level;
1413 }
1414
1415 static void
throttle_init_throttle_window(void)1416 throttle_init_throttle_window(void)
1417 {
1418 int throttle_window_size;
1419
1420 /*
1421 * The hierarchy of throttle window values is as follows:
1422 * - Global defaults
1423 * - Device tree properties
1424 * - Boot-args
1425 * All values are specified in msecs.
1426 */
1427
1428 #if (XNU_TARGET_OS_OSX && __arm64__)
1429 /*
1430 * IO Tier EDT overrides are meant for
1431 * some arm platforms but not for
1432 * macs.
1433 */
1434 #else /* (XNU_TARGET_OS_OSX && __arm64__) */
1435 /* Override global values with device-tree properties */
1436 if (PE_get_default("kern.io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1437 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1438 }
1439
1440 if (PE_get_default("kern.io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1441 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1442 }
1443
1444 if (PE_get_default("kern.io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1445 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1446 }
1447 #endif /* (XNU_TARGET_OS_OSX && __arm64__) */
1448
1449 /* Override with boot-args */
1450 if (PE_parse_boot_argn("io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1451 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1452 }
1453
1454 if (PE_parse_boot_argn("io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1455 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1456 }
1457
1458 if (PE_parse_boot_argn("io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1459 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1460 }
1461 }
1462
1463 static void
throttle_init_throttle_period(struct _throttle_io_info_t * info,boolean_t isssd)1464 throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd)
1465 {
1466 int throttle_period_size;
1467
1468 /*
1469 * The hierarchy of throttle period values is as follows:
1470 * - Global defaults
1471 * - Device tree properties
1472 * - Boot-args
1473 * All values are specified in msecs.
1474 */
1475
1476 /* Assign global defaults */
1477 if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0)) {
1478 info->throttle_io_periods = &throttle_io_period_ssd_msecs[0];
1479 } else {
1480 info->throttle_io_periods = &throttle_io_period_msecs[0];
1481 }
1482
1483 #if (XNU_TARGET_OS_OSX && __arm64__)
1484 /*
1485 * IO Tier EDT overrides are meant for
1486 * some arm platforms but not for
1487 * macs.
1488 */
1489 #else /* (XNU_TARGET_OS_OSX && __arm64__) */
1490 /* Override global values with device-tree properties */
1491 if (PE_get_default("kern.io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1492 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1493 }
1494
1495 if (PE_get_default("kern.io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1496 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1497 }
1498
1499 if (PE_get_default("kern.io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1500 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1501 }
1502 #endif /* (XNU_TARGET_OS_OSX && __arm64__) */
1503
1504 /* Override with boot-args */
1505 if (PE_parse_boot_argn("io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1506 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1507 }
1508
1509 if (PE_parse_boot_argn("io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1510 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1511 }
1512
1513 if (PE_parse_boot_argn("io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1514 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1515 }
1516 }
1517
1518 #if CONFIG_IOSCHED
1519 int iosched_enabled = 1;
1520 #endif
1521
1522 void
throttle_init(void)1523 throttle_init(void)
1524 {
1525 struct _throttle_io_info_t *info;
1526 int i;
1527 int level;
1528 #if CONFIG_IOSCHED
1529 int iosched;
1530 #endif
1531
1532 /* Update throttle parameters based on device tree configuration */
1533 throttle_init_throttle_window();
1534
1535 for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) {
1536 info = &_throttle_io_info[i];
1537
1538 lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
1539 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1540
1541 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1542 TAILQ_INIT(&info->throttle_uthlist[level]);
1543 info->throttle_last_IO_pid[level] = 0;
1544 info->throttle_inflight_count[level] = 0;
1545 }
1546 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1547 info->throttle_disabled = 0;
1548 info->throttle_is_fusion_with_priority = 0;
1549 }
1550 #if CONFIG_IOSCHED
1551 if (PE_parse_boot_argn("iosched", &iosched, sizeof(iosched))) {
1552 iosched_enabled = iosched;
1553 }
1554 if (iosched_enabled) {
1555 /* Initialize I/O Reprioritization mechanism */
1556 vm_io_reprioritize_init();
1557 }
1558 #endif
1559 }
1560
1561 void
sys_override_io_throttle(boolean_t enable_override)1562 sys_override_io_throttle(boolean_t enable_override)
1563 {
1564 if (enable_override) {
1565 lowpri_throttle_enabled = 0;
1566 } else {
1567 lowpri_throttle_enabled = 1;
1568 }
1569 }
1570
1571 int rethrottle_wakeups = 0;
1572
1573 /*
1574 * the uu_rethrottle_lock is used to synchronize this function
1575 * with "throttle_lowpri_io" which is where a throttled thread
1576 * will block... that function will grab this lock before beginning
1577 * it's decision making process concerning the need to block, and
1578 * hold it through the assert_wait. When that thread is awakened
1579 * for any reason (timer or rethrottle), it will reacquire the
1580 * uu_rethrottle_lock before determining if it really is ok for
1581 * it to now run. This is the point at which the thread could
1582 * enter a different throttling queue and reblock or return from
1583 * the throttle w/o having waited out it's entire throttle if
1584 * the rethrottle has now moved it out of any currently
1585 * active throttle window.
1586 *
1587 *
1588 * NOTES:
1589 * 1 - This may be called with the task lock held.
1590 * 2 - This may be called with preemption and interrupts disabled
1591 * in the kqueue wakeup path so we can't take the throttle_lock which is a mutex
1592 * 3 - This cannot safely dereference uu_throttle_info, as it may
1593 * get deallocated out from under us
1594 */
1595
1596 void
rethrottle_thread(uthread_t ut)1597 rethrottle_thread(uthread_t ut)
1598 {
1599 /*
1600 * If uthread doesn't have throttle state, then there's no chance
1601 * of it needing a rethrottle.
1602 */
1603 if (ut->uu_throttle_info == NULL) {
1604 return;
1605 }
1606
1607 boolean_t s = ml_set_interrupts_enabled(FALSE);
1608 lck_spin_lock(&ut->uu_rethrottle_lock);
1609
1610 if (!ut->uu_is_throttled) {
1611 ut->uu_was_rethrottled = true;
1612 } else {
1613 int my_new_level = throttle_get_thread_throttle_level(ut);
1614
1615 if (my_new_level != ut->uu_on_throttlelist) {
1616 /*
1617 * ut is currently blocked (as indicated by
1618 * ut->uu_is_throttled == true)
1619 * and we're changing it's throttle level, so
1620 * we need to wake it up.
1621 */
1622 ut->uu_is_throttled = false;
1623 wakeup(&ut->uu_on_throttlelist);
1624
1625 rethrottle_wakeups++;
1626 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)),
1627 uthread_tid(ut), ut->uu_on_throttlelist, my_new_level, 0, 0);
1628 }
1629 }
1630 lck_spin_unlock(&ut->uu_rethrottle_lock);
1631 ml_set_interrupts_enabled(s);
1632 }
1633
1634
1635 /*
1636 * KPI routine
1637 *
1638 * Create and take a reference on a throttle info structure and return a
1639 * pointer for the file system to use when calling throttle_info_update.
1640 * Calling file system must have a matching release for every create.
1641 */
1642 void *
throttle_info_create(void)1643 throttle_info_create(void)
1644 {
1645 struct _throttle_io_info_t *info;
1646 int level;
1647
1648 info = kalloc_type(struct _throttle_io_info_t,
1649 Z_ZERO | Z_WAITOK | Z_NOFAIL);
1650 /* Mark that this one was allocated and needs to be freed */
1651 DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n", info, info );
1652 info->throttle_alloc = TRUE;
1653
1654 lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
1655 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1656
1657 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1658 TAILQ_INIT(&info->throttle_uthlist[level]);
1659 }
1660 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1661
1662 /* Take a reference */
1663 OSIncrementAtomic(&info->throttle_refcnt);
1664 return info;
1665 }
1666
1667 /*
1668 * KPI routine
1669 *
1670 * Release the throttle info pointer if all the reference are gone. Should be
1671 * called to release reference taken by throttle_info_create
1672 */
1673 void
throttle_info_release(void * throttle_info)1674 throttle_info_release(void *throttle_info)
1675 {
1676 DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n",
1677 (struct _throttle_io_info_t *)throttle_info,
1678 (struct _throttle_io_info_t *)throttle_info);
1679 if (throttle_info) { /* Just to be careful */
1680 throttle_info_rel(throttle_info);
1681 }
1682 }
1683
1684 /*
1685 * KPI routine
1686 *
1687 * File Systems that create an info structure, need to call this routine in
1688 * their mount routine (used by cluster code). File Systems that call this in
1689 * their mount routines must call throttle_info_mount_rel in their unmount
1690 * routines.
1691 */
1692 void
throttle_info_mount_ref(mount_t mp,void * throttle_info)1693 throttle_info_mount_ref(mount_t mp, void *throttle_info)
1694 {
1695 if ((throttle_info == NULL) || (mp == NULL)) {
1696 return;
1697 }
1698 throttle_info_ref(throttle_info);
1699
1700 /*
1701 * We already have a reference release it before adding the new one
1702 */
1703 if (mp->mnt_throttle_info) {
1704 throttle_info_rel(mp->mnt_throttle_info);
1705 }
1706 mp->mnt_throttle_info = throttle_info;
1707 }
1708
1709 /*
1710 * Private KPI routine
1711 *
1712 * return a handle for accessing throttle_info given a throttle_mask. The
1713 * handle must be released by throttle_info_rel_by_mask
1714 */
1715 int
throttle_info_ref_by_mask(uint64_t throttle_mask,throttle_info_handle_t * throttle_info_handle)1716 throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle)
1717 {
1718 int dev_index;
1719 struct _throttle_io_info_t *info;
1720
1721 /*
1722 * The 'throttle_mask' is not expected to be 0 otherwise num_trailing_0()
1723 * would return value of 64 and this will cause '_throttle_io_info' to
1724 * go out of bounds as '_throttle_io_info' is only LOWPRI_MAX_NUM_DEV (64)
1725 * elements long.
1726 */
1727 if (throttle_info_handle == NULL || throttle_mask == 0) {
1728 return EINVAL;
1729 }
1730
1731 dev_index = num_trailing_0(throttle_mask);
1732 info = &_throttle_io_info[dev_index];
1733 throttle_info_ref(info);
1734 *(struct _throttle_io_info_t**)throttle_info_handle = info;
1735
1736 return 0;
1737 }
1738
1739 /*
1740 * Private KPI routine
1741 *
1742 * release the handle obtained by throttle_info_ref_by_mask
1743 */
1744 void
throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)1745 throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)
1746 {
1747 /*
1748 * for now the handle is just a pointer to _throttle_io_info_t
1749 */
1750 throttle_info_rel((struct _throttle_io_info_t*)throttle_info_handle);
1751 }
1752
1753 /*
1754 * KPI routine
1755 *
1756 * File Systems that throttle_info_mount_ref, must call this routine in their
1757 * umount routine.
1758 */
1759 void
throttle_info_mount_rel(mount_t mp)1760 throttle_info_mount_rel(mount_t mp)
1761 {
1762 if (mp->mnt_throttle_info) {
1763 throttle_info_rel(mp->mnt_throttle_info);
1764 }
1765 mp->mnt_throttle_info = NULL;
1766 }
1767
1768 /*
1769 * Reset throttling periods for the given mount point
1770 *
1771 * private interface used by disk conditioner to reset
1772 * throttling periods when 'is_ssd' status changes
1773 */
1774 void
throttle_info_mount_reset_period(mount_t mp,int isssd)1775 throttle_info_mount_reset_period(mount_t mp, int isssd)
1776 {
1777 struct _throttle_io_info_t *info;
1778
1779 if (mp == NULL) {
1780 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1781 } else if (mp->mnt_throttle_info == NULL) {
1782 info = &_throttle_io_info[mp->mnt_devbsdunit];
1783 } else {
1784 info = mp->mnt_throttle_info;
1785 }
1786
1787 throttle_init_throttle_period(info, isssd);
1788 }
1789
1790 void
throttle_info_get_last_io_time(mount_t mp,struct timeval * tv)1791 throttle_info_get_last_io_time(mount_t mp, struct timeval *tv)
1792 {
1793 struct _throttle_io_info_t *info;
1794
1795 if (mp == NULL) {
1796 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1797 } else if (mp->mnt_throttle_info == NULL) {
1798 info = &_throttle_io_info[mp->mnt_devbsdunit];
1799 } else {
1800 info = mp->mnt_throttle_info;
1801 }
1802
1803 *tv = info->throttle_last_write_timestamp;
1804 }
1805
1806 void
update_last_io_time(mount_t mp)1807 update_last_io_time(mount_t mp)
1808 {
1809 struct _throttle_io_info_t *info;
1810
1811 if (mp == NULL) {
1812 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1813 } else if (mp->mnt_throttle_info == NULL) {
1814 info = &_throttle_io_info[mp->mnt_devbsdunit];
1815 } else {
1816 info = mp->mnt_throttle_info;
1817 }
1818
1819 microuptime(&info->throttle_last_write_timestamp);
1820 if (mp != NULL) {
1821 mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp;
1822 }
1823 }
1824
1825 int
throttle_get_io_policy(uthread_t * ut)1826 throttle_get_io_policy(uthread_t *ut)
1827 {
1828 if (ut != NULL) {
1829 *ut = current_uthread();
1830 }
1831
1832 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
1833 }
1834
1835 int
throttle_get_passive_io_policy(uthread_t * ut)1836 throttle_get_passive_io_policy(uthread_t *ut)
1837 {
1838 if (ut != NULL) {
1839 *ut = current_uthread();
1840 }
1841
1842 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_PASSIVE_IO);
1843 }
1844
1845
1846 static int
throttle_get_thread_throttle_level(uthread_t ut)1847 throttle_get_thread_throttle_level(uthread_t ut)
1848 {
1849 uthread_t *ut_p = (ut == NULL) ? &ut : NULL;
1850 int io_tier = throttle_get_io_policy(ut_p);
1851
1852 return throttle_get_thread_throttle_level_internal(ut, io_tier);
1853 }
1854
1855 /*
1856 * Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy)
1857 */
1858 static int
throttle_get_thread_throttle_level_internal(uthread_t ut,int io_tier)1859 throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier)
1860 {
1861 int thread_throttle_level = io_tier;
1862 int user_idle_level;
1863
1864 assert(ut != NULL);
1865
1866 /* Bootcache misses should always be throttled */
1867 if (ut->uu_throttle_bc) {
1868 thread_throttle_level = THROTTLE_LEVEL_TIER3;
1869 }
1870
1871 /*
1872 * Issue tier3 I/O as tier2 when the user is idle
1873 * to allow maintenance tasks to make more progress.
1874 *
1875 * Assume any positive idle level is enough... for now it's
1876 * only ever 0 or 128 but this is not defined anywhere.
1877 */
1878 if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) {
1879 user_idle_level = timer_get_user_idle_level();
1880 if (user_idle_level > 0) {
1881 thread_throttle_level--;
1882 }
1883 }
1884
1885 return thread_throttle_level;
1886 }
1887
1888 /*
1889 * I/O will be throttled if either of the following are true:
1890 * - Higher tiers have in-flight I/O
1891 * - The time delta since the last start/completion of a higher tier is within the throttle window interval
1892 *
1893 * In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal
1894 */
1895 static int
throttle_io_will_be_throttled_internal(void * throttle_info,int * mylevel,int * throttling_level)1896 throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level)
1897 {
1898 struct _throttle_io_info_t *info = throttle_info;
1899 struct timeval elapsed;
1900 struct timeval now;
1901 uint64_t elapsed_msecs;
1902 int thread_throttle_level;
1903 int throttle_level;
1904
1905 if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED) {
1906 return THROTTLE_DISENGAGED;
1907 }
1908
1909 microuptime(&now);
1910
1911 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
1912 if (info->throttle_inflight_count[throttle_level]) {
1913 break;
1914 }
1915 elapsed = now;
1916 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1917 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1918
1919 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
1920 break;
1921 }
1922 }
1923 if (throttle_level >= thread_throttle_level) {
1924 /*
1925 * we're beyond all of the throttle windows
1926 * that affect the throttle level of this thread,
1927 * so go ahead and treat as normal I/O
1928 */
1929 return THROTTLE_DISENGAGED;
1930 }
1931 if (mylevel) {
1932 *mylevel = thread_throttle_level;
1933 }
1934 if (throttling_level) {
1935 *throttling_level = throttle_level;
1936 }
1937
1938 if (info->throttle_io_count != info->throttle_io_count_begin) {
1939 /*
1940 * we've already issued at least one throttleable I/O
1941 * in the current I/O window, so avoid issuing another one
1942 */
1943 return THROTTLE_NOW;
1944 }
1945 /*
1946 * we're in the throttle window, so
1947 * cut the I/O size back
1948 */
1949 return THROTTLE_ENGAGED;
1950 }
1951
1952 /*
1953 * If we have a mount point and it has a throttle info pointer then
1954 * use it to do the check, otherwise use the device unit number to find
1955 * the correct throttle info array element.
1956 */
1957 int
throttle_io_will_be_throttled(__unused int lowpri_window_msecs,mount_t mp)1958 throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp)
1959 {
1960 struct _throttle_io_info_t *info;
1961
1962 /*
1963 * Should we just return zero if no mount point
1964 */
1965 if (mp == NULL) {
1966 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1967 } else if (mp->mnt_throttle_info == NULL) {
1968 info = &_throttle_io_info[mp->mnt_devbsdunit];
1969 } else {
1970 info = mp->mnt_throttle_info;
1971 }
1972
1973 if (info->throttle_is_fusion_with_priority) {
1974 uthread_t ut = current_uthread();
1975 if (ut->uu_lowpri_window == 0) {
1976 return THROTTLE_DISENGAGED;
1977 }
1978 }
1979
1980 if (info->throttle_disabled) {
1981 return THROTTLE_DISENGAGED;
1982 } else {
1983 return throttle_io_will_be_throttled_internal(info, NULL, NULL);
1984 }
1985 }
1986
1987 /*
1988 * Routine to increment I/O throttling counters maintained in the proc
1989 */
1990
1991 static void
throttle_update_proc_stats(pid_t throttling_pid,int count)1992 throttle_update_proc_stats(pid_t throttling_pid, int count)
1993 {
1994 proc_t throttling_proc;
1995 proc_t throttled_proc = current_proc();
1996
1997 /* The throttled_proc is always the current proc; so we are not concerned with refs */
1998 OSAddAtomic64(count, &(throttled_proc->was_throttled));
1999
2000 /* The throttling pid might have exited by now */
2001 throttling_proc = proc_find(throttling_pid);
2002 if (throttling_proc != PROC_NULL) {
2003 OSAddAtomic64(count, &(throttling_proc->did_throttle));
2004 proc_rele(throttling_proc);
2005 }
2006 }
2007
2008 /*
2009 * Block until woken up by the throttle timer or by a rethrottle call.
2010 * As long as we hold the throttle_lock while querying the throttle tier, we're
2011 * safe against seeing an old throttle tier after a rethrottle.
2012 */
2013 uint32_t
throttle_lowpri_io(int sleep_amount)2014 throttle_lowpri_io(int sleep_amount)
2015 {
2016 uthread_t ut;
2017 struct _throttle_io_info_t *info;
2018 int throttle_type = 0;
2019 int mylevel = 0;
2020 int throttling_level = THROTTLE_LEVEL_NONE;
2021 int sleep_cnt = 0;
2022 uint32_t throttle_io_period_num = 0;
2023 boolean_t insert_tail = TRUE;
2024 boolean_t s;
2025
2026 ut = current_uthread();
2027
2028 if (ut->uu_lowpri_window == 0) {
2029 return 0;
2030 }
2031 if (current_thread_in_kernel_fault()) {
2032 /* do not throttle kernel faults */
2033 return 0;
2034 }
2035
2036 info = ut->uu_throttle_info;
2037
2038 if (info == NULL) {
2039 ut->uu_throttle_bc = false;
2040 ut->uu_lowpri_window = 0;
2041 return 0;
2042 }
2043 lck_mtx_lock(&info->throttle_lock);
2044 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
2045
2046 if (sleep_amount == 0) {
2047 goto done;
2048 }
2049
2050 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
2051 sleep_amount = 0;
2052 }
2053
2054 throttle_io_period_num = info->throttle_io_period_num;
2055
2056 ut->uu_was_rethrottled = false;
2057
2058 while ((throttle_type = throttle_io_will_be_throttled_internal(info, &mylevel, &throttling_level))) {
2059 if (throttle_type == THROTTLE_ENGAGED) {
2060 if (sleep_amount == 0) {
2061 break;
2062 }
2063 if (info->throttle_io_period_num < throttle_io_period_num) {
2064 break;
2065 }
2066 if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
2067 break;
2068 }
2069 }
2070 /*
2071 * keep the same position in the list if "rethrottle_thread" changes our throttle level and
2072 * then puts us back to the original level before we get a chance to run
2073 */
2074 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) {
2075 /*
2076 * must have been awakened via "rethrottle_thread" (the timer pulls us off the list)
2077 * and we've changed our throttling level, so pull ourselves off of the appropriate list
2078 * and make sure we get put on the tail of the new list since we're starting anew w/r to
2079 * the throttling engine
2080 */
2081 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
2082 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
2083 insert_tail = TRUE;
2084 }
2085 if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) {
2086 if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END) {
2087 goto done;
2088 }
2089 }
2090 assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END);
2091
2092 s = ml_set_interrupts_enabled(FALSE);
2093 lck_spin_lock(&ut->uu_rethrottle_lock);
2094
2095 /*
2096 * this is the critical section w/r to our interaction
2097 * with "rethrottle_thread"
2098 */
2099 if (ut->uu_was_rethrottled) {
2100 lck_spin_unlock(&ut->uu_rethrottle_lock);
2101 ml_set_interrupts_enabled(s);
2102 lck_mtx_yield(&info->throttle_lock);
2103
2104 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)),
2105 uthread_tid(ut), ut->uu_on_throttlelist, 0, 0, 0);
2106
2107 ut->uu_was_rethrottled = false;
2108 continue;
2109 }
2110 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE,
2111 info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0);
2112
2113 if (sleep_cnt == 0) {
2114 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
2115 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2116 throttled_count[mylevel]++;
2117 }
2118 ut->uu_wmesg = "throttle_lowpri_io";
2119
2120 assert_wait((caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT);
2121
2122 ut->uu_is_throttled = true;
2123 lck_spin_unlock(&ut->uu_rethrottle_lock);
2124 ml_set_interrupts_enabled(s);
2125
2126 lck_mtx_unlock(&info->throttle_lock);
2127
2128 thread_block(THREAD_CONTINUE_NULL);
2129
2130 ut->uu_wmesg = NULL;
2131
2132 ut->uu_is_throttled = false;
2133 ut->uu_was_rethrottled = false;
2134
2135 lck_mtx_lock(&info->throttle_lock);
2136
2137 sleep_cnt++;
2138
2139 if (sleep_amount == 0) {
2140 insert_tail = FALSE;
2141 } else if (info->throttle_io_period_num < throttle_io_period_num ||
2142 (info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
2143 insert_tail = FALSE;
2144 sleep_amount = 0;
2145 }
2146 }
2147 done:
2148 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
2149 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
2150 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
2151 }
2152 lck_mtx_unlock(&info->throttle_lock);
2153
2154 if (sleep_cnt) {
2155 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
2156 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2157 /*
2158 * We update the stats for the last pid which opened a throttle window for the throttled thread.
2159 * This might not be completely accurate since the multiple throttles seen by the lower tier pid
2160 * might have been caused by various higher prio pids. However, updating these stats accurately
2161 * means doing a proc_find while holding the throttle lock which leads to deadlock.
2162 */
2163 throttle_update_proc_stats(info->throttle_last_IO_pid[throttling_level], sleep_cnt);
2164 }
2165
2166 ut->uu_throttle_info = NULL;
2167 ut->uu_throttle_bc = false;
2168 ut->uu_lowpri_window = 0;
2169
2170 throttle_info_rel(info);
2171
2172 return sleep_cnt;
2173 }
2174
2175 /*
2176 * returns TRUE if the throttle_lowpri_io called with the same sleep_amount would've slept
2177 * This function mimics the most of the throttle_lowpri_io checks but without actual sleeping
2178 */
2179 int
throttle_lowpri_io_will_be_throttled(int sleep_amount)2180 throttle_lowpri_io_will_be_throttled(int sleep_amount)
2181 {
2182 if (sleep_amount == 0) {
2183 return FALSE;
2184 }
2185
2186 uthread_t ut = current_uthread();
2187 if (ut->uu_lowpri_window == 0) {
2188 return FALSE;
2189 }
2190
2191 struct _throttle_io_info_t *info = ut->uu_throttle_info;
2192 if (info == NULL) {
2193 return FALSE;
2194 }
2195
2196 lck_mtx_lock(&info->throttle_lock);
2197 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
2198
2199 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
2200 sleep_amount = 0;
2201 }
2202
2203 int result = FALSE;
2204
2205 int throttle_type = throttle_io_will_be_throttled_internal(info, NULL, NULL);
2206 if (throttle_type > THROTTLE_DISENGAGED) {
2207 result = TRUE;
2208 if ((throttle_type == THROTTLE_ENGAGED) && (sleep_amount == 0)) {
2209 result = FALSE;
2210 }
2211 }
2212
2213 lck_mtx_unlock(&info->throttle_lock);
2214
2215 return result;
2216 }
2217
2218
2219 /*
2220 * KPI routine
2221 *
2222 * set a kernel thread's IO policy. policy can be:
2223 * IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD
2224 *
2225 * explanations about these policies are in the man page of setiopolicy_np
2226 */
2227 void
throttle_set_thread_io_policy(int policy)2228 throttle_set_thread_io_policy(int policy)
2229 {
2230 proc_set_thread_policy(current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, policy);
2231 }
2232
2233 int
throttle_get_thread_effective_io_policy()2234 throttle_get_thread_effective_io_policy()
2235 {
2236 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
2237 }
2238
2239 int
throttle_thread_io_tier_above_metadata(void)2240 throttle_thread_io_tier_above_metadata(void)
2241 {
2242 return throttle_get_thread_effective_io_policy() < IOSCHED_METADATA_TIER;
2243 }
2244
2245 void
throttle_info_reset_window(uthread_t ut)2246 throttle_info_reset_window(uthread_t ut)
2247 {
2248 struct _throttle_io_info_t *info;
2249
2250 if (ut == NULL) {
2251 ut = current_uthread();
2252 }
2253
2254 if ((info = ut->uu_throttle_info)) {
2255 throttle_info_rel(info);
2256
2257 ut->uu_throttle_info = NULL;
2258 ut->uu_lowpri_window = 0;
2259 ut->uu_throttle_bc = false;
2260 }
2261 }
2262
2263 static
2264 void
throttle_info_set_initial_window(uthread_t ut,struct _throttle_io_info_t * info,boolean_t BC_throttle,boolean_t isssd)2265 throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd)
2266 {
2267 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2268 return;
2269 }
2270
2271 if (info->throttle_io_periods == 0) {
2272 throttle_init_throttle_period(info, isssd);
2273 }
2274 if (ut->uu_throttle_info == NULL) {
2275 ut->uu_throttle_info = info;
2276 throttle_info_ref(info);
2277 DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n", info, info );
2278
2279 ut->uu_lowpri_window = 1;
2280 ut->uu_throttle_bc = BC_throttle;
2281 }
2282 }
2283
2284 /*
2285 * Update inflight IO count and throttling window
2286 * Should be called when an IO is done
2287 *
2288 * Only affects IO that was sent through spec_strategy
2289 */
2290 void
throttle_info_end_io(buf_t bp)2291 throttle_info_end_io(buf_t bp)
2292 {
2293 vnode_t vp;
2294 mount_t mp;
2295 struct bufattr *bap;
2296 struct _throttle_io_info_t *info;
2297 int io_tier;
2298
2299 bap = &bp->b_attr;
2300 if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) {
2301 return;
2302 }
2303 CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2304
2305 vp = buf_vnode(bp);
2306 mp = vp->v_mount;
2307
2308 if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
2309 info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
2310 } else if (mp != NULL) {
2311 info = &_throttle_io_info[mp->mnt_devbsdunit];
2312 } else {
2313 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2314 }
2315
2316 io_tier = GET_BUFATTR_IO_TIER(bap);
2317 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2318 io_tier--;
2319 }
2320
2321 throttle_info_end_io_internal(info, io_tier);
2322 }
2323
2324 /*
2325 * Decrement inflight count initially incremented by throttle_info_update_internal
2326 */
2327 static
2328 void
throttle_info_end_io_internal(struct _throttle_io_info_t * info,int throttle_level)2329 throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level)
2330 {
2331 if (throttle_level == THROTTLE_LEVEL_NONE) {
2332 return;
2333 }
2334
2335 microuptime(&info->throttle_window_start_timestamp[throttle_level]);
2336 OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]);
2337 assert(info->throttle_inflight_count[throttle_level] >= 0);
2338 }
2339
2340 /*
2341 * If inflight is TRUE and bap is NULL then the caller is responsible for calling
2342 * throttle_info_end_io_internal to avoid leaking in-flight I/O.
2343 */
2344 static
2345 int
throttle_info_update_internal(struct _throttle_io_info_t * info,uthread_t ut,int flags,boolean_t isssd,boolean_t inflight,struct bufattr * bap)2346 throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap)
2347 {
2348 int thread_throttle_level;
2349
2350 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2351 return THROTTLE_LEVEL_NONE;
2352 }
2353
2354 if (ut == NULL) {
2355 ut = current_uthread();
2356 }
2357
2358 if (bap && inflight && !ut->uu_throttle_bc) {
2359 thread_throttle_level = GET_BUFATTR_IO_TIER(bap);
2360 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2361 thread_throttle_level--;
2362 }
2363 } else {
2364 thread_throttle_level = throttle_get_thread_throttle_level(ut);
2365 }
2366
2367 if (thread_throttle_level != THROTTLE_LEVEL_NONE) {
2368 if (!ISSET(flags, B_PASSIVE)) {
2369 info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid();
2370 if (inflight && !ut->uu_throttle_bc) {
2371 if (NULL != bap) {
2372 SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2373 }
2374 OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]);
2375 } else {
2376 microuptime(&info->throttle_window_start_timestamp[thread_throttle_level]);
2377 }
2378 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE,
2379 proc_getpid(current_proc()), thread_throttle_level, 0, 0, 0);
2380 }
2381 microuptime(&info->throttle_last_IO_timestamp[thread_throttle_level]);
2382 }
2383
2384
2385 if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) {
2386 /*
2387 * I'd really like to do the IOSleep here, but
2388 * we may be holding all kinds of filesystem related locks
2389 * and the pages for this I/O marked 'busy'...
2390 * we don't want to cause a normal task to block on
2391 * one of these locks while we're throttling a task marked
2392 * for low priority I/O... we'll mark the uthread and
2393 * do the delay just before we return from the system
2394 * call that triggered this I/O or from vnode_pagein
2395 */
2396 OSAddAtomic(1, &info->throttle_io_count);
2397
2398 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2399 }
2400
2401 return thread_throttle_level;
2402 }
2403
2404 void *
throttle_info_update_by_mount(mount_t mp)2405 throttle_info_update_by_mount(mount_t mp)
2406 {
2407 struct _throttle_io_info_t *info;
2408 uthread_t ut;
2409 boolean_t isssd = FALSE;
2410
2411 ut = current_uthread();
2412
2413 if (mp != NULL) {
2414 if (disk_conditioner_mount_is_ssd(mp)) {
2415 isssd = TRUE;
2416 }
2417 info = &_throttle_io_info[mp->mnt_devbsdunit];
2418 } else {
2419 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2420 }
2421
2422 if (!ut->uu_lowpri_window) {
2423 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2424 }
2425
2426 return info;
2427 }
2428
2429
2430 /*
2431 * KPI routine
2432 *
2433 * this is usually called before every I/O, used for throttled I/O
2434 * book keeping. This routine has low overhead and does not sleep
2435 */
2436 void
throttle_info_update(void * throttle_info,int flags)2437 throttle_info_update(void *throttle_info, int flags)
2438 {
2439 if (throttle_info) {
2440 throttle_info_update_internal(throttle_info, NULL, flags, FALSE, FALSE, NULL);
2441 }
2442 }
2443
2444 /*
2445 * KPI routine (private)
2446 *
2447 * similar to throttle_info_update() but takes an additional argument to
2448 * indicate if the backing device type is SSD or not.
2449 */
2450 void
throttle_info_update_with_type(void * throttle_info,int flags,boolean_t isssd)2451 throttle_info_update_with_type(void *throttle_info, int flags, boolean_t isssd)
2452 {
2453 if (throttle_info) {
2454 throttle_info_update_internal(throttle_info, NULL, flags, isssd, FALSE, NULL);
2455 }
2456 }
2457
2458 /*
2459 * KPI routine
2460 *
2461 * this is usually called before every I/O, used for throttled I/O
2462 * book keeping. This routine has low overhead and does not sleep
2463 */
2464 void
throttle_info_update_by_mask(void * throttle_info_handle,int flags)2465 throttle_info_update_by_mask(void *throttle_info_handle, int flags)
2466 {
2467 void *throttle_info = throttle_info_handle;
2468
2469 /*
2470 * for now we only use the lowest bit of the throttle mask, so the
2471 * handle is the same as the throttle_info. Later if we store a
2472 * set of throttle infos in the handle, we will want to loop through
2473 * them and call throttle_info_update in a loop
2474 */
2475 throttle_info_update(throttle_info, flags);
2476 }
2477 /*
2478 * KPI routine
2479 *
2480 * This routine marks the throttle info as disabled. Used for mount points which
2481 * support I/O scheduling.
2482 */
2483
2484 void
throttle_info_disable_throttle(int devno,boolean_t isfusion)2485 throttle_info_disable_throttle(int devno, boolean_t isfusion)
2486 {
2487 struct _throttle_io_info_t *info;
2488
2489 if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV) {
2490 panic("Illegal devno (%d) passed into throttle_info_disable_throttle()", devno);
2491 }
2492
2493 info = &_throttle_io_info[devno];
2494 // don't disable software throttling on devices that are part of a fusion device
2495 // and override the software throttle periods to use HDD periods
2496 if (isfusion) {
2497 info->throttle_is_fusion_with_priority = isfusion;
2498 throttle_init_throttle_period(info, FALSE);
2499 }
2500 info->throttle_disabled = !info->throttle_is_fusion_with_priority;
2501 return;
2502 }
2503
2504
2505 /*
2506 * KPI routine (private)
2507 * Called to determine if this IO is being throttled to this level so that it can be treated specially
2508 */
2509 int
throttle_info_io_will_be_throttled(void * throttle_info,int policy)2510 throttle_info_io_will_be_throttled(void * throttle_info, int policy)
2511 {
2512 struct _throttle_io_info_t *info = throttle_info;
2513 struct timeval elapsed;
2514 uint64_t elapsed_msecs;
2515 int throttle_level;
2516 int thread_throttle_level;
2517
2518 switch (policy) {
2519 case IOPOL_THROTTLE:
2520 thread_throttle_level = THROTTLE_LEVEL_TIER3;
2521 break;
2522 case IOPOL_UTILITY:
2523 thread_throttle_level = THROTTLE_LEVEL_TIER2;
2524 break;
2525 case IOPOL_STANDARD:
2526 thread_throttle_level = THROTTLE_LEVEL_TIER1;
2527 break;
2528 default:
2529 thread_throttle_level = THROTTLE_LEVEL_TIER0;
2530 break;
2531 }
2532 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
2533 if (info->throttle_inflight_count[throttle_level]) {
2534 break;
2535 }
2536
2537 microuptime(&elapsed);
2538 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
2539 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
2540
2541 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
2542 break;
2543 }
2544 }
2545 if (throttle_level >= thread_throttle_level) {
2546 /*
2547 * we're beyond all of the throttle windows
2548 * so go ahead and treat as normal I/O
2549 */
2550 return THROTTLE_DISENGAGED;
2551 }
2552 /*
2553 * we're in the throttle window
2554 */
2555 return THROTTLE_ENGAGED;
2556 }
2557
2558 int
throttle_lowpri_window(void)2559 throttle_lowpri_window(void)
2560 {
2561 return current_uthread()->uu_lowpri_window;
2562 }
2563
2564 #if CONFIG_PHYS_WRITE_ACCT
2565 extern thread_t pm_sync_thread;
2566 #endif /* CONFIG_PHYS_WRITE_ACCT */
2567
2568 int
spec_strategy(struct vnop_strategy_args * ap)2569 spec_strategy(struct vnop_strategy_args *ap)
2570 {
2571 buf_t bp;
2572 int bflags;
2573 int io_tier;
2574 int passive;
2575 dev_t bdev;
2576 uthread_t ut;
2577 vnode_t vp;
2578 mount_t mp;
2579 struct bufattr *bap;
2580 int strategy_ret;
2581 struct _throttle_io_info_t *throttle_info;
2582 boolean_t isssd = FALSE;
2583 boolean_t inflight = FALSE;
2584 boolean_t upgrade = FALSE;
2585 int code = 0;
2586
2587 #if CONFIG_DELAY_IDLE_SLEEP
2588 proc_t curproc = current_proc();
2589 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2590
2591 bp = ap->a_bp;
2592 bdev = buf_device(bp);
2593 vp = buf_vnode(bp);
2594 mp = vp ? vp->v_mount : NULL;
2595 bap = &bp->b_attr;
2596
2597 #if CONFIG_PHYS_WRITE_ACCT
2598 if (current_thread() == pm_sync_thread) {
2599 OSAddAtomic64(buf_count(bp), (SInt64 *)&(kernel_pm_writes));
2600 }
2601 #endif /* CONFIG_PHYS_WRITE_ACCT */
2602
2603 #if CONFIG_IOSCHED
2604 if (bp->b_flags & B_CLUSTER) {
2605 io_tier = upl_get_cached_tier(bp->b_upl);
2606
2607 if (io_tier == -1) {
2608 io_tier = throttle_get_io_policy(&ut);
2609 }
2610 #if DEVELOPMENT || DEBUG
2611 else {
2612 int my_io_tier = throttle_get_io_policy(&ut);
2613
2614 if (io_tier != my_io_tier) {
2615 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0);
2616 }
2617 }
2618 #endif
2619 } else {
2620 io_tier = throttle_get_io_policy(&ut);
2621 }
2622 #else
2623 io_tier = throttle_get_io_policy(&ut);
2624 #endif
2625 passive = throttle_get_passive_io_policy(&ut);
2626
2627 /*
2628 * Mark if the I/O was upgraded by throttle_get_thread_throttle_level
2629 * while preserving the original issued tier (throttle_get_io_policy
2630 * does not return upgraded tiers)
2631 */
2632 if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) {
2633 #if CONFIG_IOSCHED
2634 if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
2635 upgrade = TRUE;
2636 }
2637 #else /* CONFIG_IOSCHED */
2638 upgrade = TRUE;
2639 #endif /* CONFIG_IOSCHED */
2640 }
2641
2642 if (bp->b_flags & B_META) {
2643 bap->ba_flags |= BA_META;
2644 }
2645
2646 #if CONFIG_IOSCHED
2647 /*
2648 * For metadata reads, ceil the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited, otherwise
2649 * ceil it to IOSCHED_METADATA_TIER. Mark them passive if the I/O tier was upgraded.
2650 * For metadata writes, set the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited. Otherwise
2651 * set it to IOSCHED_METADATA_TIER. In addition, mark them as passive.
2652 */
2653 if (bap->ba_flags & BA_META) {
2654 if ((mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) || (bap->ba_flags & BA_IO_SCHEDULED)) {
2655 if (bp->b_flags & B_READ) {
2656 if ((bap->ba_flags & BA_EXPEDITED_META_IO) && (io_tier > IOSCHED_METADATA_EXPEDITED_TIER)) {
2657 io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
2658 passive = 1;
2659 } else if (io_tier > IOSCHED_METADATA_TIER) {
2660 io_tier = IOSCHED_METADATA_TIER;
2661 passive = 1;
2662 }
2663 } else {
2664 if (bap->ba_flags & BA_EXPEDITED_META_IO) {
2665 io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
2666 } else {
2667 io_tier = IOSCHED_METADATA_TIER;
2668 }
2669 passive = 1;
2670 }
2671 }
2672 }
2673 #endif /* CONFIG_IOSCHED */
2674
2675 SET_BUFATTR_IO_TIER(bap, io_tier);
2676
2677 if (passive) {
2678 bp->b_flags |= B_PASSIVE;
2679 bap->ba_flags |= BA_PASSIVE;
2680 }
2681
2682 #if CONFIG_DELAY_IDLE_SLEEP
2683 if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) {
2684 bap->ba_flags |= BA_DELAYIDLESLEEP;
2685 }
2686 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2687
2688 bflags = bp->b_flags;
2689
2690 if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0)) {
2691 bufattr_markquickcomplete(bap);
2692 }
2693
2694 if (bflags & B_READ) {
2695 code |= DKIO_READ;
2696 }
2697 if (bflags & B_ASYNC) {
2698 code |= DKIO_ASYNC;
2699 }
2700
2701 if (bap->ba_flags & BA_META) {
2702 code |= DKIO_META;
2703 } else if (bflags & B_PAGEIO) {
2704 code |= DKIO_PAGING;
2705 }
2706
2707 if (io_tier != 0) {
2708 code |= DKIO_THROTTLE;
2709 }
2710
2711 code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK);
2712
2713 if (bflags & B_PASSIVE) {
2714 code |= DKIO_PASSIVE;
2715 }
2716
2717 if (bap->ba_flags & BA_NOCACHE) {
2718 code |= DKIO_NOCACHE;
2719 }
2720
2721 if (upgrade) {
2722 code |= DKIO_TIER_UPGRADE;
2723 SET(bap->ba_flags, BA_IO_TIER_UPGRADE);
2724 }
2725
2726 if (kdebug_enable) {
2727 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
2728 buf_kernel_addrperm_addr(bp), bdev, buf_blkno(bp), buf_count(bp), 0);
2729 }
2730
2731 #if CONFIG_IO_COMPRESSION_STATS
2732 // Do not run IO Compression Stats when a privilege thread is active
2733 if (!is_vm_privileged() && !is_external_pageout_thread()) {
2734 io_compression_stats(bp);
2735 }
2736 #endif /* CONFIG_IO_COMPRESSION_STATS */
2737 thread_update_io_stats(current_thread(), buf_count(bp), code);
2738
2739 if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
2740 if (!vp->v_un.vu_specinfo->si_initted) {
2741 SPEC_INIT_BSDUNIT(vp, vfs_context_current());
2742 }
2743 if (vp->v_un.vu_specinfo->si_devbsdunit > (LOWPRI_MAX_NUM_DEV - 1)) {
2744 panic("Invalid value (%d) for si_devbsdunit for vnode %p",
2745 vp->v_un.vu_specinfo->si_devbsdunit, vp);
2746 }
2747 if (vp->v_un.vu_specinfo->si_isssd > 1) {
2748 panic("Invalid value (%d) for si_isssd for vnode %p",
2749 vp->v_un.vu_specinfo->si_isssd, vp);
2750 }
2751 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
2752 isssd = vp->v_un.vu_specinfo->si_isssd;
2753 } else if (mp != NULL) {
2754 if (disk_conditioner_mount_is_ssd(mp)) {
2755 isssd = TRUE;
2756 }
2757 /*
2758 * Partially initialized mounts don't have a final devbsdunit and should not be tracked.
2759 * Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value
2760 * (mnt_throttle_mask is initialized and num_trailing_0 would be 0)
2761 */
2762 if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) {
2763 inflight = TRUE;
2764 }
2765 throttle_info = &_throttle_io_info[mp->mnt_devbsdunit];
2766 } else {
2767 throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2768 }
2769
2770 throttle_info_update_internal(throttle_info, ut, bflags, isssd, inflight, bap);
2771
2772 if ((bflags & B_READ) == 0) {
2773 microuptime(&throttle_info->throttle_last_write_timestamp);
2774
2775 if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
2776 mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp;
2777 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size);
2778 }
2779 } else if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
2780 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size);
2781 }
2782 /*
2783 * The BootCache may give us special information about
2784 * the IO, so it returns special values that we check
2785 * for here.
2786 *
2787 * IO_SATISFIED_BY_CACHE
2788 * The read has been satisfied by the boot cache. Don't
2789 * throttle the thread unnecessarily.
2790 *
2791 * IO_SHOULD_BE_THROTTLED
2792 * The boot cache is playing back a playlist and this IO
2793 * cut through. Throttle it so we're not cutting through
2794 * the boot cache too often.
2795 *
2796 * Note that typical strategy routines are defined with
2797 * a void return so we'll get garbage here. In the
2798 * unlikely case the garbage matches our special return
2799 * value, it's not a big deal since we're only adjusting
2800 * the throttling delay.
2801 */
2802 #define IO_SATISFIED_BY_CACHE ((int)0xcafefeed)
2803 #define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef)
2804 #pragma clang diagnostic push
2805 #pragma clang diagnostic ignored "-Wcast-function-type"
2806
2807 typedef int strategy_fcn_ret_t(struct buf *bp);
2808
2809 strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp);
2810
2811 #pragma clang diagnostic pop
2812
2813 // disk conditioner needs to track when this I/O actually starts
2814 // which means track it after `strategy` which may include delays
2815 // from inflight I/Os
2816 microuptime(&bp->b_timestamp_tv);
2817
2818 if (IO_SATISFIED_BY_CACHE == strategy_ret) {
2819 /*
2820 * If this was a throttled IO satisfied by the boot cache,
2821 * don't delay the thread.
2822 */
2823 throttle_info_reset_window(ut);
2824 } else if (IO_SHOULD_BE_THROTTLED == strategy_ret) {
2825 /*
2826 * If the boot cache indicates this IO should be throttled,
2827 * delay the thread.
2828 */
2829 throttle_info_set_initial_window(ut, throttle_info, TRUE, isssd);
2830 }
2831 return 0;
2832 }
2833
2834
2835 /*
2836 * This is a noop, simply returning what one has been given.
2837 */
2838 int
spec_blockmap(__unused struct vnop_blockmap_args * ap)2839 spec_blockmap(__unused struct vnop_blockmap_args *ap)
2840 {
2841 return ENOTSUP;
2842 }
2843
2844 static int
spec_close_internal(struct vnode * vp,dev_t dev,int flags,vfs_context_t ctx)2845 spec_close_internal(struct vnode *vp, dev_t dev, int flags, vfs_context_t ctx)
2846 {
2847 int error = 0;
2848 struct proc *p = vfs_context_proc(ctx);
2849 struct session *sessp;
2850 struct pgrp *pg;
2851
2852 switch (vp->v_type) {
2853 case VCHR:
2854 /*
2855 * Hack: a tty device that is a controlling terminal
2856 * has a reference from the session structure.
2857 * We cannot easily tell that a character device is
2858 * a controlling terminal, unless it is the closing
2859 * process' controlling terminal. In that case,
2860 * if the reference count is 1 (this is the very
2861 * last close)
2862 */
2863 pg = proc_pgrp(p, &sessp);
2864 devsw_lock(dev, S_IFCHR);
2865 if (sessp != SESSION_NULL) {
2866 if (vp == sessp->s_ttyvp && vcount(vp) == 1) {
2867 struct tty *tp = TTY_NULL;
2868
2869 devsw_unlock(dev, S_IFCHR);
2870 session_lock(sessp);
2871 if (vp == sessp->s_ttyvp) {
2872 tp = session_clear_tty_locked(sessp);
2873 }
2874 session_unlock(sessp);
2875
2876 if (tp != TTY_NULL) {
2877 ttyfree(tp);
2878 }
2879 devsw_lock(dev, S_IFCHR);
2880 }
2881 }
2882 pgrp_rele(pg);
2883
2884 if (--vp->v_specinfo->si_opencount < 0) {
2885 panic("negative open count (c, %u, %u)", major(dev), minor(dev));
2886 }
2887
2888 /*
2889 * close on last reference or on vnode revoke call
2890 */
2891 if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0) {
2892 bool revoke_done_needed = false;
2893
2894 if (vnode_istty(vp) && vnode_isrecycled(vp)) {
2895 vnode_spec_revoke_wait(vp);
2896 revoke_done_needed = true;
2897 }
2898
2899 error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p);
2900
2901 if (revoke_done_needed) {
2902 vnode_spec_revoke_done(vp);
2903 }
2904 }
2905
2906 devsw_unlock(dev, S_IFCHR);
2907 break;
2908
2909 case VBLK:
2910 /*
2911 * If there is more than one outstanding open, don't
2912 * send the close to the device.
2913 */
2914 devsw_lock(dev, S_IFBLK);
2915 if (vcount(vp) > 1) {
2916 vp->v_specinfo->si_opencount--;
2917 devsw_unlock(dev, S_IFBLK);
2918 return 0;
2919 }
2920 devsw_unlock(dev, S_IFBLK);
2921
2922 /*
2923 * On last close of a block device (that isn't mounted)
2924 * we must invalidate any in core blocks, so that
2925 * we can, for instance, change floppy disks.
2926 */
2927 if ((error = spec_fsync_internal(vp, MNT_WAIT, ctx))) {
2928 return error;
2929 }
2930
2931 error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
2932 if (error) {
2933 return error;
2934 }
2935
2936 devsw_lock(dev, S_IFBLK);
2937
2938 if (--vp->v_specinfo->si_opencount < 0) {
2939 panic("negative open count (b, %u, %u)", major(dev), minor(dev));
2940 }
2941
2942 if (vcount(vp) == 0) {
2943 error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p);
2944 }
2945
2946 devsw_unlock(dev, S_IFBLK);
2947 break;
2948
2949 default:
2950 panic("spec_close: not special");
2951 return EBADF;
2952 }
2953
2954 return error;
2955 }
2956
2957 /*
2958 * Device close routine
2959 */
2960 int
spec_close(struct vnop_close_args * ap)2961 spec_close(struct vnop_close_args *ap)
2962 {
2963 return spec_close_internal(ap->a_vp, ap->a_vp->v_rdev, ap->a_fflag, ap->a_context);
2964 }
2965
2966 /*
2967 * Return POSIX pathconf information applicable to special devices.
2968 */
2969 int
spec_pathconf(struct vnop_pathconf_args * ap)2970 spec_pathconf(struct vnop_pathconf_args *ap)
2971 {
2972 switch (ap->a_name) {
2973 case _PC_LINK_MAX:
2974 *ap->a_retval = LINK_MAX;
2975 return 0;
2976 case _PC_MAX_CANON:
2977 *ap->a_retval = MAX_CANON;
2978 return 0;
2979 case _PC_MAX_INPUT:
2980 *ap->a_retval = MAX_INPUT;
2981 return 0;
2982 case _PC_PIPE_BUF:
2983 *ap->a_retval = PIPE_BUF;
2984 return 0;
2985 case _PC_CHOWN_RESTRICTED:
2986 *ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */
2987 return 0;
2988 case _PC_VDISABLE:
2989 *ap->a_retval = _POSIX_VDISABLE;
2990 return 0;
2991 default:
2992 return EINVAL;
2993 }
2994 /* NOTREACHED */
2995 }
2996
2997 /*
2998 * Special device failed operation
2999 */
3000 int
spec_ebadf(__unused void * dummy)3001 spec_ebadf(__unused void *dummy)
3002 {
3003 return EBADF;
3004 }
3005
3006 /* Blktooff derives file offset from logical block number */
3007 int
spec_blktooff(struct vnop_blktooff_args * ap)3008 spec_blktooff(struct vnop_blktooff_args *ap)
3009 {
3010 struct vnode *vp = ap->a_vp;
3011
3012 switch (vp->v_type) {
3013 case VCHR:
3014 *ap->a_offset = (off_t)-1; /* failure */
3015 return ENOTSUP;
3016
3017 case VBLK:
3018 printf("spec_blktooff: not implemented for VBLK\n");
3019 *ap->a_offset = (off_t)-1; /* failure */
3020 return ENOTSUP;
3021
3022 default:
3023 panic("spec_blktooff type");
3024 }
3025 /* NOTREACHED */
3026
3027 return 0;
3028 }
3029
3030 /* Offtoblk derives logical block number from file offset */
3031 int
spec_offtoblk(struct vnop_offtoblk_args * ap)3032 spec_offtoblk(struct vnop_offtoblk_args *ap)
3033 {
3034 struct vnode *vp = ap->a_vp;
3035
3036 switch (vp->v_type) {
3037 case VCHR:
3038 *ap->a_lblkno = (daddr64_t)-1; /* failure */
3039 return ENOTSUP;
3040
3041 case VBLK:
3042 printf("spec_offtoblk: not implemented for VBLK\n");
3043 *ap->a_lblkno = (daddr64_t)-1; /* failure */
3044 return ENOTSUP;
3045
3046 default:
3047 panic("spec_offtoblk type");
3048 }
3049 /* NOTREACHED */
3050
3051 return 0;
3052 }
3053
3054 static int filt_specattach(struct knote *kn, struct kevent_qos_s *kev);
3055 static void filt_specdetach(struct knote *kn);
3056 static int filt_specevent(struct knote *kn, long hint);
3057 static int filt_spectouch(struct knote *kn, struct kevent_qos_s *kev);
3058 static int filt_specprocess(struct knote *kn, struct kevent_qos_s *kev);
3059
3060 SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = {
3061 .f_isfd = 1,
3062 .f_attach = filt_specattach,
3063 .f_detach = filt_specdetach,
3064 .f_event = filt_specevent,
3065 .f_touch = filt_spectouch,
3066 .f_process = filt_specprocess,
3067 };
3068
3069 static void
filt_spec_make_eof(struct knote * kn)3070 filt_spec_make_eof(struct knote *kn)
3071 {
3072 /*
3073 * The spec filter might touch kn_flags from f_event
3074 * without holding "the primitive lock", so make it atomic.
3075 */
3076 os_atomic_or(&kn->kn_flags, EV_EOF | EV_ONESHOT, relaxed);
3077 }
3078
3079 static int
filt_spec_common(struct knote * kn,struct kevent_qos_s * kev,bool attach)3080 filt_spec_common(struct knote *kn, struct kevent_qos_s *kev, bool attach)
3081 {
3082 uthread_t uth = current_uthread();
3083 vfs_context_t ctx = vfs_context_current();
3084 vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp);
3085 __block bool selrecorded = false;
3086 struct select_set *old_wqs;
3087 int64_t data = 0;
3088 int ret, selret;
3089
3090 if (kn->kn_flags & EV_EOF) {
3091 ret = FILTER_ACTIVE;
3092 goto out;
3093 }
3094
3095 if (!attach && vnode_getwithvid(vp, vnode_vid(vp)) != 0) {
3096 filt_spec_make_eof(kn);
3097 ret = FILTER_ACTIVE;
3098 goto out;
3099 }
3100
3101 selspec_record_hook_t cb = ^(struct selinfo *si) {
3102 selspec_attach(kn, si);
3103 selrecorded = true;
3104 };
3105
3106 old_wqs = uth->uu_selset;
3107 uth->uu_selset = SELSPEC_RECORD_MARKER;
3108 selret = VNOP_SELECT(vp, knote_get_seltype(kn), 0, cb, ctx);
3109 uth->uu_selset = old_wqs;
3110
3111 if (!attach) {
3112 vnode_put(vp);
3113 }
3114
3115 if (!selrecorded && selret == 0) {
3116 /*
3117 * The device indicated that there's no data to read,
3118 * but didn't call `selrecord`.
3119 *
3120 * Nothing will be notified of changes to this vnode,
3121 * so return an error back to user space on attach,
3122 * or pretend the knote disappeared for other cases,
3123 * to make it clear that the knote is not attached.
3124 */
3125 if (attach) {
3126 knote_set_error(kn, ENODEV);
3127 return 0;
3128 }
3129
3130 filt_spec_make_eof(kn);
3131 ret = FILTER_ACTIVE;
3132 goto out;
3133 }
3134
3135 if (kn->kn_vnode_use_ofst) {
3136 if (kn->kn_fp->fp_glob->fg_offset >= (uint32_t)selret) {
3137 data = 0;
3138 } else {
3139 data = ((uint32_t)selret) - kn->kn_fp->fp_glob->fg_offset;
3140 }
3141 } else {
3142 data = selret;
3143 }
3144
3145 if (data >= knote_low_watermark(kn)) {
3146 ret = FILTER_ACTIVE;
3147 } else {
3148 ret = 0;
3149 }
3150 out:
3151 if (ret) {
3152 knote_fill_kevent(kn, kev, data);
3153 }
3154 return ret;
3155 }
3156
3157 static int
filt_specattach(struct knote * kn,__unused struct kevent_qos_s * kev)3158 filt_specattach(struct knote *kn, __unused struct kevent_qos_s *kev)
3159 {
3160 vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp); /* Already have iocount, and vnode is alive */
3161 dev_t dev;
3162
3163 assert(vnode_ischr(vp));
3164
3165 dev = vnode_specrdev(vp);
3166
3167 /*
3168 * For a few special kinds of devices, we can attach knotes with
3169 * no restrictions because their "select" vectors return the amount
3170 * of data available. Others require an explicit NOTE_LOWAT with
3171 * data of 1, indicating that the caller doesn't care about actual
3172 * data counts, just an indication that the device has data.
3173 */
3174 if (!kn->kn_vnode_kqok &&
3175 ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) {
3176 knote_set_error(kn, EINVAL);
3177 return 0;
3178 }
3179
3180 return filt_spec_common(kn, kev, true);
3181 }
3182
3183 static void
filt_specdetach(struct knote * kn)3184 filt_specdetach(struct knote *kn)
3185 {
3186 selspec_detach(kn);
3187 }
3188
3189 static int
filt_specevent(struct knote * kn,long hint)3190 filt_specevent(struct knote *kn, long hint)
3191 {
3192 /* Due to selwakeup_internal() on SI_SELSPEC */
3193 assert(KNOTE_IS_AUTODETACHED(kn));
3194 knote_kn_hook_set_raw(kn, NULL);
3195
3196 /* called by selwakeup with the selspec_lock lock held */
3197 if (hint & NOTE_REVOKE) {
3198 filt_spec_make_eof(kn);
3199 }
3200 return FILTER_ACTIVE;
3201 }
3202
3203 static int
filt_spectouch(struct knote * kn,struct kevent_qos_s * kev)3204 filt_spectouch(struct knote *kn, struct kevent_qos_s *kev)
3205 {
3206 kn->kn_sdata = kev->data;
3207 kn->kn_sfflags = kev->fflags;
3208
3209 return filt_spec_common(kn, kev, false);
3210 }
3211
3212 static int
filt_specprocess(struct knote * kn,struct kevent_qos_s * kev)3213 filt_specprocess(struct knote *kn, struct kevent_qos_s *kev)
3214 {
3215 return filt_spec_common(kn, kev, false);
3216 }
3217