1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/buf_internal.h>
67 #include <sys/mount_internal.h>
68 #include <sys/vnode_internal.h>
69 #include <sys/trace.h>
70 #include <kern/kalloc.h>
71 #include <sys/time.h>
72 #include <sys/kernel.h>
73 #include <sys/resourcevar.h>
74 #include <miscfs/specfs/specdev.h>
75 #include <sys/uio_internal.h>
76 #include <libkern/libkern.h>
77 #include <machine/machine_routines.h>
78 #include <machine/smp.h>
79
80 #include <sys/ubc_internal.h>
81 #include <vm/vnode_pager.h>
82 #include <vm/vm_upl.h>
83
84 #include <mach/mach_types.h>
85 #include <mach/memory_object_types.h>
86 #include <mach/vm_map.h>
87 #include <mach/upl.h>
88 #include <mach/thread_info.h>
89 #include <kern/task.h>
90 #include <kern/policy_internal.h>
91 #include <kern/thread.h>
92
93 #include <vm/vm_kern_xnu.h>
94 #include <vm/vm_map_xnu.h>
95 #include <vm/vm_pageout_xnu.h>
96 #include <vm/vm_fault.h>
97 #include <vm/vm_ubc.h>
98
99 #include <sys/kdebug.h>
100 #include <sys/kdebug_triage.h>
101 #include <libkern/OSAtomic.h>
102
103 #include <sys/sdt.h>
104
105 #include <stdbool.h>
106
107 #include <vfs/vfs_disk_conditioner.h>
108
109 #if 0
110 #undef KERNEL_DEBUG
111 #define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT
112 #endif
113
114
115 #define CL_READ 0x01
116 #define CL_WRITE 0x02
117 #define CL_ASYNC 0x04
118 #define CL_COMMIT 0x08
119 #define CL_PAGEOUT 0x10
120 #define CL_AGE 0x20
121 #define CL_NOZERO 0x40
122 #define CL_PAGEIN 0x80
123 #define CL_DEV_MEMORY 0x100
124 #define CL_PRESERVE 0x200
125 #define CL_THROTTLE 0x400
126 #define CL_KEEPCACHED 0x800
127 #define CL_DIRECT_IO 0x1000
128 #define CL_PASSIVE 0x2000
129 #define CL_IOSTREAMING 0x4000
130 #define CL_CLOSE 0x8000
131 #define CL_ENCRYPTED 0x10000
132 #define CL_RAW_ENCRYPTED 0x20000
133 #define CL_NOCACHE 0x40000
134 #define CL_DIRECT_IO_FSBLKSZ 0x80000
135
136 #define MAX_VECTOR_UPL_SIZE (2 * MAX_UPL_SIZE_BYTES)
137
138 #define CLUSTER_IO_WAITING ((buf_t)1)
139
140 extern void vector_upl_set_iostate(upl_t, upl_t, vm_offset_t, upl_size_t);
141
142 struct clios {
143 lck_mtx_t io_mtxp;
144 u_int io_completed; /* amount of io that has currently completed */
145 u_int io_issued; /* amount of io that was successfully issued */
146 int io_error; /* error code of first error encountered */
147 int io_wanted; /* someone is sleeping waiting for a change in state */
148 };
149
150 struct cl_direct_read_lock {
151 LIST_ENTRY(cl_direct_read_lock) chain;
152 int32_t ref_count;
153 vnode_t vp;
154 lck_rw_t rw_lock;
155 };
156
157 #define CL_DIRECT_READ_LOCK_BUCKETS 61
158
159 static LIST_HEAD(cl_direct_read_locks, cl_direct_read_lock)
160 cl_direct_read_locks[CL_DIRECT_READ_LOCK_BUCKETS];
161
162 static LCK_GRP_DECLARE(cl_mtx_grp, "cluster I/O");
163 static LCK_MTX_DECLARE(cl_transaction_mtxp, &cl_mtx_grp);
164 static LCK_SPIN_DECLARE(cl_direct_read_spin_lock, &cl_mtx_grp);
165
166 static ZONE_DEFINE(cl_rd_zone, "cluster_read",
167 sizeof(struct cl_readahead), ZC_ZFREE_CLEARMEM);
168
169 static ZONE_DEFINE(cl_wr_zone, "cluster_write",
170 sizeof(struct cl_writebehind), ZC_ZFREE_CLEARMEM);
171
172 #define IO_UNKNOWN 0
173 #define IO_DIRECT 1
174 #define IO_CONTIG 2
175 #define IO_COPY 3
176
177 #define PUSH_DELAY 0x01
178 #define PUSH_ALL 0x02
179 #define PUSH_SYNC 0x04
180
181
182 static void cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset, size_t verify_block_size);
183 static void cluster_wait_IO(buf_t cbp_head, int async);
184 static void cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait);
185
186 static int cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length);
187
188 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
189 int flags, buf_t real_bp, struct clios *iostate, int (*)(buf_t, void *), void *callback_arg);
190 static void cluster_iodone_verify_continue(void);
191 static int cluster_iodone(buf_t bp, void *callback_arg);
192 static int cluster_iodone_finish(buf_t cbp_head, void *callback_arg);
193 static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags, vnode_t vp);
194 static int cluster_is_throttled(vnode_t vp);
195
196 static void cluster_iostate_wait(struct clios *iostate, u_int target, const char *wait_name);
197
198 static void cluster_syncup(vnode_t vp, off_t newEOF, int (*)(buf_t, void *), void *callback_arg, int flags);
199
200 static int cluster_handle_split_pagein(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
201 u_int io_size, int rounded_size, int local_flags, int (*callback)(buf_t, void *), void *callback_arg);
202
203 static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference);
204 static int cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference);
205
206 static int cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags,
207 int (*)(buf_t, void *), void *callback_arg) __attribute__((noinline));
208 static int cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
209 int flags, int (*)(buf_t, void *), void *callback_arg) __attribute__((noinline));
210 static int cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
211 int (*)(buf_t, void *), void *callback_arg, int flags) __attribute__((noinline));
212
213 static int cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF,
214 off_t headOff, off_t tailOff, int flags, int (*)(buf_t, void *), void *callback_arg) __attribute__((noinline));
215 static int cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
216 int flags, int (*callback)(buf_t, void *), void *callback_arg, uint32_t min_io_size) __attribute__((noinline));
217 static int cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF,
218 int *write_type, u_int32_t *write_length, int (*)(buf_t, void *), void *callback_arg, int bflag) __attribute__((noinline));
219
220 static void cluster_update_state_internal(vnode_t vp, struct cl_extent *cl, int flags, boolean_t defer_writes, boolean_t *first_pass,
221 off_t write_off, int write_cnt, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated);
222
223 static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*)(buf_t, void *), void *callback_arg);
224
225 static int cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
226 static void cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra,
227 int (*callback)(buf_t, void *), void *callback_arg, int bflag);
228
229 static int cluster_push_now(vnode_t vp, struct cl_extent *, off_t EOF, int flags, int (*)(buf_t, void *), void *callback_arg, boolean_t vm_ioitiated);
230
231 static int cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int flags, int (*)(buf_t, void *),
232 void *callback_arg, int *err, boolean_t vm_initiated);
233
234 static int sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF, int (*)(buf_t, void *), void *callback_arg, boolean_t vm_initiated);
235 static int sparse_cluster_push(struct cl_writebehind *, void **cmapp, vnode_t vp, off_t EOF, int push_flag,
236 int io_flags, int (*)(buf_t, void *), void *callback_arg, boolean_t vm_initiated);
237 static int sparse_cluster_add(struct cl_writebehind *, void **cmapp, vnode_t vp, struct cl_extent *, off_t EOF,
238 int (*)(buf_t, void *), void *callback_arg, boolean_t vm_initiated);
239
240 static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp);
241 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
242 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
243 static kern_return_t vfs_get_scmap_push_behavior_internal(void **cmapp, int *push_flag);
244
245
246 /*
247 * For throttled IO to check whether
248 * a block is cached by the boot cache
249 * and thus it can avoid delaying the IO.
250 *
251 * bootcache_contains_block is initially
252 * NULL. The BootCache will set it while
253 * the cache is active and clear it when
254 * the cache is jettisoned.
255 *
256 * Returns 0 if the block is not
257 * contained in the cache, 1 if it is
258 * contained.
259 *
260 * The function pointer remains valid
261 * after the cache has been evicted even
262 * if bootcache_contains_block has been
263 * cleared.
264 *
265 * See rdar://9974130 The new throttling mechanism breaks the boot cache for throttled IOs
266 */
267 int (*bootcache_contains_block)(dev_t device, u_int64_t blkno) = NULL;
268
269
270 /*
271 * limit the internal I/O size so that we
272 * can represent it in a 32 bit int
273 */
274 #define MAX_IO_REQUEST_SIZE (1024 * 1024 * 512)
275 #define MAX_IO_CONTIG_SIZE MAX_UPL_SIZE_BYTES
276 #define MAX_VECTS 16
277 /*
278 * The MIN_DIRECT_WRITE_SIZE governs how much I/O should be issued before we consider
279 * allowing the caller to bypass the buffer cache. For small I/Os (less than 16k),
280 * we have not historically allowed the write to bypass the UBC.
281 */
282 #define MIN_DIRECT_WRITE_SIZE (16384)
283
284 #define WRITE_THROTTLE 6
285 #define WRITE_THROTTLE_SSD 2
286 #define WRITE_BEHIND 1
287 #define WRITE_BEHIND_SSD 1
288
289 #if !defined(XNU_TARGET_OS_OSX)
290 #define PREFETCH 1
291 #define PREFETCH_SSD 1
292 uint32_t speculative_prefetch_max = (2048 * 1024); /* maximum bytes in a specluative read-ahead */
293 uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead */
294 #else /* XNU_TARGET_OS_OSX */
295 #define PREFETCH 3
296 #define PREFETCH_SSD 2
297 uint32_t speculative_prefetch_max = (MAX_UPL_SIZE_BYTES * 3); /* maximum bytes in a specluative read-ahead */
298 uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead on SSDs*/
299 #endif /* ! XNU_TARGET_OS_OSX */
300
301 /* maximum bytes for read-ahead */
302 uint32_t prefetch_max = (1024 * 1024 * 1024);
303 /* maximum bytes for outstanding reads */
304 uint32_t overlapping_read_max = (1024 * 1024 * 1024);
305 /* maximum bytes for outstanding writes */
306 uint32_t overlapping_write_max = (1024 * 1024 * 1024);
307
308 #define IO_SCALE(vp, base) (vp->v_mount->mnt_ioscale * (base))
309 #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE))
310
311 int speculative_reads_disabled = 0;
312
313 /*
314 * throttle the number of async writes that
315 * can be outstanding on a single vnode
316 * before we issue a synchronous write
317 */
318 #define THROTTLE_MAXCNT 0
319
320 uint32_t throttle_max_iosize = (128 * 1024);
321
322 #define THROTTLE_MAX_IOSIZE (throttle_max_iosize)
323
324 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_max_iosize, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_max_iosize, 0, "");
325
326 uint32_t split_pgin = 1;
327 uint32_t split_all_pgin = 1;
328 uint32_t split_all_pgin_equal = 0;
329 uint32_t split_pgin_headio = 0;
330
331 SYSCTL_INT(_kern, OID_AUTO, split_pagein_io, CTLFLAG_RW | CTLFLAG_LOCKED, &split_pgin, 0, "");
332 #if DEVELOPMENT || DEBUG
333 SYSCTL_INT(_kern, OID_AUTO, split_pagein_io_all, CTLFLAG_RW | CTLFLAG_LOCKED, &split_all_pgin, 0, "");
334 SYSCTL_INT(_kern, OID_AUTO, split_pagein_io_equal, CTLFLAG_RW | CTLFLAG_LOCKED, &split_all_pgin_equal, 0, "");
335 SYSCTL_INT(_kern, OID_AUTO, split_pagein_do_headio, CTLFLAG_RW | CTLFLAG_LOCKED, &split_pgin_headio, 0, "");
336 #endif
337
338 struct verify_buf {
339 TAILQ_ENTRY(verify_buf) vb_entry;
340 buf_t vb_cbp;
341 void* vb_callback_arg;
342 int32_t vb_whichq;
343 };
344
345 TAILQ_HEAD(, verify_buf) verify_free_head;
346 TAILQ_HEAD(, verify_buf) verify_work_head;
347
348 #define MAX_VERIFY_THREADS 4
349 #define MAX_REQUESTS_PER_THREAD 2
350
351 static struct verify_buf verify_bufs[MAX_VERIFY_THREADS * MAX_REQUESTS_PER_THREAD];
352 /*
353 * Each thread needs to check if the item at the head of the queue has a UPL
354 * pointer that is any of the threads are currently operating on.
355 * slot 0 is for the io completion thread to do the request inline if there are no free
356 * queue slots.
357 */
358 static int verify_in_flight = 0;
359
360 #if defined(XNU_TARGET_OS_IOS) || defined(XNU_TARGET_OS_XR)
361 #define NUM_DEFAULT_THREADS 2
362 #elif defined(XNU_TARGET_OS_OSX)
363 #define NUM_DEFAULT_THREADS 4
364 #else
365 #define NUM_DEFAULT_THREADS 0
366 #endif
367
368 static TUNABLE(uint32_t, num_verify_threads, "num_verify_threads", NUM_DEFAULT_THREADS);
369 static uint32_t cluster_verify_threads = 0; /* will be launched as needed upto num_verify_threads */
370
371 #if __AMP__
372 static TUNABLE(uint32_t, ecore_verify_threads, "ecore_verify_threads", false);
373 #endif /* __AMP__ */
374
375 static void
cluster_verify_init(void)376 cluster_verify_init(void)
377 {
378 TAILQ_INIT(&verify_free_head);
379 TAILQ_INIT(&verify_work_head);
380
381 if (num_verify_threads > MAX_VERIFY_THREADS) {
382 num_verify_threads = MAX_VERIFY_THREADS;
383 }
384
385 for (int i = 0; i < num_verify_threads * MAX_REQUESTS_PER_THREAD; i++) {
386 TAILQ_INSERT_TAIL(&verify_free_head, &verify_bufs[i], vb_entry);
387 }
388 }
389
390 void
cluster_init(void)391 cluster_init(void)
392 {
393 for (int i = 0; i < CL_DIRECT_READ_LOCK_BUCKETS; ++i) {
394 LIST_INIT(&cl_direct_read_locks[i]);
395 }
396
397 cluster_verify_init();
398 }
399
400 uint32_t
cluster_max_io_size(mount_t mp,int type)401 cluster_max_io_size(mount_t mp, int type)
402 {
403 uint32_t max_io_size;
404 uint32_t segcnt;
405 uint32_t maxcnt;
406
407 switch (type) {
408 case CL_READ:
409 segcnt = mp->mnt_segreadcnt;
410 maxcnt = mp->mnt_maxreadcnt;
411 break;
412 case CL_WRITE:
413 segcnt = mp->mnt_segwritecnt;
414 maxcnt = mp->mnt_maxwritecnt;
415 break;
416 default:
417 segcnt = min(mp->mnt_segreadcnt, mp->mnt_segwritecnt);
418 maxcnt = min(mp->mnt_maxreadcnt, mp->mnt_maxwritecnt);
419 break;
420 }
421 if (segcnt > (MAX_UPL_SIZE_BYTES >> PAGE_SHIFT)) {
422 /*
423 * don't allow a size beyond the max UPL size we can create
424 */
425 segcnt = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
426 }
427 max_io_size = min((segcnt * PAGE_SIZE), maxcnt);
428
429 if (max_io_size < MAX_UPL_TRANSFER_BYTES) {
430 /*
431 * don't allow a size smaller than the old fixed limit
432 */
433 max_io_size = MAX_UPL_TRANSFER_BYTES;
434 } else {
435 /*
436 * make sure the size specified is a multiple of PAGE_SIZE
437 */
438 max_io_size &= ~PAGE_MASK;
439 }
440 return max_io_size;
441 }
442
443 /*
444 * Returns max prefetch value. If the value overflows or exceeds the specified
445 * 'prefetch_limit', it will be capped at 'prefetch_limit' value.
446 */
447 static inline uint32_t
cluster_max_prefetch(vnode_t vp,uint32_t max_io_size,uint32_t prefetch_limit)448 cluster_max_prefetch(vnode_t vp, uint32_t max_io_size, uint32_t prefetch_limit)
449 {
450 bool is_ssd = disk_conditioner_mount_is_ssd(vp->v_mount);
451 uint32_t io_scale = IO_SCALE(vp, is_ssd ? PREFETCH_SSD : PREFETCH);
452 uint32_t prefetch = 0;
453
454 if (__improbable(os_mul_overflow(max_io_size, io_scale, &prefetch) ||
455 (prefetch > prefetch_limit))) {
456 prefetch = prefetch_limit;
457 }
458
459 return prefetch;
460 }
461
462 static inline uint32_t
calculate_max_throttle_size(vnode_t vp)463 calculate_max_throttle_size(vnode_t vp)
464 {
465 bool is_ssd = disk_conditioner_mount_is_ssd(vp->v_mount);
466 uint32_t io_scale = IO_SCALE(vp, is_ssd ? 2 : 1);
467
468 return MIN(io_scale * THROTTLE_MAX_IOSIZE, MAX_UPL_TRANSFER_BYTES);
469 }
470
471 static inline uint32_t
calculate_max_throttle_cnt(vnode_t vp)472 calculate_max_throttle_cnt(vnode_t vp)
473 {
474 bool is_ssd = disk_conditioner_mount_is_ssd(vp->v_mount);
475 uint32_t io_scale = IO_SCALE(vp, 1);
476
477 return is_ssd ? MIN(io_scale, 4) : THROTTLE_MAXCNT;
478 }
479
480 #define CLW_ALLOCATE 0x01
481 #define CLW_RETURNLOCKED 0x02
482 #define CLW_IONOCACHE 0x04
483 #define CLW_IOPASSIVE 0x08
484
485 /*
486 * if the read ahead context doesn't yet exist,
487 * allocate and initialize it...
488 * the vnode lock serializes multiple callers
489 * during the actual assignment... first one
490 * to grab the lock wins... the other callers
491 * will release the now unnecessary storage
492 *
493 * once the context is present, try to grab (but don't block on)
494 * the lock associated with it... if someone
495 * else currently owns it, than the read
496 * will run without read-ahead. this allows
497 * multiple readers to run in parallel and
498 * since there's only 1 read ahead context,
499 * there's no real loss in only allowing 1
500 * reader to have read-ahead enabled.
501 */
502 static struct cl_readahead *
cluster_get_rap(vnode_t vp)503 cluster_get_rap(vnode_t vp)
504 {
505 struct ubc_info *ubc;
506 struct cl_readahead *rap;
507
508 ubc = vp->v_ubcinfo;
509
510 if ((rap = ubc->cl_rahead) == NULL) {
511 rap = zalloc_flags(cl_rd_zone, Z_WAITOK | Z_ZERO);
512 rap->cl_lastr = -1;
513 lck_mtx_init(&rap->cl_lockr, &cl_mtx_grp, LCK_ATTR_NULL);
514
515 vnode_lock(vp);
516
517 if (ubc->cl_rahead == NULL) {
518 ubc->cl_rahead = rap;
519 } else {
520 lck_mtx_destroy(&rap->cl_lockr, &cl_mtx_grp);
521 zfree(cl_rd_zone, rap);
522 rap = ubc->cl_rahead;
523 }
524 vnode_unlock(vp);
525 }
526 if (lck_mtx_try_lock(&rap->cl_lockr) == TRUE) {
527 return rap;
528 }
529
530 return (struct cl_readahead *)NULL;
531 }
532
533
534 /*
535 * if the write behind context doesn't yet exist,
536 * and CLW_ALLOCATE is specified, allocate and initialize it...
537 * the vnode lock serializes multiple callers
538 * during the actual assignment... first one
539 * to grab the lock wins... the other callers
540 * will release the now unnecessary storage
541 *
542 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
543 * the lock associated with the write behind context before
544 * returning
545 */
546
547 static struct cl_writebehind *
cluster_get_wbp(vnode_t vp,int flags)548 cluster_get_wbp(vnode_t vp, int flags)
549 {
550 struct ubc_info *ubc;
551 struct cl_writebehind *wbp;
552
553 ubc = vp->v_ubcinfo;
554
555 if ((wbp = ubc->cl_wbehind) == NULL) {
556 if (!(flags & CLW_ALLOCATE)) {
557 return (struct cl_writebehind *)NULL;
558 }
559
560 wbp = zalloc_flags(cl_wr_zone, Z_WAITOK | Z_ZERO);
561
562 lck_mtx_init(&wbp->cl_lockw, &cl_mtx_grp, LCK_ATTR_NULL);
563
564 vnode_lock(vp);
565
566 if (ubc->cl_wbehind == NULL) {
567 ubc->cl_wbehind = wbp;
568 } else {
569 lck_mtx_destroy(&wbp->cl_lockw, &cl_mtx_grp);
570 zfree(cl_wr_zone, wbp);
571 wbp = ubc->cl_wbehind;
572 }
573 vnode_unlock(vp);
574 }
575 if (flags & CLW_RETURNLOCKED) {
576 lck_mtx_lock(&wbp->cl_lockw);
577 }
578
579 return wbp;
580 }
581
582
583 static void
cluster_syncup(vnode_t vp,off_t newEOF,int (* callback)(buf_t,void *),void * callback_arg,int flags)584 cluster_syncup(vnode_t vp, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg, int flags)
585 {
586 struct cl_writebehind *wbp;
587
588 if ((wbp = cluster_get_wbp(vp, 0)) != NULL) {
589 if (wbp->cl_number) {
590 lck_mtx_lock(&wbp->cl_lockw);
591
592 cluster_try_push(wbp, vp, newEOF, PUSH_ALL | flags, 0, callback, callback_arg, NULL, FALSE);
593
594 lck_mtx_unlock(&wbp->cl_lockw);
595 }
596 }
597 }
598
599
600 static int
cluster_io_present_in_BC(vnode_t vp,off_t f_offset)601 cluster_io_present_in_BC(vnode_t vp, off_t f_offset)
602 {
603 daddr64_t blkno;
604 size_t io_size;
605 int (*bootcache_check_fn)(dev_t device, u_int64_t blkno) = bootcache_contains_block;
606
607 if (bootcache_check_fn && vp->v_mount && vp->v_mount->mnt_devvp) {
608 if (VNOP_BLOCKMAP(vp, f_offset, PAGE_SIZE, &blkno, &io_size, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL)) {
609 return 0;
610 }
611
612 if (io_size == 0) {
613 return 0;
614 }
615
616 if (bootcache_check_fn(vp->v_mount->mnt_devvp->v_rdev, blkno)) {
617 return 1;
618 }
619 }
620 return 0;
621 }
622
623
624 static int
cluster_is_throttled(vnode_t vp)625 cluster_is_throttled(vnode_t vp)
626 {
627 return throttle_io_will_be_throttled(-1, vp->v_mount);
628 }
629
630
631 static void
cluster_iostate_wait(struct clios * iostate,u_int target,const char * wait_name)632 cluster_iostate_wait(struct clios *iostate, u_int target, const char *wait_name)
633 {
634 lck_mtx_lock(&iostate->io_mtxp);
635
636 while ((iostate->io_issued - iostate->io_completed) > target) {
637 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
638 iostate->io_issued, iostate->io_completed, target, 0, 0);
639
640 iostate->io_wanted = 1;
641 msleep((caddr_t)&iostate->io_wanted, &iostate->io_mtxp, PRIBIO + 1, wait_name, NULL);
642
643 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
644 iostate->io_issued, iostate->io_completed, target, 0, 0);
645 }
646 lck_mtx_unlock(&iostate->io_mtxp);
647 }
648
649
650 static void
cluster_handle_associated_upl(struct clios * iostate,upl_t upl,upl_offset_t upl_offset,upl_size_t size,off_t f_offset)651 cluster_handle_associated_upl(struct clios *iostate, upl_t upl,
652 upl_offset_t upl_offset, upl_size_t size, off_t f_offset)
653 {
654 if (!size) {
655 return;
656 }
657
658 upl_t associated_upl = upl_associated_upl(upl);
659
660 if (!associated_upl) {
661 return;
662 }
663
664 /*
665 * The associated upl functions as a "range lock" for the file.
666 *
667 * The associated upl is created and is attached to to the upl in
668 * cluster_io when the direct io write is being started. Since the
669 * upl may be released in parts so the corresponding associated upl
670 * has to be released in parts as well.
671 *
672 * We have the f_offset, upl_offset and size and from that we have figure
673 * out the associated upl offset and length, we are interested in.
674 */
675 upl_offset_t assoc_upl_offset, assoc_upl_end;
676
677 /* ALIGNED UPL's */
678 if ((upl_offset & PAGE_MASK) == (f_offset & PAGE_MASK)) {
679 assoc_upl_offset = trunc_page_32(upl_offset);
680 assoc_upl_end = round_page_32(upl_offset + size);
681 goto do_commit;
682 }
683
684 /*
685 * HANDLE UNALIGNED UPLS
686 *
687 * ( See also cluster_io where the associated upl is created )
688 * While we create the upl in one go, we will be dumping the pages in
689 * the upl in "transaction sized chunks" relative to the upl. Except
690 * for the first transction, the upl_offset will always be page aligned.
691 * and when the upl's are not aligned the associated upl offset will not
692 * be page aligned and so we have to truncate and round up the starting
693 * and the end of the pages in question and see if they are shared with
694 * other transctions or not. If two transctions "share" a page in the
695 * associated upl, the first one to complete "marks" it and skips that
696 * page and the second one will include it in the "commit range"
697 *
698 * As an example, consider the case where 4 transctions are needed (this
699 * is the worst case).
700 *
701 * Transaction for 0-1 (size -> PAGE_SIZE - upl_offset)
702 *
703 * This covers the associated upl from a -> c. a->b is not shared but
704 * b-c is shared with the next transction so the first one to complete
705 * will only "mark" it.
706 *
707 * Transaction for 1-2 (size -> PAGE_SIZE)
708 *
709 * For transaction 1, assoc_upl_offset would be 0 (corresponding to the
710 * file offset a or b depending on what file offset the upl_offset
711 * corrssponds to ) and assoc_upl_end would correspond to the file
712 * offset c.
713 *
714 * (associated_upl - based on f_offset alignment)
715 * 0 a b c d e f
716 * <----|----|----|----|----|----|-----|---->
717 *
718 *
719 * (upl - based on user buffer address alignment)
720 * <__--|----|----|--__>
721 *
722 * 0 1 2 3
723 *
724 */
725 upl_size_t assoc_upl_size = upl_get_size(associated_upl);
726 #if 0
727 /* knock off the simple case first -> this transaction covers the entire UPL */
728 upl_offset_t upl_end = round_page_32(upl_offset + size);
729 upl_size_t upl_size = vector_upl_get_size(upl);
730
731 if ((trunc_page_32(upl_offset) == 0) && (upl_end == upl_size)) {
732 assoc_upl_offset = 0;
733 assoc_upl_end = assoc_upl_size;
734 goto do_commit;
735 }
736 #endif
737 off_t assoc_upl_start_f_offset = upl_adjusted_offset(associated_upl, PAGE_MASK);
738
739 assoc_upl_offset = (upl_offset_t)trunc_page_64(f_offset - assoc_upl_start_f_offset);
740 assoc_upl_end = round_page_64(f_offset + size) - assoc_upl_start_f_offset;
741
742 /*
743 * We can only sanity check the offset returned by upl_adjusted_offset
744 * for the first transaction for this UPL i.e. when (upl_offset < PAGE_SIZE)
745 */
746 assertf((upl_offset >= PAGE_SIZE) || ((assoc_upl_start_f_offset == trunc_page_64(f_offset)) && (assoc_upl_offset == 0)),
747 "upl_offset = %d, f_offset = %lld, size = %d, start_f_offset = %lld, assoc_upl_offset = %d",
748 upl_offset, f_offset, size, assoc_upl_start_f_offset, assoc_upl_offset);
749
750 assertf((upl_offset == assoc_upl_offset) || (upl_offset > assoc_upl_offset && ((upl_offset - assoc_upl_offset) <= PAGE_SIZE)) ||
751 (assoc_upl_offset > upl_offset && ((assoc_upl_offset - upl_offset) <= PAGE_SIZE)),
752 "abs(upl_offset - assoc_upl_offset) > PAGE_SIZE : "
753 "upl_offset = %d, f_offset = %lld, size = %d, start_f_offset = %lld, assoc_upl_offset = %d",
754 upl_offset, f_offset, size, assoc_upl_start_f_offset, assoc_upl_offset);
755
756 assertf(assoc_upl_end <= assoc_upl_size,
757 "upl_offset = %d, f_offset = %lld, size = %d, start_f_offset = %lld, assoc_upl_size = %d, assoc_upl_offset = %d, assoc_upl_end = %d",
758 upl_offset, f_offset, size, assoc_upl_start_f_offset, assoc_upl_size, assoc_upl_offset, assoc_upl_end);
759
760 assertf((assoc_upl_size > PAGE_SIZE) || (assoc_upl_offset == 0 && assoc_upl_end == PAGE_SIZE),
761 "upl_offset = %d, f_offset = %lld, size = %d, start_f_offset = %lld, assoc_upl_size = %d, assoc_upl_offset = %d, assoc_upl_end = %d",
762 upl_offset, f_offset, size, assoc_upl_start_f_offset, assoc_upl_size, assoc_upl_offset, assoc_upl_end);
763
764 if (assoc_upl_size == PAGE_SIZE) {
765 assoc_upl_offset = 0;
766 assoc_upl_end = PAGE_SIZE;
767 goto do_commit;
768 }
769
770 /*
771 * We have to check if the first and last pages of the associated UPL
772 * range could potentially be shared with other transactions and if the
773 * "sharing transactions" are both done. The first one sets the mark bit
774 * and the second one checks it and if set it includes that page in the
775 * pages to be "freed".
776 */
777 bool check_first_pg = (assoc_upl_offset != 0) || ((f_offset + size) < (assoc_upl_start_f_offset + PAGE_SIZE));
778 bool check_last_pg = (assoc_upl_end != assoc_upl_size) || (f_offset > ((assoc_upl_start_f_offset + assoc_upl_size) - PAGE_SIZE));
779
780 if (check_first_pg || check_last_pg) {
781 int first_pg = assoc_upl_offset >> PAGE_SHIFT;
782 int last_pg = trunc_page_32(assoc_upl_end - 1) >> PAGE_SHIFT;
783 upl_page_info_t *assoc_pl = UPL_GET_INTERNAL_PAGE_LIST(associated_upl);
784
785 lck_mtx_lock_spin(&iostate->io_mtxp);
786 if (check_first_pg && !upl_page_get_mark(assoc_pl, first_pg)) {
787 /*
788 * The first page isn't marked so let another transaction
789 * completion handle it.
790 */
791 upl_page_set_mark(assoc_pl, first_pg, true);
792 assoc_upl_offset += PAGE_SIZE;
793 }
794 if (check_last_pg && !upl_page_get_mark(assoc_pl, last_pg)) {
795 /*
796 * The last page isn't marked so mark the page and let another
797 * transaction completion handle it.
798 */
799 upl_page_set_mark(assoc_pl, last_pg, true);
800 assoc_upl_end -= PAGE_SIZE;
801 }
802 lck_mtx_unlock(&iostate->io_mtxp);
803 }
804
805 if (assoc_upl_end <= assoc_upl_offset) {
806 return;
807 }
808
809 do_commit:
810 size = assoc_upl_end - assoc_upl_offset;
811
812 boolean_t empty;
813
814 /*
815 * We can unlock these pages now and as this is for a
816 * direct/uncached write, we want to dump the pages too.
817 */
818 kern_return_t kr = upl_abort_range(associated_upl, assoc_upl_offset, size,
819 UPL_ABORT_DUMP_PAGES, &empty);
820
821 assert(!kr);
822
823 if (!kr && empty) {
824 upl_set_associated_upl(upl, NULL);
825 upl_deallocate(associated_upl);
826 }
827 }
828
829 static void
cluster_iodone_verify_continue(void)830 cluster_iodone_verify_continue(void)
831 {
832 lck_mtx_lock_spin(&cl_transaction_mtxp);
833 for (;;) {
834 struct verify_buf *vb = TAILQ_FIRST(&verify_work_head);
835
836 if (!vb) {
837 assert_wait(&verify_work_head, (THREAD_UNINT));
838 break;
839 }
840 buf_t cbp = vb->vb_cbp;
841 void* callback_arg = vb->vb_callback_arg;
842
843 TAILQ_REMOVE(&verify_work_head, vb, vb_entry);
844 vb->vb_cbp = NULL;
845 vb->vb_callback_arg = NULL;
846 vb->vb_whichq = 0;
847 TAILQ_INSERT_TAIL(&verify_free_head, vb, vb_entry);
848 lck_mtx_unlock(&cl_transaction_mtxp);
849
850 (void)cluster_iodone_finish(cbp, callback_arg);
851 cbp = NULL;
852 lck_mtx_lock_spin(&cl_transaction_mtxp);
853 }
854 lck_mtx_unlock(&cl_transaction_mtxp);
855 thread_block((thread_continue_t)cluster_iodone_verify_continue);
856 /* NOT REACHED */
857 }
858
859 static void
cluster_verify_thread(void)860 cluster_verify_thread(void)
861 {
862 thread_t self = current_thread();
863
864 thread_set_thread_name(self, "cluster_verify_thread");
865 #if __AMP__
866 if (ecore_verify_threads) {
867 thread_soft_bind_cluster_type(self, 'E');
868 }
869 #endif /* __AMP__ */
870 #if !defined(__x86_64__)
871 thread_group_join_io_storage();
872 #endif /* __x86_64__ */
873 cluster_iodone_verify_continue();
874 /* NOT REACHED */
875 }
876
877 static bool
enqueue_buf_for_verify(buf_t cbp,void * callback_arg)878 enqueue_buf_for_verify(buf_t cbp, void *callback_arg)
879 {
880 struct verify_buf *vb;
881
882 vb = TAILQ_FIRST(&verify_free_head);
883 if (vb) {
884 TAILQ_REMOVE(&verify_free_head, vb, vb_entry);
885 vb->vb_cbp = cbp;
886 vb->vb_callback_arg = callback_arg;
887 vb->vb_whichq = 1;
888 TAILQ_INSERT_TAIL(&verify_work_head, vb, vb_entry);
889 return true;
890 } else {
891 return false;
892 }
893 }
894
895 static int
cluster_handle_verification(buf_t cbp_head,vnode_t vp,upl_t upl,int upl_offset,int transaction_size,int error)896 cluster_handle_verification(buf_t cbp_head, vnode_t vp, upl_t upl, int upl_offset, int transaction_size, int error)
897 {
898 off_t start_off = cbp_head->b_clfoffset;
899 void *verify_ctx = cbp_head->b_attr.ba_un.verify_ctx;
900 caddr_t verify_buf = NULL;
901 uint32_t verify_length = transaction_size;
902 vnode_verify_flags_t verify_flags = VNODE_VERIFY_CONTEXT_FREE;
903 int verify_error = EAGAIN;
904
905 assert(cbp_head->b_attr.ba_flags & BA_WILL_VERIFY);
906
907 cbp_head->b_attr.ba_un.verify_ctx = NULL;
908 if (error) {
909 goto free_context;
910 }
911
912 /*
913 * If we don't have a precomputed hash, we make a single call to both
914 * verify and free the context. If we have a precomputed hash, then we
915 * make two separate calls - one to verify the hash and the second one to
916 * free. If the filesystem returns EAGAIN we fall back to the non
917 * precomputed hash case.
918 */
919 if (cbp_head->b_attr.ba_verify_type && cbp_head->b_attr.ba_flags & BA_VERIFY_VALID) {
920 verify_buf = (caddr_t)buf_verifyptr_with_size(cbp_head, transaction_size, &verify_length);
921 verify_flags = VNODE_VERIFY_WITH_CONTEXT | VNODE_VERIFY_PRECOMPUTED;
922
923 if (verify_buf && verify_length) {
924 verify_error = VNOP_VERIFY(vp, start_off, (uint8_t *)verify_buf, verify_length,
925 NULL, &verify_ctx, verify_flags, NULL, NULL);
926 } else {
927 verify_error = EAGAIN;
928 }
929
930 verify_buf = NULL;
931 verify_length = transaction_size;
932 verify_flags = VNODE_VERIFY_CONTEXT_FREE;
933 }
934
935 if (verify_error != EAGAIN) {
936 error = verify_error;
937 } else {
938 vm_offset_t vaddr;
939
940 /*
941 * Map it in.
942 *
943 * ubc_upl_map_range unfortunately cannot handle concurrent map
944 * requests for the same UPL and returns failures when it can't
945 * map. The map exclusive mechanism enforces mutual exclusion
946 * for concurrent requests.
947 */
948 verify_error = 0;
949 os_atomic_inc(&verify_in_flight, relaxed);
950 upl_set_map_exclusive(upl);
951 error = ubc_upl_map_range(upl, upl_offset, round_page(transaction_size), VM_PROT_DEFAULT, &vaddr);
952 if (error) {
953 upl_clear_map_exclusive(upl);
954 printf("ubc_upl_map_range returned error %d upl = %p, upl_offset = %d, size = %d",
955 error, upl, (int)upl_offset, (int)round_page(transaction_size));
956 error = EIO;
957 if (os_atomic_dec_orig(&verify_in_flight, relaxed) == 0) {
958 panic("verify_in_flight underflow");
959 }
960 } else {
961 verify_buf = (caddr_t)vaddr;
962 verify_flags |= VNODE_VERIFY_WITH_CONTEXT;
963 }
964 }
965
966 free_context:
967 verify_error = VNOP_VERIFY(vp, start_off, (uint8_t *)verify_buf, verify_length,
968 NULL, &verify_ctx, verify_flags, NULL, NULL);
969 if (!error) {
970 error = verify_error;
971 }
972
973 if (verify_buf) {
974 (void)ubc_upl_unmap_range(upl, upl_offset, round_page(transaction_size));
975 upl_clear_map_exclusive(upl);
976 verify_buf = NULL;
977 if (os_atomic_dec_orig(&verify_in_flight, relaxed) == 0) {
978 panic("verify_in_flight underflow");
979 }
980 }
981
982 return error;
983 }
984
985 static int
cluster_ioerror(upl_t upl,int upl_offset,int abort_size,int error,int io_flags,vnode_t vp)986 cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags, vnode_t vp)
987 {
988 int upl_abort_code = 0;
989 int page_in = 0;
990 int page_out = 0;
991
992 if ((io_flags & (B_PHYS | B_CACHE)) == (B_PHYS | B_CACHE)) {
993 /*
994 * direct write of any flavor, or a direct read that wasn't aligned
995 */
996 ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
997 } else {
998 if (io_flags & B_PAGEIO) {
999 if (io_flags & B_READ) {
1000 page_in = 1;
1001 } else {
1002 page_out = 1;
1003 }
1004 }
1005 if (io_flags & B_CACHE) {
1006 /*
1007 * leave pages in the cache unchanged on error
1008 */
1009 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
1010 } else if (((io_flags & B_READ) == 0) && ((error != ENXIO) || vnode_isswap(vp))) {
1011 /*
1012 * transient error on pageout/write path... leave pages unchanged
1013 */
1014 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
1015 } else if (page_in) {
1016 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
1017 } else {
1018 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
1019 }
1020
1021 ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
1022 }
1023 return upl_abort_code;
1024 }
1025
1026
1027 static int
cluster_iodone(buf_t bp,void * callback_arg)1028 cluster_iodone(buf_t bp, void *callback_arg)
1029 {
1030 buf_t cbp;
1031 buf_t cbp_head;
1032 int error = 0;
1033 boolean_t transaction_complete = FALSE;
1034 bool async;
1035
1036 __IGNORE_WCASTALIGN(cbp_head = (buf_t)(bp->b_trans_head));
1037
1038 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_START,
1039 cbp_head, bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
1040
1041 async = cluster_verify_threads &&
1042 (os_atomic_load(&cbp_head->b_attr.ba_flags, acquire) & BA_ASYNC_VERIFY);
1043
1044 assert(!async || cbp_head->b_attr.ba_un.verify_ctx);
1045
1046 if (cbp_head->b_trans_next || !(cbp_head->b_flags & B_EOT)) {
1047 lck_mtx_lock_spin(&cl_transaction_mtxp);
1048
1049 bp->b_flags |= B_TDONE;
1050
1051 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
1052 /*
1053 * all I/O requests that are part of this transaction
1054 * have to complete before we can process it
1055 */
1056 if (!(cbp->b_flags & B_TDONE)) {
1057 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
1058 cbp_head, cbp, cbp->b_bcount, cbp->b_flags, 0);
1059
1060 lck_mtx_unlock(&cl_transaction_mtxp);
1061
1062 return 0;
1063 }
1064
1065 if (cbp->b_trans_next == CLUSTER_IO_WAITING) {
1066 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
1067 cbp_head, cbp, cbp->b_bcount, cbp->b_flags, 0);
1068
1069 lck_mtx_unlock(&cl_transaction_mtxp);
1070 wakeup(cbp);
1071
1072 return 0;
1073 }
1074
1075 if (cbp->b_flags & B_EOT) {
1076 transaction_complete = TRUE;
1077
1078 if (async) {
1079 async = enqueue_buf_for_verify(cbp_head, callback_arg);
1080 }
1081 }
1082 }
1083 lck_mtx_unlock(&cl_transaction_mtxp);
1084
1085 if (transaction_complete == FALSE) {
1086 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
1087 cbp_head, 0, 0, 0, 0);
1088 return 0;
1089 }
1090 } else if (async) {
1091 lck_mtx_lock_spin(&cl_transaction_mtxp);
1092 async = enqueue_buf_for_verify(cbp_head, callback_arg);
1093 lck_mtx_unlock(&cl_transaction_mtxp);
1094 }
1095
1096 if (async) {
1097 wakeup(&verify_work_head);
1098 } else {
1099 error = cluster_iodone_finish(cbp_head, callback_arg);
1100 }
1101
1102 return error;
1103 }
1104
1105 static int
cluster_iodone_finish(buf_t cbp_head,void * callback_arg)1106 cluster_iodone_finish(buf_t cbp_head, void *callback_arg)
1107 {
1108 int b_flags;
1109 int error;
1110 int total_size;
1111 int total_resid;
1112 int upl_offset;
1113 int zero_offset;
1114 int pg_offset = 0;
1115 int commit_size = 0;
1116 int upl_flags = 0;
1117 int transaction_size = 0;
1118 upl_t upl;
1119 buf_t cbp;
1120 buf_t cbp_next;
1121 buf_t real_bp;
1122 vnode_t vp;
1123 struct clios *iostate;
1124
1125 error = 0;
1126 total_size = 0;
1127 total_resid = 0;
1128
1129 cbp = cbp_head;
1130 vp = cbp->b_vp;
1131 upl_offset = cbp->b_uploffset;
1132 upl = cbp->b_upl;
1133 b_flags = cbp->b_flags;
1134 real_bp = cbp->b_real_bp;
1135 zero_offset = cbp->b_validend;
1136 iostate = (struct clios *)cbp->b_iostate;
1137
1138 if (real_bp) {
1139 real_bp->b_dev = cbp->b_dev;
1140 }
1141
1142 while (cbp) {
1143 if ((cbp->b_flags & B_ERROR) && error == 0) {
1144 error = cbp->b_error;
1145 }
1146
1147 total_resid += cbp->b_resid;
1148 total_size += cbp->b_bcount;
1149
1150 cbp_next = cbp->b_trans_next;
1151
1152 if (cbp_next == NULL) {
1153 /*
1154 * compute the overall size of the transaction
1155 * in case we created one that has 'holes' in it
1156 * 'total_size' represents the amount of I/O we
1157 * did, not the span of the transaction w/r to the UPL
1158 */
1159 transaction_size = cbp->b_uploffset + cbp->b_bcount - upl_offset;
1160 }
1161
1162 cbp = cbp_next;
1163 }
1164
1165 if (ISSET(b_flags, B_COMMIT_UPL)) {
1166 cluster_handle_associated_upl(iostate,
1167 cbp_head->b_upl,
1168 upl_offset,
1169 transaction_size,
1170 cbp_head->b_clfoffset);
1171 }
1172
1173 if (error == 0 && total_resid) {
1174 error = EIO;
1175 }
1176
1177 if (error == 0) {
1178 int (*cliodone_func)(buf_t, void *) = (int (*)(buf_t, void *))(cbp_head->b_cliodone);
1179
1180 if (cliodone_func != NULL) {
1181 cbp_head->b_bcount = transaction_size;
1182
1183 error = (*cliodone_func)(cbp_head, callback_arg);
1184 }
1185 }
1186 if (zero_offset) {
1187 cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
1188 }
1189
1190 if (cbp_head->b_attr.ba_un.verify_ctx) {
1191 error = cluster_handle_verification(cbp_head, vp, upl, upl_offset, transaction_size, error);
1192 } else if (cbp_head->b_attr.ba_flags & BA_WILL_VERIFY) {
1193 error = EBADMSG;
1194 }
1195
1196 if (iostate) {
1197 int need_wakeup = 0;
1198
1199 /*
1200 * someone has issued multiple I/Os asynchrounsly
1201 * and is waiting for them to complete (streaming)
1202 */
1203 lck_mtx_lock_spin(&iostate->io_mtxp);
1204
1205 if (error && iostate->io_error == 0) {
1206 iostate->io_error = error;
1207 }
1208
1209 iostate->io_completed += total_size;
1210
1211 if (iostate->io_wanted) {
1212 /*
1213 * someone is waiting for the state of
1214 * this io stream to change
1215 */
1216 iostate->io_wanted = 0;
1217 need_wakeup = 1;
1218 }
1219 lck_mtx_unlock(&iostate->io_mtxp);
1220
1221 if (need_wakeup) {
1222 wakeup((caddr_t)&iostate->io_wanted);
1223 }
1224 }
1225
1226 if (b_flags & B_COMMIT_UPL) {
1227 pg_offset = upl_offset & PAGE_MASK;
1228 commit_size = (pg_offset + transaction_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1229
1230 if (error) {
1231 upl_set_iodone_error(upl, error);
1232
1233 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags, vp);
1234 } else {
1235 upl_flags = UPL_COMMIT_FREE_ON_EMPTY;
1236
1237 if ((b_flags & B_PHYS) && (b_flags & B_READ)) {
1238 upl_flags |= UPL_COMMIT_SET_DIRTY;
1239 }
1240
1241 if (b_flags & B_AGE) {
1242 upl_flags |= UPL_COMMIT_INACTIVATE;
1243 }
1244
1245 ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
1246 }
1247 }
1248
1249 cbp = cbp_head->b_trans_next;
1250 while (cbp) {
1251 cbp_next = cbp->b_trans_next;
1252
1253 if (cbp != cbp_head) {
1254 free_io_buf(cbp);
1255 }
1256
1257 cbp = cbp_next;
1258 }
1259 free_io_buf(cbp_head);
1260
1261 if (real_bp) {
1262 if (error) {
1263 real_bp->b_flags |= B_ERROR;
1264 real_bp->b_error = error;
1265 }
1266 real_bp->b_resid = total_resid;
1267
1268 buf_biodone(real_bp);
1269 }
1270 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
1271 upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
1272
1273 return error;
1274 }
1275
1276
1277 uint32_t
cluster_throttle_io_limit(vnode_t vp,uint32_t * limit)1278 cluster_throttle_io_limit(vnode_t vp, uint32_t *limit)
1279 {
1280 if (cluster_is_throttled(vp)) {
1281 *limit = calculate_max_throttle_size(vp);
1282 return 1;
1283 }
1284 return 0;
1285 }
1286
1287
1288 void
cluster_zero(upl_t upl,upl_offset_t upl_offset,int size,buf_t bp)1289 cluster_zero(upl_t upl, upl_offset_t upl_offset, int size, buf_t bp)
1290 {
1291 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
1292 upl_offset, size, bp, 0, 0);
1293
1294 if (bp == NULL || bp->b_datap == 0) {
1295 upl_page_info_t *pl;
1296 addr64_t zero_addr;
1297
1298 pl = ubc_upl_pageinfo(upl);
1299
1300 if (upl_device_page(pl) == TRUE) {
1301 zero_addr = ((addr64_t)upl_phys_page(pl, 0) << PAGE_SHIFT) + upl_offset;
1302
1303 bzero_phys_nc(zero_addr, size);
1304 } else {
1305 while (size) {
1306 int page_offset;
1307 int page_index;
1308 int zero_cnt;
1309
1310 page_index = upl_offset / PAGE_SIZE;
1311 page_offset = upl_offset & PAGE_MASK;
1312
1313 zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << PAGE_SHIFT) + page_offset;
1314 zero_cnt = min(PAGE_SIZE - page_offset, size);
1315
1316 bzero_phys(zero_addr, zero_cnt);
1317
1318 size -= zero_cnt;
1319 upl_offset += zero_cnt;
1320 }
1321 }
1322 } else {
1323 bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
1324 }
1325
1326 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_END,
1327 upl_offset, size, 0, 0, 0);
1328 }
1329
1330
1331 static void
cluster_EOT(buf_t cbp_head,buf_t cbp_tail,int zero_offset,size_t verify_block_size)1332 cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset, size_t verify_block_size)
1333 {
1334 /*
1335 * We will assign a verification context to cbp_head.
1336 * This will be passed back to the filesystem when
1337 * verifying (in cluster_iodone).
1338 */
1339 if (verify_block_size) {
1340 off_t start_off = cbp_head->b_clfoffset;
1341 size_t length;
1342 void *verify_ctx = NULL;
1343 int error = 0;
1344 vnode_t vp = buf_vnode(cbp_head);
1345
1346 if (cbp_head == cbp_tail) {
1347 length = cbp_head->b_bcount;
1348 } else {
1349 length = (cbp_tail->b_clfoffset + cbp_tail->b_bcount) - start_off;
1350 }
1351
1352 /*
1353 * zero_offset is non zero for the transaction containing the EOF
1354 * (if the filesize is not page aligned). In that case we might
1355 * have the transaction size not be page/verify block size aligned
1356 */
1357 if ((zero_offset == 0) &&
1358 ((length < verify_block_size) || (length % verify_block_size)) != 0) {
1359 panic("%s length = %zu, verify_block_size = %zu",
1360 __FUNCTION__, length, verify_block_size);
1361 }
1362
1363 error = VNOP_VERIFY(vp, start_off, NULL, length,
1364 &verify_block_size, &verify_ctx, VNODE_VERIFY_CONTEXT_ALLOC, NULL, NULL);
1365
1366 assert(!(error && verify_ctx));
1367
1368 if (verify_ctx) {
1369 if (num_verify_threads && (os_atomic_load(&cluster_verify_threads, relaxed) == 0)) {
1370 if (os_atomic_inc_orig(&cluster_verify_threads, relaxed) == 0) {
1371 thread_t thread;
1372 int i;
1373
1374 for (i = 0; i < num_verify_threads && i < MAX_VERIFY_THREADS; i++) {
1375 kernel_thread_start((thread_continue_t)cluster_verify_thread, NULL, &thread);
1376 thread_deallocate(thread);
1377 }
1378 os_atomic_store(&cluster_verify_threads, i, relaxed);
1379 } else {
1380 os_atomic_dec(&cluster_verify_threads, relaxed);
1381 }
1382 }
1383 cbp_head->b_attr.ba_un.verify_ctx = verify_ctx;
1384 /*
1385 * At least one thread is busy (at the time we
1386 * checked), so we can let it get queued for
1387 * async processing. It's fine if we occasionally get
1388 * this wrong.
1389 */
1390 if (os_atomic_load(&verify_in_flight, relaxed)) {
1391 /* This flag and the setting of ba_un.verify_ctx needs to be ordered */
1392 os_atomic_or(&cbp_head->b_attr.ba_flags, BA_ASYNC_VERIFY, release);
1393 }
1394 }
1395 } else {
1396 cbp_head->b_attr.ba_un.verify_ctx = NULL;
1397 }
1398
1399 cbp_head->b_validend = zero_offset;
1400 cbp_tail->b_flags |= B_EOT;
1401 }
1402
1403 static void
cluster_wait_IO(buf_t cbp_head,int async)1404 cluster_wait_IO(buf_t cbp_head, int async)
1405 {
1406 buf_t cbp;
1407
1408 if (async) {
1409 /*
1410 * Async callback completion will not normally generate a
1411 * wakeup upon I/O completion. To get woken up, we set
1412 * b_trans_next (which is safe for us to modify) on the last
1413 * buffer to CLUSTER_IO_WAITING so that cluster_iodone knows
1414 * to wake us up when all buffers as part of this transaction
1415 * are completed. This is done under the umbrella of
1416 * cl_transaction_mtxp which is also taken in cluster_iodone.
1417 */
1418 bool done = true;
1419 buf_t last = NULL;
1420
1421 lck_mtx_lock_spin(&cl_transaction_mtxp);
1422
1423 for (cbp = cbp_head; cbp; last = cbp, cbp = cbp->b_trans_next) {
1424 if (!ISSET(cbp->b_flags, B_TDONE)) {
1425 done = false;
1426 }
1427 }
1428
1429 if (!done) {
1430 last->b_trans_next = CLUSTER_IO_WAITING;
1431
1432 DTRACE_IO1(wait__start, buf_t, last);
1433 do {
1434 msleep(last, &cl_transaction_mtxp, PSPIN | (PRIBIO + 1), "cluster_wait_IO", NULL);
1435
1436 /*
1437 * We should only have been woken up if all the
1438 * buffers are completed, but just in case...
1439 */
1440 done = true;
1441 for (cbp = cbp_head; cbp != CLUSTER_IO_WAITING; cbp = cbp->b_trans_next) {
1442 if (!ISSET(cbp->b_flags, B_TDONE)) {
1443 done = false;
1444 break;
1445 }
1446 }
1447 } while (!done);
1448 DTRACE_IO1(wait__done, buf_t, last);
1449
1450 last->b_trans_next = NULL;
1451 }
1452
1453 lck_mtx_unlock(&cl_transaction_mtxp);
1454 } else { // !async
1455 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
1456 buf_biowait(cbp);
1457 }
1458 }
1459 }
1460
1461 static void
cluster_complete_transaction(buf_t * cbp_head,void * callback_arg,int * retval,int flags,int needwait)1462 cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait)
1463 {
1464 buf_t cbp;
1465 int error;
1466 boolean_t isswapout = FALSE;
1467
1468 /*
1469 * cluster_complete_transaction will
1470 * only be called if we've issued a complete chain in synchronous mode
1471 * or, we've already done a cluster_wait_IO on an incomplete chain
1472 */
1473 if (needwait) {
1474 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next) {
1475 buf_biowait(cbp);
1476 }
1477 }
1478 /*
1479 * we've already waited on all of the I/Os in this transaction,
1480 * so mark all of the buf_t's in this transaction as B_TDONE
1481 * so that cluster_iodone sees the transaction as completed
1482 */
1483 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next) {
1484 cbp->b_flags |= B_TDONE;
1485 cbp->b_attr.ba_flags &= ~BA_ASYNC_VERIFY;
1486 }
1487 cbp = *cbp_head;
1488
1489 if ((flags & (CL_ASYNC | CL_PAGEOUT)) == CL_PAGEOUT && vnode_isswap(cbp->b_vp)) {
1490 isswapout = TRUE;
1491 }
1492
1493 error = cluster_iodone(cbp, callback_arg);
1494
1495 if (!(flags & CL_ASYNC) && error && *retval == 0) {
1496 if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) != CL_PAGEOUT) || (error != ENXIO)) {
1497 *retval = error;
1498 } else if (isswapout == TRUE) {
1499 *retval = error;
1500 }
1501 }
1502 *cbp_head = (buf_t)NULL;
1503 }
1504
1505 uint64_t cluster_direct_write_wired = 0;
1506
1507 static int
cluster_io(vnode_t vp,upl_t upl,vm_offset_t upl_offset,off_t f_offset,int non_rounded_size,int flags,buf_t real_bp,struct clios * iostate,int (* callback)(buf_t,void *),void * callback_arg)1508 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
1509 int flags, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
1510 {
1511 buf_t cbp;
1512 u_int size;
1513 u_int io_size;
1514 int io_flags;
1515 int bmap_flags;
1516 int error = 0;
1517 int retval = 0;
1518 buf_t cbp_head = NULL;
1519 buf_t cbp_tail = NULL;
1520 int trans_count = 0;
1521 int max_trans_count;
1522 u_int pg_count;
1523 int pg_offset;
1524 u_int max_iosize;
1525 u_int max_vectors;
1526 int priv;
1527 int zero_offset = 0;
1528 int async_throttle = 0;
1529 mount_t mp;
1530 size_t verify_block_size = 0;
1531 vm_offset_t upl_end_offset;
1532 vnode_verify_kind_t verify_kind = VK_HASH_NONE;
1533 boolean_t need_EOT = FALSE;
1534
1535 /*
1536 * we currently don't support buffers larger than a page
1537 */
1538 if (real_bp && non_rounded_size > PAGE_SIZE) {
1539 panic("%s(): Called with real buffer of size %d bytes which "
1540 "is greater than the maximum allowed size of "
1541 "%d bytes (the system PAGE_SIZE).\n",
1542 __FUNCTION__, non_rounded_size, PAGE_SIZE);
1543 }
1544
1545 mp = vp->v_mount;
1546
1547 /*
1548 * we don't want to do any funny rounding of the size for IO requests
1549 * coming through the DIRECT or CONTIGUOUS paths... those pages don't
1550 * belong to us... we can't extend (nor do we need to) the I/O to fill
1551 * out a page
1552 */
1553 if (mp->mnt_devblocksize > 1 && !(flags & (CL_DEV_MEMORY | CL_DIRECT_IO))) {
1554 /*
1555 * round the requested size up so that this I/O ends on a
1556 * page boundary in case this is a 'write'... if the filesystem
1557 * has blocks allocated to back the page beyond the EOF, we want to
1558 * make sure to write out the zero's that are sitting beyond the EOF
1559 * so that in case the filesystem doesn't explicitly zero this area
1560 * if a hole is created via a lseek/write beyond the current EOF,
1561 * it will return zeros when it's read back from the disk. If the
1562 * physical allocation doesn't extend for the whole page, we'll
1563 * only write/read from the disk up to the end of this allocation
1564 * via the extent info returned from the VNOP_BLOCKMAP call.
1565 */
1566 pg_offset = upl_offset & PAGE_MASK;
1567
1568 size = (((non_rounded_size + pg_offset) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - pg_offset;
1569 } else {
1570 /*
1571 * anyone advertising a blocksize of 1 byte probably
1572 * can't deal with us rounding up the request size
1573 * AFP is one such filesystem/device
1574 */
1575 size = non_rounded_size;
1576 }
1577 upl_end_offset = upl_offset + size;
1578
1579 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START, (int)f_offset, size, upl_offset, flags, 0);
1580
1581 /*
1582 * Set the maximum transaction size to the maximum desired number of
1583 * buffers.
1584 */
1585 max_trans_count = 8;
1586 if (flags & CL_DEV_MEMORY) {
1587 max_trans_count = 16;
1588 }
1589
1590 if (flags & CL_READ) {
1591 io_flags = B_READ;
1592 bmap_flags = VNODE_READ;
1593
1594 max_iosize = mp->mnt_maxreadcnt;
1595 max_vectors = mp->mnt_segreadcnt;
1596
1597 /* See if we can do cluster verification (pageins and aligned reads) */
1598 if ((flags & CL_PAGEIN || cluster_verify_threads) &&
1599 !(mp->mnt_kern_flag & MNTK_VIRTUALDEV) &&
1600 (VNOP_VERIFY(vp, f_offset, NULL, 0, &verify_block_size, NULL, VNODE_VERIFY_DEFAULT, NULL, &verify_kind) == 0) &&
1601 verify_block_size) {
1602 if (verify_block_size != PAGE_SIZE) {
1603 verify_block_size = 0;
1604 }
1605 if (real_bp && verify_block_size) {
1606 panic("%s(): Called with real buffer and needs verification ",
1607 __FUNCTION__);
1608 }
1609 /*
1610 * For reads, only allow cluster verification if f_offset
1611 * and upl_offset are both page aligned. Additionally, for direct reads,
1612 * require that the length of the write also be page aligned.
1613 * If they are not page aligned, leave it to the filesystem to do verification.
1614 * Strictly speaking, the alignments need to be for verify_block_size
1615 * but since the only verify_block_size that is currently supported
1616 * is page size, we check against page alignment.
1617 */
1618 if (verify_block_size && !(flags & CL_PAGEIN) &&
1619 ((f_offset & PAGE_MASK) || (upl_offset & PAGE_MASK) ||
1620 ((flags & CL_DIRECT_IO) && (non_rounded_size & PAGE_MASK)))) {
1621 verify_block_size = 0;
1622 verify_kind = VK_HASH_NONE;
1623 }
1624 if (verify_block_size && verify_kind && !upl_has_fs_verify_info(upl)) {
1625 upl_set_fs_verify_info(upl,
1626 (upl_adjusted_size(upl, PAGE_MASK) / mp->mnt_devblocksize) * get_num_bytes_for_verify_kind(verify_kind));
1627 }
1628 }
1629 } else {
1630 io_flags = B_WRITE;
1631 bmap_flags = VNODE_WRITE;
1632
1633 max_iosize = mp->mnt_maxwritecnt;
1634 max_vectors = mp->mnt_segwritecnt;
1635 }
1636 if (verify_block_size) {
1637 bmap_flags |= VNODE_CLUSTER_VERIFY;
1638 }
1639 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_NONE, max_iosize, max_vectors, mp->mnt_devblocksize, 0, 0);
1640
1641 /*
1642 * make sure the maximum iosize is a
1643 * multiple of the page size
1644 */
1645 max_iosize &= ~PAGE_MASK;
1646
1647 /*
1648 * Ensure the maximum iosize is sensible.
1649 */
1650 if (!max_iosize) {
1651 max_iosize = PAGE_SIZE;
1652 }
1653
1654 if (flags & CL_THROTTLE) {
1655 if (!(flags & CL_PAGEOUT) && cluster_is_throttled(vp)) {
1656 uint32_t max_throttle_size = calculate_max_throttle_size(vp);
1657
1658 if (max_iosize > max_throttle_size) {
1659 max_iosize = max_throttle_size;
1660 }
1661 async_throttle = calculate_max_throttle_cnt(vp);
1662 } else {
1663 if ((flags & CL_DEV_MEMORY)) {
1664 async_throttle = IO_SCALE(vp, VNODE_ASYNC_THROTTLE);
1665 } else {
1666 u_int max_cluster;
1667 u_int max_cluster_size;
1668 u_int scale;
1669
1670 if (vp->v_mount->mnt_minsaturationbytecount) {
1671 max_cluster_size = vp->v_mount->mnt_minsaturationbytecount;
1672
1673 scale = 1;
1674 } else {
1675 max_cluster_size = MAX_CLUSTER_SIZE(vp);
1676
1677 if (disk_conditioner_mount_is_ssd(vp->v_mount)) {
1678 scale = WRITE_THROTTLE_SSD;
1679 } else {
1680 scale = WRITE_THROTTLE;
1681 }
1682 }
1683 if (max_iosize > max_cluster_size) {
1684 max_cluster = max_cluster_size;
1685 } else {
1686 max_cluster = max_iosize;
1687 }
1688
1689 if (size < max_cluster) {
1690 max_cluster = size;
1691 }
1692
1693 if (flags & CL_CLOSE) {
1694 scale += MAX_CLUSTERS;
1695 }
1696
1697 async_throttle = min(IO_SCALE(vp, VNODE_ASYNC_THROTTLE), ((scale * max_cluster_size) / max_cluster) - 1);
1698 }
1699 }
1700 }
1701 if (flags & CL_AGE) {
1702 io_flags |= B_AGE;
1703 }
1704 if (flags & (CL_PAGEIN | CL_PAGEOUT)) {
1705 io_flags |= B_PAGEIO;
1706 }
1707 if (flags & (CL_IOSTREAMING)) {
1708 io_flags |= B_IOSTREAMING;
1709 }
1710 if (flags & CL_COMMIT) {
1711 io_flags |= B_COMMIT_UPL;
1712 }
1713 if (flags & CL_DIRECT_IO) {
1714 io_flags |= B_PHYS;
1715 }
1716 if (flags & (CL_PRESERVE | CL_KEEPCACHED)) {
1717 io_flags |= B_CACHE;
1718 }
1719 if (flags & CL_PASSIVE) {
1720 io_flags |= B_PASSIVE;
1721 }
1722 if (flags & CL_ENCRYPTED) {
1723 io_flags |= B_ENCRYPTED_IO;
1724 }
1725
1726 if (vp->v_flag & VSYSTEM) {
1727 io_flags |= B_META;
1728 }
1729
1730 if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
1731 /*
1732 * then we are going to end up
1733 * with a page that we can't complete (the file size wasn't a multiple
1734 * of PAGE_SIZE and we're trying to read to the end of the file
1735 * so we'll go ahead and zero out the portion of the page we can't
1736 * read in from the file
1737 */
1738 zero_offset = (int)(upl_offset + non_rounded_size);
1739 } else if (!ISSET(flags, CL_READ) && ISSET(flags, CL_DIRECT_IO)) {
1740 assert(ISSET(flags, CL_COMMIT));
1741
1742 // For a direct/uncached write, we need to lock pages...
1743 upl_t cached_upl = NULL;
1744 upl_page_info_t *cached_pl;
1745
1746 assert(upl_offset < PAGE_SIZE);
1747
1748 /*
1749 *
1750 * f_offset = b
1751 * upl_offset = 8K
1752 *
1753 * (cached_upl - based on f_offset alignment)
1754 * 0 a b c
1755 * <----|----|----|----|----|----|-----|---->
1756 *
1757 *
1758 * (upl - based on user buffer address alignment)
1759 * <__--|----|----|--__>
1760 *
1761 * 0 1x 2x 3x
1762 *
1763 */
1764 const off_t cached_upl_f_offset = trunc_page_64(f_offset);
1765 const int cached_upl_size = round_page_32((f_offset - cached_upl_f_offset) + non_rounded_size);
1766 int num_retries = 0;
1767
1768 /*
1769 * Create a UPL to lock the pages in the cache whilst the
1770 * write is in progress.
1771 */
1772 create_cached_upl:
1773 ubc_create_upl_kernel(vp, cached_upl_f_offset, cached_upl_size, &cached_upl,
1774 &cached_pl, UPL_SET_LITE | UPL_WILL_MODIFY, VM_KERN_MEMORY_FILE);
1775 if (cached_upl && upl_has_wired_pages(cached_upl)) {
1776 /*
1777 * Pages in this UPL would contain stale data after our direct write
1778 * (which is intended to overwrite these pages on disk). The UPL is
1779 * just holding these pages "busy" to synchronize with any other I/O
1780 * or mmap() access and we have to dump these pages when the direct
1781 * write is done.
1782 * But we can't do that for wired pages, so let's release this UPL
1783 * and fall back to the "cached" path.
1784 */
1785 // printf("******* FBDP %s:%d vp %p offset 0x%llx size 0x%llx - switching from direct to cached write\n", __FUNCTION__, __LINE__, vp, cached_upl_f_offset, (uint64_t)cached_upl_size);
1786 ubc_upl_abort_range(cached_upl, 0, cached_upl_size, UPL_ABORT_FREE_ON_EMPTY);
1787 cached_upl = NULL;
1788 cached_pl = NULL;
1789 cluster_direct_write_wired++;
1790 return ENOTSUP;
1791 }
1792
1793 /*
1794 * If we are not overwriting the first and last pages completely
1795 * we need to write them out first if they are dirty. These pages
1796 * will be discarded after the write completes so we might lose
1797 * the writes for the parts that are not overwrrtten.
1798 */
1799 bool first_page_needs_sync = false;
1800 bool last_page_needs_sync = false;
1801
1802 if (cached_upl && (cached_upl_f_offset < f_offset) && upl_dirty_page(cached_pl, 0)) {
1803 first_page_needs_sync = true;
1804 }
1805
1806 if (cached_upl && (cached_upl_f_offset + cached_upl_size) > (f_offset + non_rounded_size)) {
1807 int last_page = (cached_upl_size / PAGE_SIZE) - 1;
1808
1809 if ((last_page != 0 || !first_page_needs_sync) && upl_dirty_page(cached_pl, last_page)) {
1810 last_page_needs_sync = true;
1811 }
1812 }
1813
1814 if (first_page_needs_sync || last_page_needs_sync) {
1815 ubc_upl_abort_range(cached_upl, 0, cached_upl_size, UPL_ABORT_FREE_ON_EMPTY);
1816 cached_upl = NULL;
1817 cached_pl = NULL;
1818 if (first_page_needs_sync) {
1819 ubc_msync(vp, cached_upl_f_offset, cached_upl_f_offset + PAGE_SIZE, NULL, UBC_PUSHALL | UBC_INVALIDATE | UBC_SYNC);
1820 }
1821 if (last_page_needs_sync) {
1822 off_t cached_upl_end_offset = cached_upl_f_offset + cached_upl_size;
1823
1824 ubc_msync(vp, cached_upl_end_offset - PAGE_SIZE, cached_upl_end_offset, NULL, UBC_PUSHALL | UBC_INVALIDATE | UBC_SYNC);
1825 }
1826 if (++num_retries < 16) {
1827 goto create_cached_upl;
1828 }
1829 printf("%s : Number of retries for syncing first or last page reached %d\n", __FUNCTION__, num_retries);
1830 assertf(num_retries < 16, "%s : Number of retries for syncing first or last page reached %d\n", __FUNCTION__, num_retries);
1831 }
1832
1833 /*
1834 * Attach this UPL to the other UPL so that we can find it
1835 * later.
1836 */
1837 upl_set_associated_upl(upl, cached_upl);
1838 assertf(!cached_upl ||
1839 (upl_adjusted_offset(cached_upl, PAGE_MASK) == cached_upl_f_offset),
1840 "upl_adjusted_offset(cached_upl, PAGE_MASK) = %lld, cached_upl_f_offset = %lld",
1841 upl_adjusted_offset(cached_upl, PAGE_MASK), cached_upl_f_offset);
1842 }
1843
1844 while (size) {
1845 daddr64_t blkno;
1846 daddr64_t lblkno;
1847 size_t io_size_tmp;
1848 u_int io_size_wanted;
1849
1850 if (size > max_iosize) {
1851 io_size = max_iosize;
1852 } else {
1853 io_size = size;
1854 }
1855
1856 io_size_wanted = io_size;
1857 io_size_tmp = (size_t)io_size;
1858
1859 if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, &io_size_tmp, NULL, bmap_flags, NULL))) {
1860 break;
1861 }
1862
1863 if (io_size_tmp > io_size_wanted) {
1864 io_size = io_size_wanted;
1865 } else {
1866 io_size = (u_int)io_size_tmp;
1867 }
1868
1869 if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno)) {
1870 real_bp->b_blkno = blkno;
1871 }
1872
1873 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
1874 (int)f_offset, (int)(blkno >> 32), (int)blkno, io_size, 0);
1875
1876 if (io_size == 0) {
1877 /*
1878 * vnop_blockmap didn't return an error... however, it did
1879 * return an extent size of 0 which means we can't
1880 * make forward progress on this I/O... a hole in the
1881 * file would be returned as a blkno of -1 with a non-zero io_size
1882 * a real extent is returned with a blkno != -1 and a non-zero io_size
1883 */
1884 error = EINVAL;
1885 break;
1886 }
1887 if (!(flags & CL_READ) && blkno == -1) {
1888 off_t e_offset;
1889 int pageout_flags;
1890
1891 if (upl_get_internal_vectorupl(upl)) {
1892 panic("Vector UPLs should not take this code-path");
1893 }
1894 /*
1895 * we're writing into a 'hole'
1896 */
1897 if (flags & CL_PAGEOUT) {
1898 /*
1899 * if we got here via cluster_pageout
1900 * then just error the request and return
1901 * the 'hole' should already have been covered
1902 */
1903 error = EINVAL;
1904 break;
1905 }
1906 /*
1907 * we can get here if the cluster code happens to
1908 * pick up a page that was dirtied via mmap vs
1909 * a 'write' and the page targets a 'hole'...
1910 * i.e. the writes to the cluster were sparse
1911 * and the file was being written for the first time
1912 *
1913 * we can also get here if the filesystem supports
1914 * 'holes' that are less than PAGE_SIZE.... because
1915 * we can't know if the range in the page that covers
1916 * the 'hole' has been dirtied via an mmap or not,
1917 * we have to assume the worst and try to push the
1918 * entire page to storage.
1919 *
1920 * Try paging out the page individually before
1921 * giving up entirely and dumping it (the pageout
1922 * path will insure that the zero extent accounting
1923 * has been taken care of before we get back into cluster_io)
1924 *
1925 * go direct to vnode_pageout so that we don't have to
1926 * unbusy the page from the UPL... we used to do this
1927 * so that we could call ubc_msync, but that results
1928 * in a potential deadlock if someone else races us to acquire
1929 * that page and wins and in addition needs one of the pages
1930 * we're continuing to hold in the UPL
1931 */
1932 pageout_flags = UPL_MSYNC | UPL_VNODE_PAGER | UPL_NESTED_PAGEOUT;
1933
1934 if (!(flags & CL_ASYNC)) {
1935 pageout_flags |= UPL_IOSYNC;
1936 }
1937 if (!(flags & CL_COMMIT)) {
1938 pageout_flags |= UPL_NOCOMMIT;
1939 }
1940
1941 if (cbp_head) {
1942 buf_t prev_cbp;
1943 uint32_t bytes_in_last_page;
1944
1945 /*
1946 * first we have to wait for the the current outstanding I/Os
1947 * to complete... EOT hasn't been set yet on this transaction
1948 * so the pages won't be released
1949 */
1950 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1951
1952 bytes_in_last_page = cbp_head->b_uploffset & PAGE_MASK;
1953 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
1954 bytes_in_last_page += cbp->b_bcount;
1955 }
1956 bytes_in_last_page &= PAGE_MASK;
1957
1958 while (bytes_in_last_page) {
1959 /*
1960 * we've got a transcation that
1961 * includes the page we're about to push out through vnode_pageout...
1962 * find the bp's in the list which intersect this page and either
1963 * remove them entirely from the transaction (there could be multiple bp's), or
1964 * round it's iosize down to the page boundary (there can only be one)...
1965 *
1966 * find the last bp in the list and act on it
1967 */
1968 for (prev_cbp = cbp = cbp_head; cbp->b_trans_next; cbp = cbp->b_trans_next) {
1969 prev_cbp = cbp;
1970 }
1971
1972 if (bytes_in_last_page >= cbp->b_bcount) {
1973 /*
1974 * this buf no longer has any I/O associated with it
1975 */
1976 bytes_in_last_page -= cbp->b_bcount;
1977 cbp->b_bcount = 0;
1978
1979 free_io_buf(cbp);
1980
1981 if (cbp == cbp_head) {
1982 assert(bytes_in_last_page == 0);
1983 /*
1984 * the buf we just freed was the only buf in
1985 * this transaction... so there's no I/O to do
1986 */
1987 cbp_head = NULL;
1988 cbp_tail = NULL;
1989 } else {
1990 /*
1991 * remove the buf we just freed from
1992 * the transaction list
1993 */
1994 prev_cbp->b_trans_next = NULL;
1995 cbp_tail = prev_cbp;
1996 }
1997 } else {
1998 /*
1999 * this is the last bp that has I/O
2000 * intersecting the page of interest
2001 * only some of the I/O is in the intersection
2002 * so clip the size but keep it in the transaction list
2003 */
2004 cbp->b_bcount -= bytes_in_last_page;
2005 cbp_tail = cbp;
2006 bytes_in_last_page = 0;
2007 }
2008 }
2009 if (cbp_head) {
2010 /*
2011 * there was more to the current transaction
2012 * than just the page we are pushing out via vnode_pageout...
2013 * mark it as finished and complete it... we've already
2014 * waited for the I/Os to complete above in the call to cluster_wait_IO
2015 */
2016 cluster_EOT(cbp_head, cbp_tail, 0, 0);
2017
2018 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
2019
2020 trans_count = 0;
2021 }
2022 }
2023 if (vnode_pageout(vp, upl, (upl_offset_t)trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
2024 error = EINVAL;
2025 }
2026 e_offset = round_page_64(f_offset + 1);
2027 io_size = (u_int)(e_offset - f_offset);
2028
2029 f_offset += io_size;
2030 upl_offset += io_size;
2031
2032 if (size >= io_size) {
2033 size -= io_size;
2034 } else {
2035 size = 0;
2036 }
2037 /*
2038 * keep track of how much of the original request
2039 * that we've actually completed... non_rounded_size
2040 * may go negative due to us rounding the request
2041 * to a page size multiple (i.e. size > non_rounded_size)
2042 */
2043 non_rounded_size -= io_size;
2044
2045 if (non_rounded_size <= 0) {
2046 /*
2047 * we've transferred all of the data in the original
2048 * request, but we were unable to complete the tail
2049 * of the last page because the file didn't have
2050 * an allocation to back that portion... this is ok.
2051 */
2052 size = 0;
2053 }
2054 if (error) {
2055 if (size == 0) {
2056 flags &= ~CL_COMMIT;
2057 }
2058 break;
2059 }
2060 continue;
2061 }
2062
2063 lblkno = (daddr64_t)(f_offset / CLUSTER_IO_BLOCK_SIZE);
2064
2065 /*
2066 * we have now figured out how much I/O we can do - this is in 'io_size'
2067 * pg_offset is the starting point in the first page for the I/O
2068 * pg_count is the number of full and partial pages that 'io_size' encompasses
2069 */
2070 pg_offset = upl_offset & PAGE_MASK;
2071
2072 if (flags & CL_DEV_MEMORY) {
2073 /*
2074 * treat physical requests as one 'giant' page
2075 */
2076 pg_count = 1;
2077 } else {
2078 pg_count = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
2079 }
2080
2081 if ((flags & CL_READ) && blkno == -1) {
2082 vm_offset_t commit_offset;
2083 int bytes_to_zero;
2084 int complete_transaction_now = 0;
2085
2086 /*
2087 * if we're reading and blkno == -1, then we've got a
2088 * 'hole' in the file that we need to deal with by zeroing
2089 * out the affected area in the upl
2090 */
2091 if (io_size >= (u_int)non_rounded_size) {
2092 /*
2093 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
2094 * than 'zero_offset' will be non-zero
2095 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
2096 * (indicated by the io_size finishing off the I/O request for this UPL)
2097 * than we're not going to issue an I/O for the
2098 * last page in this upl... we need to zero both the hole and the tail
2099 * of the page beyond the EOF, since the delayed zero-fill won't kick in
2100 */
2101 bytes_to_zero = non_rounded_size;
2102 if (!(flags & CL_NOZERO)) {
2103 bytes_to_zero = (int)((((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset);
2104 }
2105
2106 zero_offset = 0;
2107 } else {
2108 bytes_to_zero = io_size;
2109 }
2110
2111 pg_count = 0;
2112
2113 cluster_zero(upl, (upl_offset_t)upl_offset, bytes_to_zero, real_bp);
2114
2115 if (cbp_head) {
2116 int pg_resid;
2117
2118 /*
2119 * if there is a current I/O chain pending
2120 * then the first page of the group we just zero'd
2121 * will be handled by the I/O completion if the zero
2122 * fill started in the middle of the page
2123 */
2124 commit_offset = (upl_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2125
2126 pg_resid = (int)(commit_offset - upl_offset);
2127
2128 if (bytes_to_zero >= pg_resid) {
2129 /*
2130 * the last page of the current I/O
2131 * has been completed...
2132 * compute the number of fully zero'd
2133 * pages that are beyond it
2134 * plus the last page if its partial
2135 * and we have no more I/O to issue...
2136 * otherwise a partial page is left
2137 * to begin the next I/O
2138 */
2139 if ((int)io_size >= non_rounded_size) {
2140 pg_count = (bytes_to_zero - pg_resid + (PAGE_SIZE - 1)) / PAGE_SIZE;
2141 } else {
2142 pg_count = (bytes_to_zero - pg_resid) / PAGE_SIZE;
2143 }
2144
2145 complete_transaction_now = 1;
2146 }
2147 } else {
2148 /*
2149 * no pending I/O to deal with
2150 * so, commit all of the fully zero'd pages
2151 * plus the last page if its partial
2152 * and we have no more I/O to issue...
2153 * otherwise a partial page is left
2154 * to begin the next I/O
2155 */
2156 if ((int)io_size >= non_rounded_size) {
2157 pg_count = (pg_offset + bytes_to_zero + (PAGE_SIZE - 1)) / PAGE_SIZE;
2158 } else {
2159 pg_count = (pg_offset + bytes_to_zero) / PAGE_SIZE;
2160 }
2161
2162 commit_offset = upl_offset & ~PAGE_MASK;
2163 }
2164
2165 // Associated UPL is currently only used in the direct write path
2166 assert(!upl_associated_upl(upl));
2167
2168 if ((flags & CL_COMMIT) && pg_count) {
2169 ubc_upl_commit_range(upl, (upl_offset_t)commit_offset,
2170 pg_count * PAGE_SIZE,
2171 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
2172 }
2173 upl_offset += io_size;
2174 f_offset += io_size;
2175 size -= io_size;
2176
2177 /*
2178 * keep track of how much of the original request
2179 * that we've actually completed... non_rounded_size
2180 * may go negative due to us rounding the request
2181 * to a page size multiple (i.e. size > non_rounded_size)
2182 */
2183 non_rounded_size -= io_size;
2184
2185 if (non_rounded_size <= 0) {
2186 /*
2187 * we've transferred all of the data in the original
2188 * request, but we were unable to complete the tail
2189 * of the last page because the file didn't have
2190 * an allocation to back that portion... this is ok.
2191 */
2192 size = 0;
2193 }
2194 if (cbp_head && (complete_transaction_now || size == 0)) {
2195 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
2196
2197 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0, verify_block_size);
2198
2199 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
2200
2201 trans_count = 0;
2202 }
2203 continue;
2204 }
2205 if (pg_count > max_vectors) {
2206 if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
2207 io_size = PAGE_SIZE - pg_offset;
2208 pg_count = 1;
2209 } else {
2210 io_size -= (pg_count - max_vectors) * PAGE_SIZE;
2211 pg_count = max_vectors;
2212 }
2213 }
2214 /*
2215 * If the transaction is going to reach the maximum number of
2216 * desired elements, truncate the i/o to the nearest page so
2217 * that the actual i/o is initiated after this buffer is
2218 * created and added to the i/o chain.
2219 *
2220 * I/O directed to physically contiguous memory
2221 * doesn't have a requirement to make sure we 'fill' a page
2222 */
2223 if (!(flags & CL_DEV_MEMORY) && trans_count >= max_trans_count &&
2224 ((upl_offset + io_size) & PAGE_MASK)) {
2225 vm_offset_t aligned_ofs;
2226
2227 aligned_ofs = (upl_offset + io_size) & ~PAGE_MASK;
2228 /*
2229 * If the io_size does not actually finish off even a
2230 * single page we have to keep adding buffers to the
2231 * transaction despite having reached the desired limit.
2232 *
2233 * Eventually we get here with the page being finished
2234 * off (and exceeded) and then we truncate the size of
2235 * this i/o request so that it is page aligned so that
2236 * we can finally issue the i/o on the transaction.
2237 */
2238 if (aligned_ofs > upl_offset) {
2239 io_size = (u_int)(aligned_ofs - upl_offset);
2240 pg_count--;
2241 }
2242 }
2243
2244 if (!(mp->mnt_kern_flag & MNTK_VIRTUALDEV)) {
2245 /*
2246 * if we're not targeting a virtual device i.e. a disk image
2247 * it's safe to dip into the reserve pool since real devices
2248 * can complete this I/O request without requiring additional
2249 * bufs from the alloc_io_buf pool
2250 */
2251 priv = 1;
2252 } else if ((flags & CL_ASYNC) && !(flags & CL_PAGEOUT) && !cbp_head) {
2253 /*
2254 * Throttle the speculative IO
2255 *
2256 * We can only throttle this if it is the first iobuf
2257 * for the transaction. alloc_io_buf implements
2258 * additional restrictions for diskimages anyway.
2259 */
2260 priv = 0;
2261 } else {
2262 priv = 1;
2263 }
2264
2265 cbp = alloc_io_buf(vp, priv);
2266
2267 if (flags & CL_PAGEOUT) {
2268 u_int i;
2269
2270 /*
2271 * since blocks are in offsets of CLUSTER_IO_BLOCK_SIZE, scale
2272 * iteration to (PAGE_SIZE * pg_count) of blks.
2273 */
2274 for (i = 0; i < (PAGE_SIZE * pg_count) / CLUSTER_IO_BLOCK_SIZE; i++) {
2275 if (buf_invalblkno(vp, lblkno + i, 0) == EBUSY) {
2276 panic("BUSY bp found in cluster_io");
2277 }
2278 }
2279 }
2280 if (flags & CL_ASYNC) {
2281 if (buf_setcallback(cbp, (void *)cluster_iodone, callback_arg)) {
2282 panic("buf_setcallback failed");
2283 }
2284 }
2285 cbp->b_cliodone = (void *)callback;
2286 cbp->b_flags |= io_flags;
2287 if (flags & CL_NOCACHE) {
2288 cbp->b_attr.ba_flags |= BA_NOCACHE;
2289 }
2290 if (verify_block_size) {
2291 cbp->b_attr.ba_flags |= BA_WILL_VERIFY;
2292 if (verify_kind) {
2293 cbp->b_attr.ba_verify_type = verify_kind;
2294 }
2295 }
2296
2297 cbp->b_lblkno = lblkno;
2298 cbp->b_clfoffset = f_offset;
2299 cbp->b_blkno = blkno;
2300 cbp->b_bcount = io_size;
2301
2302 if (buf_setupl(cbp, upl, (uint32_t)upl_offset)) {
2303 panic("buf_setupl failed");
2304 }
2305 #if CONFIG_IOSCHED
2306 upl_set_blkno(upl, upl_offset, io_size, blkno);
2307 #endif
2308 cbp->b_trans_next = (buf_t)NULL;
2309
2310 if ((cbp->b_iostate = (void *)iostate)) {
2311 /*
2312 * caller wants to track the state of this
2313 * io... bump the amount issued against this stream
2314 */
2315 iostate->io_issued += io_size;
2316 }
2317
2318 if (flags & CL_READ) {
2319 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 26)) | DBG_FUNC_NONE,
2320 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
2321 } else {
2322 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 27)) | DBG_FUNC_NONE,
2323 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
2324 }
2325
2326 if (cbp_head) {
2327 cbp_tail->b_trans_next = cbp;
2328 cbp_tail = cbp;
2329 } else {
2330 cbp_head = cbp;
2331 cbp_tail = cbp;
2332
2333 if ((cbp_head->b_real_bp = real_bp)) {
2334 real_bp = (buf_t)NULL;
2335 }
2336 }
2337 *(buf_t *)(&cbp->b_trans_head) = cbp_head;
2338
2339 trans_count++;
2340
2341 upl_offset += io_size;
2342 f_offset += io_size;
2343 size -= io_size;
2344 /*
2345 * keep track of how much of the original request
2346 * that we've actually completed... non_rounded_size
2347 * may go negative due to us rounding the request
2348 * to a page size multiple (i.e. size > non_rounded_size)
2349 */
2350 non_rounded_size -= io_size;
2351
2352 if (non_rounded_size <= 0) {
2353 /*
2354 * we've transferred all of the data in the original
2355 * request, but we were unable to complete the tail
2356 * of the last page because the file didn't have
2357 * an allocation to back that portion... this is ok.
2358 */
2359 size = 0;
2360 }
2361 if (size == 0) {
2362 /*
2363 * we have no more I/O to issue, so go
2364 * finish the final transaction
2365 */
2366 need_EOT = TRUE;
2367 } else if (((flags & CL_DEV_MEMORY) || (upl_offset & PAGE_MASK) == 0) &&
2368 ((flags & CL_ASYNC) || trans_count > max_trans_count)) {
2369 /*
2370 * I/O directed to physically contiguous memory...
2371 * which doesn't have a requirement to make sure we 'fill' a page
2372 * or...
2373 * the current I/O we've prepared fully
2374 * completes the last page in this request
2375 * and ...
2376 * it's either an ASYNC request or
2377 * we've already accumulated more than 8 I/O's into
2378 * this transaction so mark it as complete so that
2379 * it can finish asynchronously or via the cluster_complete_transaction
2380 * below if the request is synchronous
2381 */
2382 need_EOT = TRUE;
2383 }
2384 if (need_EOT == TRUE) {
2385 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0, verify_block_size);
2386 }
2387
2388 if (flags & CL_THROTTLE) {
2389 (void)vnode_waitforwrites(vp, async_throttle, 0, 0, "cluster_io");
2390 }
2391
2392 if (!(io_flags & B_READ)) {
2393 vnode_startwrite(vp);
2394 }
2395
2396 if (flags & CL_RAW_ENCRYPTED) {
2397 /*
2398 * User requested raw encrypted bytes.
2399 * Twiddle the bit in the ba_flags for the buffer
2400 */
2401 cbp->b_attr.ba_flags |= BA_RAW_ENCRYPTED_IO;
2402 }
2403
2404 (void) VNOP_STRATEGY(cbp);
2405
2406 if (need_EOT == TRUE) {
2407 if (!(flags & CL_ASYNC)) {
2408 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 1);
2409 }
2410
2411 need_EOT = FALSE;
2412 trans_count = 0;
2413 cbp_head = NULL;
2414 }
2415 }
2416 if (error) {
2417 int abort_size;
2418
2419 io_size = 0;
2420
2421 if (cbp_head) {
2422 /*
2423 * Wait until all of the outstanding I/O
2424 * for this partial transaction has completed
2425 */
2426 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
2427
2428 /*
2429 * Rewind the upl offset to the beginning of the
2430 * transaction.
2431 */
2432 upl_offset = cbp_head->b_uploffset;
2433 }
2434
2435 if (ISSET(flags, CL_COMMIT)) {
2436 cluster_handle_associated_upl(iostate, upl,
2437 (upl_offset_t)upl_offset,
2438 (upl_size_t)(upl_end_offset - upl_offset),
2439 cbp_head ? cbp_head->b_clfoffset : f_offset);
2440 }
2441
2442 // Free all the IO buffers in this transaction
2443 for (cbp = cbp_head; cbp;) {
2444 buf_t cbp_next;
2445
2446 size += cbp->b_bcount;
2447 io_size += cbp->b_bcount;
2448
2449 cbp_next = cbp->b_trans_next;
2450 free_io_buf(cbp);
2451 cbp = cbp_next;
2452 }
2453
2454 if (iostate) {
2455 int need_wakeup = 0;
2456
2457 /*
2458 * update the error condition for this stream
2459 * since we never really issued the io
2460 * just go ahead and adjust it back
2461 */
2462 lck_mtx_lock_spin(&iostate->io_mtxp);
2463
2464 if (iostate->io_error == 0) {
2465 iostate->io_error = error;
2466 }
2467 iostate->io_issued -= io_size;
2468
2469 if (iostate->io_wanted) {
2470 /*
2471 * someone is waiting for the state of
2472 * this io stream to change
2473 */
2474 iostate->io_wanted = 0;
2475 need_wakeup = 1;
2476 }
2477 lck_mtx_unlock(&iostate->io_mtxp);
2478
2479 if (need_wakeup) {
2480 wakeup((caddr_t)&iostate->io_wanted);
2481 }
2482 }
2483
2484 if (flags & CL_COMMIT) {
2485 int upl_flags;
2486
2487 pg_offset = upl_offset & PAGE_MASK;
2488 abort_size = (int)((upl_end_offset - upl_offset + PAGE_MASK) & ~PAGE_MASK);
2489
2490 upl_flags = cluster_ioerror(upl, (int)(upl_offset - pg_offset),
2491 abort_size, error, io_flags, vp);
2492
2493 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
2494 upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
2495 }
2496 if (retval == 0) {
2497 retval = error;
2498 }
2499 } else if (cbp_head) {
2500 panic("%s(): cbp_head is not NULL.", __FUNCTION__);
2501 }
2502
2503 if (real_bp) {
2504 /*
2505 * can get here if we either encountered an error
2506 * or we completely zero-filled the request and
2507 * no I/O was issued
2508 */
2509 if (error) {
2510 real_bp->b_flags |= B_ERROR;
2511 real_bp->b_error = error;
2512 }
2513 buf_biodone(real_bp);
2514 }
2515 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END, (int)f_offset, size, upl_offset, retval, 0);
2516
2517 return retval;
2518 }
2519
2520 #define reset_vector_run_state() \
2521 issueVectorUPL = vector_upl_offset = vector_upl_index = vector_upl_iosize = vector_upl_size = 0;
2522
2523 static int
vector_cluster_io(vnode_t vp,upl_t vector_upl,vm_offset_t vector_upl_offset,off_t v_upl_uio_offset,int vector_upl_iosize,int io_flag,buf_t real_bp,struct clios * iostate,int (* callback)(buf_t,void *),void * callback_arg)2524 vector_cluster_io(vnode_t vp, upl_t vector_upl, vm_offset_t vector_upl_offset, off_t v_upl_uio_offset, int vector_upl_iosize,
2525 int io_flag, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
2526 {
2527 vector_upl_set_pagelist(vector_upl);
2528
2529 if (io_flag & CL_READ) {
2530 if (vector_upl_offset == 0 && ((vector_upl_iosize & PAGE_MASK) == 0)) {
2531 io_flag &= ~CL_PRESERVE; /*don't zero fill*/
2532 } else {
2533 io_flag |= CL_PRESERVE; /*zero fill*/
2534 }
2535 }
2536 return cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, real_bp, iostate, callback, callback_arg);
2537 }
2538
2539 static int
cluster_read_prefetch(vnode_t vp,off_t f_offset,u_int size,off_t filesize,int (* callback)(buf_t,void *),void * callback_arg,int bflag)2540 cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
2541 {
2542 int pages_in_prefetch;
2543
2544 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_START,
2545 (int)f_offset, size, (int)filesize, 0, 0);
2546
2547 if (f_offset >= filesize) {
2548 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
2549 (int)f_offset, 0, 0, 0, 0);
2550 return 0;
2551 }
2552 if ((off_t)size > (filesize - f_offset)) {
2553 size = (u_int)(filesize - f_offset);
2554 }
2555 pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
2556
2557 advisory_read_ext(vp, filesize, f_offset, size, callback, callback_arg, bflag);
2558
2559 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
2560 (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
2561
2562 return pages_in_prefetch;
2563 }
2564
2565
2566
2567 static void
cluster_read_ahead(vnode_t vp,struct cl_extent * extent,off_t filesize,struct cl_readahead * rap,int (* callback)(buf_t,void *),void * callback_arg,int bflag)2568 cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap, int (*callback)(buf_t, void *), void *callback_arg,
2569 int bflag)
2570 {
2571 daddr64_t r_addr;
2572 off_t f_offset;
2573 int size_of_prefetch;
2574 u_int max_prefetch;
2575
2576
2577 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_START,
2578 (int)extent->b_addr, (int)extent->e_addr, (int)rap->cl_lastr, 0, 0);
2579
2580 if (extent->b_addr == rap->cl_lastr && extent->b_addr == extent->e_addr) {
2581 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2582 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
2583 return;
2584 }
2585 if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1))) {
2586 rap->cl_ralen = 0;
2587 rap->cl_maxra = 0;
2588
2589 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2590 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 1, 0);
2591
2592 return;
2593 }
2594
2595 max_prefetch = cluster_max_prefetch(vp,
2596 cluster_max_io_size(vp->v_mount, CL_READ), speculative_prefetch_max);
2597
2598 if (max_prefetch <= PAGE_SIZE) {
2599 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2600 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 6, 0);
2601 return;
2602 }
2603 if (extent->e_addr < rap->cl_maxra && rap->cl_ralen >= 4) {
2604 if ((rap->cl_maxra - extent->e_addr) > (rap->cl_ralen / 4)) {
2605 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2606 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
2607 return;
2608 }
2609 }
2610 r_addr = MAX(extent->e_addr, rap->cl_maxra) + 1;
2611 f_offset = (off_t)(r_addr * PAGE_SIZE_64);
2612
2613 size_of_prefetch = 0;
2614
2615 ubc_range_op(vp, f_offset, f_offset + PAGE_SIZE_64, UPL_ROP_PRESENT, &size_of_prefetch);
2616
2617 if (size_of_prefetch) {
2618 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2619 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 3, 0);
2620 return;
2621 }
2622 if (f_offset < filesize) {
2623 daddr64_t read_size;
2624
2625 rap->cl_ralen = rap->cl_ralen ? min(max_prefetch / PAGE_SIZE, rap->cl_ralen << 1) : 1;
2626
2627 read_size = (extent->e_addr + 1) - extent->b_addr;
2628
2629 if (read_size > rap->cl_ralen) {
2630 if (read_size > max_prefetch / PAGE_SIZE) {
2631 rap->cl_ralen = max_prefetch / PAGE_SIZE;
2632 } else {
2633 rap->cl_ralen = (int)read_size;
2634 }
2635 }
2636 size_of_prefetch = cluster_read_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize, callback, callback_arg, bflag);
2637
2638 if (size_of_prefetch) {
2639 rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
2640 }
2641 }
2642 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
2643 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
2644 }
2645
2646
2647 int
cluster_pageout(vnode_t vp,upl_t upl,upl_offset_t upl_offset,off_t f_offset,int size,off_t filesize,int flags)2648 cluster_pageout(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
2649 int size, off_t filesize, int flags)
2650 {
2651 return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
2652 }
2653
2654
2655 int
cluster_pageout_ext(vnode_t vp,upl_t upl,upl_offset_t upl_offset,off_t f_offset,int size,off_t filesize,int flags,int (* callback)(buf_t,void *),void * callback_arg)2656 cluster_pageout_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
2657 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
2658 {
2659 int io_size;
2660 int rounded_size;
2661 off_t max_size;
2662 int local_flags;
2663
2664 local_flags = CL_PAGEOUT | CL_THROTTLE;
2665
2666 if ((flags & UPL_IOSYNC) == 0) {
2667 local_flags |= CL_ASYNC;
2668 }
2669 if ((flags & UPL_NOCOMMIT) == 0) {
2670 local_flags |= CL_COMMIT;
2671 }
2672 if ((flags & UPL_KEEPCACHED)) {
2673 local_flags |= CL_KEEPCACHED;
2674 }
2675 if (flags & UPL_PAGING_ENCRYPTED) {
2676 local_flags |= CL_ENCRYPTED;
2677 }
2678
2679
2680 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
2681 (int)f_offset, size, (int)filesize, local_flags, 0);
2682
2683 /*
2684 * If they didn't specify any I/O, then we are done...
2685 * we can't issue an abort because we don't know how
2686 * big the upl really is
2687 */
2688 if (size <= 0) {
2689 return EINVAL;
2690 }
2691
2692 if (vp->v_mount->mnt_flag & MNT_RDONLY) {
2693 if (local_flags & CL_COMMIT) {
2694 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
2695 }
2696 return EROFS;
2697 }
2698 /*
2699 * can't page-in from a negative offset
2700 * or if we're starting beyond the EOF
2701 * or if the file offset isn't page aligned
2702 * or the size requested isn't a multiple of PAGE_SIZE
2703 */
2704 if (f_offset < 0 || f_offset >= filesize ||
2705 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK)) {
2706 if (local_flags & CL_COMMIT) {
2707 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
2708 }
2709 return EINVAL;
2710 }
2711 max_size = filesize - f_offset;
2712
2713 if (size < max_size) {
2714 io_size = size;
2715 } else {
2716 io_size = (int)max_size;
2717 }
2718
2719 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2720
2721 if (size > rounded_size) {
2722 if (local_flags & CL_COMMIT) {
2723 ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
2724 UPL_ABORT_FREE_ON_EMPTY);
2725 }
2726 }
2727 return cluster_io(vp, upl, upl_offset, f_offset, io_size,
2728 local_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2729 }
2730
2731
2732 int
cluster_pagein(vnode_t vp,upl_t upl,upl_offset_t upl_offset,off_t f_offset,int size,off_t filesize,int flags)2733 cluster_pagein(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
2734 int size, off_t filesize, int flags)
2735 {
2736 return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
2737 }
2738
2739 #define SPLIT_PAGEIN_MAX_IOSIZE 32768
2740
2741 /*
2742 * Do a big pagein request as multiple I/Os - the first I/O will be for
2743 * SPLIT_PAGEIN_MAX_IOSIZE (32K)sized which includes the page that the caused
2744 * the fault and then i/o will be initiated for the remaining.
2745 */
2746 static int
cluster_handle_split_pagein(vnode_t vp,upl_t upl,upl_offset_t upl_offset,off_t f_offset,u_int io_size,int rounded_size,int local_flags,int (* callback)(buf_t,void *),void * callback_arg)2747 cluster_handle_split_pagein(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
2748 u_int io_size, int rounded_size, int local_flags, int (*callback)(buf_t, void *), void *callback_arg)
2749 {
2750 upl_page_info_t *pl = ubc_upl_pageinfo(upl);
2751 const off_t start_f_offset = f_offset;
2752 const upl_offset_t start_upl_offset = upl_offset;
2753 const int start_pg = upl_offset >> PAGE_SHIFT;
2754 const int last_pg = ((upl_offset + rounded_size) >> PAGE_SHIFT) - 1;
2755 u_int split_io_size = SPLIT_PAGEIN_MAX_IOSIZE;
2756 u_int head_io_size = 0;
2757 int retval = 0;
2758 int error = 0;
2759 int pg;
2760
2761 assert(SPLIT_PAGEIN_MAX_IOSIZE >= (2 * PAGE_SIZE));
2762
2763 for (pg = start_pg; (pg <= last_pg) && !(upl_page_is_needed(pl, pg)); pg++) {
2764 ;
2765 }
2766
2767 /*
2768 * The global variables affecting behaviour
2769 * split_all_pgin -> Split pageins even if we don't find the needed page.
2770 * split_pgin_headio -> for a pagein in which there is a head calculated,
2771 * do the head i/o or not.
2772 *
2773 * split_all_pgin_equal -> split the entire bug request into equal sized small i/os of 32K.
2774 *
2775 * Whichever way the i/o is split, the i/o for the needed page always happens first and then we decide
2776 * whether we have to do i/o for the head and then if we need to issue equal sized i/o.
2777 *
2778 * By default we are set up to do only the i/o for the needed page, followed by a "unsplit" tail.
2779 */
2780 if ((pg > start_pg) && (pg <= last_pg)) {
2781 head_io_size = ((pg - start_pg) * PAGE_SIZE);
2782
2783 if (head_io_size < SPLIT_PAGEIN_MAX_IOSIZE) {
2784 head_io_size = 0;
2785 } else if (!split_all_pgin) {
2786 goto out;
2787 } else if ((rounded_size - head_io_size) <= SPLIT_PAGEIN_MAX_IOSIZE) {
2788 head_io_size = (rounded_size - SPLIT_PAGEIN_MAX_IOSIZE);
2789 } else {
2790 head_io_size &= ~(SPLIT_PAGEIN_MAX_IOSIZE - 1);
2791 }
2792
2793 assertf(io_size > head_io_size, "io_size is %d, head_io_size = %d", io_size, head_io_size);
2794
2795 if (head_io_size) {
2796 upl_offset += head_io_size;
2797 f_offset += head_io_size;
2798 io_size -= head_io_size;
2799
2800 if (!split_pgin_headio) {
2801 if (local_flags & CL_COMMIT) {
2802 ubc_upl_abort_range(upl, start_upl_offset, head_io_size,
2803 UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
2804 }
2805 head_io_size = 0;
2806 }
2807
2808 split_io_size = MIN(SPLIT_PAGEIN_MAX_IOSIZE, io_size);
2809 }
2810
2811 assertf(io_size >= split_io_size, "io_size is %d, split_io_size = %d", io_size, split_io_size);
2812 } else if ((pg > last_pg) && !split_all_pgin) {
2813 goto out;
2814 }
2815
2816 /* This is the 32K i/o for the "needed" page */
2817 retval = cluster_io(vp, upl, upl_offset, f_offset, split_io_size,
2818 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2819
2820 io_size -= split_io_size;
2821
2822 if (io_size) {
2823 upl_offset += split_io_size;
2824 f_offset += split_io_size;
2825 } else if (head_io_size) {
2826 io_size = head_io_size;
2827 head_io_size = 0;
2828 upl_offset = start_upl_offset;
2829 f_offset = start_f_offset;
2830 }
2831
2832 while (io_size) {
2833 if (split_all_pgin_equal && (io_size > SPLIT_PAGEIN_MAX_IOSIZE)) {
2834 split_io_size = SPLIT_PAGEIN_MAX_IOSIZE;
2835 } else {
2836 split_io_size = io_size;
2837 }
2838
2839 assertf(io_size >= split_io_size, "io_size is %d, split_io_size = %d", io_size, split_io_size);
2840
2841 /* We have to issue this i/o anyway even if we get an error from any of the previous ones */
2842 error = cluster_io(vp, upl, upl_offset, f_offset, split_io_size,
2843 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2844 if (!retval) {
2845 retval = error;
2846 }
2847
2848 io_size -= split_io_size;
2849
2850 if ((io_size == 0) && head_io_size) {
2851 io_size = head_io_size;
2852 head_io_size = 0;
2853 upl_offset = start_upl_offset;
2854 f_offset = start_f_offset;
2855 } else if (io_size) {
2856 upl_offset += split_io_size;
2857 f_offset += split_io_size;
2858 }
2859 }
2860
2861 return retval;
2862 out:
2863 return cluster_io(vp, upl, upl_offset, f_offset, io_size,
2864 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2865 }
2866
2867 int
cluster_pagein_ext(vnode_t vp,upl_t upl,upl_offset_t upl_offset,off_t f_offset,int size,off_t filesize,int flags,int (* callback)(buf_t,void *),void * callback_arg)2868 cluster_pagein_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
2869 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
2870 {
2871 u_int io_size;
2872 int rounded_size;
2873 off_t max_size;
2874 int retval;
2875 int local_flags = 0;
2876
2877 if (upl == NULL || size < 0) {
2878 panic("cluster_pagein: NULL upl passed in");
2879 }
2880
2881 if ((flags & UPL_IOSYNC) == 0) {
2882 local_flags |= CL_ASYNC;
2883 }
2884 if ((flags & UPL_NOCOMMIT) == 0) {
2885 local_flags |= CL_COMMIT;
2886 }
2887 if (flags & UPL_IOSTREAMING) {
2888 local_flags |= CL_IOSTREAMING;
2889 }
2890 if (flags & UPL_PAGING_ENCRYPTED) {
2891 local_flags |= CL_ENCRYPTED;
2892 }
2893
2894
2895 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
2896 (int)f_offset, size, (int)filesize, local_flags, 0);
2897
2898 /*
2899 * can't page-in from a negative offset
2900 * or if we're starting beyond the EOF
2901 * or if the file offset isn't page aligned
2902 * or the size requested isn't a multiple of PAGE_SIZE
2903 */
2904 if (f_offset < 0 || f_offset >= filesize ||
2905 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (upl_offset & PAGE_MASK)) {
2906 if (local_flags & CL_COMMIT) {
2907 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
2908 }
2909
2910 if (f_offset >= filesize) {
2911 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_CLUSTER, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_CL_PGIN_PAST_EOF), 0 /* arg */);
2912 }
2913
2914 return EINVAL;
2915 }
2916 max_size = filesize - f_offset;
2917
2918 if (size < max_size) {
2919 io_size = size;
2920 } else {
2921 io_size = (int)max_size;
2922 }
2923
2924 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2925
2926 if (size > rounded_size && (local_flags & CL_COMMIT)) {
2927 ubc_upl_abort_range(upl, upl_offset + rounded_size,
2928 size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
2929 }
2930
2931 if ((io_size > SPLIT_PAGEIN_MAX_IOSIZE) && vnode_isonssd(vp) && split_pgin) {
2932 return cluster_handle_split_pagein(vp, upl, upl_offset, f_offset, io_size,
2933 rounded_size, local_flags, callback, callback_arg);
2934 }
2935
2936 retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
2937 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2938
2939 return retval;
2940 }
2941
2942
2943 int
cluster_bp(buf_t bp)2944 cluster_bp(buf_t bp)
2945 {
2946 return cluster_bp_ext(bp, NULL, NULL);
2947 }
2948
2949
2950 int
cluster_bp_ext(buf_t bp,int (* callback)(buf_t,void *),void * callback_arg)2951 cluster_bp_ext(buf_t bp, int (*callback)(buf_t, void *), void *callback_arg)
2952 {
2953 off_t f_offset;
2954 int flags;
2955
2956 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 19)) | DBG_FUNC_START,
2957 bp, (int)bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
2958
2959 if (bp->b_flags & B_READ) {
2960 flags = CL_ASYNC | CL_READ;
2961 } else {
2962 flags = CL_ASYNC;
2963 }
2964 if (bp->b_flags & B_PASSIVE) {
2965 flags |= CL_PASSIVE;
2966 }
2967
2968 f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
2969
2970 return cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL, callback, callback_arg);
2971 }
2972
2973
2974
2975 int
cluster_write(vnode_t vp,struct uio * uio,off_t oldEOF,off_t newEOF,off_t headOff,off_t tailOff,int xflags)2976 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
2977 {
2978 return cluster_write_ext(vp, uio, oldEOF, newEOF, headOff, tailOff, xflags, NULL, NULL);
2979 }
2980
2981
2982 int
cluster_write_ext(vnode_t vp,struct uio * uio,off_t oldEOF,off_t newEOF,off_t headOff,off_t tailOff,int xflags,int (* callback)(buf_t,void *),void * callback_arg)2983 cluster_write_ext(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff,
2984 int xflags, int (*callback)(buf_t, void *), void *callback_arg)
2985 {
2986 user_ssize_t cur_resid;
2987 int retval = 0;
2988 int flags;
2989 int zflags;
2990 int bflag;
2991 int write_type = IO_COPY;
2992 u_int32_t write_length = 0, saved_write_length;
2993 uint32_t min_direct_size = MIN_DIRECT_WRITE_SIZE;
2994
2995 flags = xflags;
2996
2997 if (flags & IO_PASSIVE) {
2998 bflag = CL_PASSIVE;
2999 } else {
3000 bflag = 0;
3001 }
3002
3003 if (vp->v_flag & VNOCACHE_DATA) {
3004 flags |= IO_NOCACHE;
3005 bflag |= CL_NOCACHE;
3006 }
3007 if (uio == NULL) {
3008 /*
3009 * no user data...
3010 * this call is being made to zero-fill some range in the file
3011 */
3012 retval = cluster_write_copy(vp, NULL, (u_int32_t)0, oldEOF, newEOF, headOff, tailOff, flags, callback, callback_arg);
3013
3014 return retval;
3015 }
3016 /*
3017 * do a write through the cache if one of the following is true....
3018 * NOCACHE is not true or NODIRECT is true
3019 * the uio request doesn't target USERSPACE
3020 * otherwise, find out if we want the direct or contig variant for
3021 * the first vector in the uio request
3022 */
3023 if (((flags & (IO_NOCACHE | IO_NODIRECT)) == IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
3024 if (flags & IO_NOCACHE_SWRITE) {
3025 uint32_t fs_bsize = vp->v_mount->mnt_vfsstat.f_bsize;
3026
3027 if (fs_bsize && (fs_bsize < MIN_DIRECT_WRITE_SIZE) &&
3028 ((fs_bsize & (fs_bsize - 1)) == 0)) {
3029 min_direct_size = fs_bsize;
3030 }
3031 }
3032 retval = cluster_io_type(uio, &write_type, &write_length, min_direct_size);
3033 }
3034
3035 if ((flags & (IO_TAILZEROFILL | IO_HEADZEROFILL)) && write_type == IO_DIRECT) {
3036 /*
3037 * must go through the cached variant in this case
3038 */
3039 write_type = IO_COPY;
3040 }
3041
3042 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < newEOF && retval == 0) {
3043 switch (write_type) {
3044 case IO_COPY:
3045 /*
3046 * make sure the uio_resid isn't too big...
3047 * internally, we want to handle all of the I/O in
3048 * chunk sizes that fit in a 32 bit int
3049 */
3050 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
3051 /*
3052 * we're going to have to call cluster_write_copy
3053 * more than once...
3054 *
3055 * only want the last call to cluster_write_copy to
3056 * have the IO_TAILZEROFILL flag set and only the
3057 * first call should have IO_HEADZEROFILL
3058 */
3059 zflags = flags & ~IO_TAILZEROFILL;
3060 flags &= ~IO_HEADZEROFILL;
3061
3062 write_length = MAX_IO_REQUEST_SIZE;
3063 } else {
3064 /*
3065 * last call to cluster_write_copy
3066 */
3067 zflags = flags;
3068
3069 write_length = (u_int32_t)cur_resid;
3070 }
3071 retval = cluster_write_copy(vp, uio, write_length, oldEOF, newEOF, headOff, tailOff, zflags, callback, callback_arg);
3072 break;
3073
3074 case IO_CONTIG:
3075 zflags = flags & ~(IO_TAILZEROFILL | IO_HEADZEROFILL);
3076
3077 if (flags & IO_HEADZEROFILL) {
3078 /*
3079 * only do this once per request
3080 */
3081 flags &= ~IO_HEADZEROFILL;
3082
3083 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, uio->uio_offset,
3084 headOff, (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
3085 if (retval) {
3086 break;
3087 }
3088 }
3089 retval = cluster_write_contig(vp, uio, newEOF, &write_type, &write_length, callback, callback_arg, bflag);
3090
3091 if (retval == 0 && (flags & IO_TAILZEROFILL) && uio_resid(uio) == 0) {
3092 /*
3093 * we're done with the data from the user specified buffer(s)
3094 * and we've been requested to zero fill at the tail
3095 * treat this as an IO_HEADZEROFILL which doesn't require a uio
3096 * by rearranging the args and passing in IO_HEADZEROFILL
3097 */
3098
3099 /*
3100 * Update the oldEOF to reflect the current EOF. If the UPL page
3101 * to zero-fill is not valid (when F_NOCACHE is set), the
3102 * cluster_write_copy() will perform RMW on the UPL page when
3103 * the oldEOF is not aligned on page boundary due to unaligned
3104 * write.
3105 */
3106 if (uio->uio_offset > oldEOF) {
3107 oldEOF = uio->uio_offset;
3108 }
3109 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)oldEOF, tailOff, uio->uio_offset,
3110 (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
3111 }
3112 break;
3113
3114 case IO_DIRECT:
3115 /*
3116 * cluster_write_direct is never called with IO_TAILZEROFILL || IO_HEADZEROFILL
3117 */
3118 saved_write_length = write_length;
3119 retval = cluster_write_direct(vp, uio, oldEOF, newEOF, &write_type, &write_length, flags, callback, callback_arg, min_direct_size);
3120 if (retval == ENOTSUP) {
3121 /* direct I/O didn't work; retry with cached I/O */
3122 // printf("******* FBDP %s:%d ENOTSUP cnt %d resid 0x%llx offset 0x%llx write_length 0x%x -> 0x%x\n", __FUNCTION__, __LINE__, uio_iovcnt(uio), (uint64_t) uio_resid(uio), uio_offset(uio), write_length, saved_write_length);
3123 write_length = saved_write_length;
3124 write_type = IO_COPY;
3125 retval = 0;
3126 }
3127 break;
3128
3129 case IO_UNKNOWN:
3130 retval = cluster_io_type(uio, &write_type, &write_length, min_direct_size);
3131 break;
3132 }
3133 /*
3134 * in case we end up calling cluster_write_copy (from cluster_write_direct)
3135 * multiple times to service a multi-vector request that is not aligned properly
3136 * we need to update the oldEOF so that we
3137 * don't zero-fill the head of a page if we've successfully written
3138 * data to that area... 'cluster_write_copy' will zero-fill the head of a
3139 * page that is beyond the oldEOF if the write is unaligned... we only
3140 * want that to happen for the very first page of the cluster_write,
3141 * NOT the first page of each vector making up a multi-vector write.
3142 */
3143 if (uio->uio_offset > oldEOF) {
3144 oldEOF = uio->uio_offset;
3145 }
3146 }
3147 return retval;
3148 }
3149
3150
3151 static int
cluster_write_direct(vnode_t vp,struct uio * uio,off_t oldEOF,off_t newEOF,int * write_type,u_int32_t * write_length,int flags,int (* callback)(buf_t,void *),void * callback_arg,uint32_t min_io_size)3152 cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
3153 int flags, int (*callback)(buf_t, void *), void *callback_arg, uint32_t min_io_size)
3154 {
3155 upl_t upl = NULL;
3156 upl_page_info_t *pl;
3157 vm_offset_t upl_offset;
3158 vm_offset_t vector_upl_offset = 0;
3159 u_int32_t io_req_size;
3160 u_int32_t offset_in_file;
3161 u_int32_t offset_in_iovbase;
3162 u_int32_t io_size;
3163 int io_flag = 0;
3164 upl_size_t upl_size = 0, vector_upl_size = 0;
3165 vm_size_t upl_needed_size;
3166 mach_msg_type_number_t pages_in_pl = 0;
3167 upl_control_flags_t upl_flags;
3168 kern_return_t kret = KERN_SUCCESS;
3169 mach_msg_type_number_t i = 0;
3170 int force_data_sync;
3171 int retval = 0;
3172 int first_IO = 1;
3173 struct clios iostate;
3174 user_addr_t iov_base;
3175 u_int32_t mem_alignment_mask;
3176 u_int32_t devblocksize;
3177 u_int32_t max_io_size;
3178 u_int32_t max_upl_size;
3179 u_int32_t max_vector_size;
3180 u_int32_t bytes_outstanding_limit;
3181 boolean_t io_throttled = FALSE;
3182
3183 u_int32_t vector_upl_iosize = 0;
3184 int issueVectorUPL = 0, useVectorUPL = (uio->uio_iovcnt > 1);
3185 off_t v_upl_uio_offset = 0;
3186 int vector_upl_index = 0;
3187 upl_t vector_upl = NULL;
3188 uio_t snapshot_uio = NULL;
3189
3190 uint32_t io_align_mask;
3191
3192 /*
3193 * When we enter this routine, we know
3194 * -- the resid will not exceed iov_len
3195 */
3196 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
3197 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
3198
3199 assert(vm_map_page_shift(current_map()) >= PAGE_SHIFT);
3200
3201 max_upl_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
3202
3203 io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE | CL_DIRECT_IO;
3204
3205 if (flags & IO_PASSIVE) {
3206 io_flag |= CL_PASSIVE;
3207 }
3208
3209 if (flags & IO_NOCACHE) {
3210 io_flag |= CL_NOCACHE;
3211 }
3212
3213 if (flags & IO_SKIP_ENCRYPTION) {
3214 io_flag |= CL_ENCRYPTED;
3215 }
3216
3217 iostate.io_completed = 0;
3218 iostate.io_issued = 0;
3219 iostate.io_error = 0;
3220 iostate.io_wanted = 0;
3221
3222 lck_mtx_init(&iostate.io_mtxp, &cl_mtx_grp, LCK_ATTR_NULL);
3223
3224 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3225 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3226
3227 if (devblocksize == 1) {
3228 /*
3229 * the AFP client advertises a devblocksize of 1
3230 * however, its BLOCKMAP routine maps to physical
3231 * blocks that are PAGE_SIZE in size...
3232 * therefore we can't ask for I/Os that aren't page aligned
3233 * or aren't multiples of PAGE_SIZE in size
3234 * by setting devblocksize to PAGE_SIZE, we re-instate
3235 * the old behavior we had before the mem_alignment_mask
3236 * changes went in...
3237 */
3238 devblocksize = PAGE_SIZE;
3239 }
3240
3241 io_align_mask = PAGE_MASK;
3242 if (min_io_size < MIN_DIRECT_WRITE_SIZE) {
3243 /* The process has opted into fs blocksize direct io writes */
3244 assert((min_io_size & (min_io_size - 1)) == 0);
3245 io_align_mask = min_io_size - 1;
3246 io_flag |= CL_DIRECT_IO_FSBLKSZ;
3247 }
3248
3249 if (uio_iovcnt(uio) > 1) {
3250 /* vector uio -> take a snapshot so we can rollback if needed */
3251 if (snapshot_uio) {
3252 uio_free(snapshot_uio);
3253 snapshot_uio = NULL;
3254 }
3255 snapshot_uio = uio_duplicate(uio);
3256 }
3257
3258 next_dwrite:
3259 io_req_size = *write_length;
3260 iov_base = uio_curriovbase(uio);
3261
3262 offset_in_file = (u_int32_t)(uio->uio_offset & io_align_mask);
3263 offset_in_iovbase = (u_int32_t)(iov_base & mem_alignment_mask);
3264
3265 if (offset_in_file || offset_in_iovbase) {
3266 /*
3267 * one of the 2 important offsets is misaligned
3268 * so fire an I/O through the cache for this entire vector
3269 */
3270 goto wait_for_dwrites;
3271 }
3272 if (iov_base & (devblocksize - 1)) {
3273 /*
3274 * the offset in memory must be on a device block boundary
3275 * so that we can guarantee that we can generate an
3276 * I/O that ends on a page boundary in cluster_io
3277 */
3278 goto wait_for_dwrites;
3279 }
3280
3281 task_update_logical_writes(current_task(), (io_req_size & ~PAGE_MASK), TASK_WRITE_IMMEDIATE, vp);
3282 while ((io_req_size >= PAGE_SIZE || io_req_size >= min_io_size) && uio->uio_offset < newEOF && retval == 0) {
3283 int throttle_type;
3284
3285 if ((throttle_type = cluster_is_throttled(vp))) {
3286 uint32_t max_throttle_size = calculate_max_throttle_size(vp);
3287
3288 /*
3289 * we're in the throttle window, at the very least
3290 * we want to limit the size of the I/O we're about
3291 * to issue
3292 */
3293 if ((flags & IO_RETURN_ON_THROTTLE) && throttle_type == THROTTLE_NOW) {
3294 /*
3295 * we're in the throttle window and at least 1 I/O
3296 * has already been issued by a throttleable thread
3297 * in this window, so return with EAGAIN to indicate
3298 * to the FS issuing the cluster_write call that it
3299 * should now throttle after dropping any locks
3300 */
3301 throttle_info_update_by_mount(vp->v_mount);
3302
3303 io_throttled = TRUE;
3304 goto wait_for_dwrites;
3305 }
3306 max_vector_size = max_throttle_size;
3307 max_io_size = max_throttle_size;
3308 } else {
3309 max_vector_size = MAX_VECTOR_UPL_SIZE;
3310 max_io_size = max_upl_size;
3311 }
3312
3313 if (first_IO) {
3314 cluster_syncup(vp, newEOF, callback, callback_arg, callback ? PUSH_SYNC : 0);
3315 first_IO = 0;
3316 }
3317 io_size = io_req_size & ~io_align_mask;
3318 iov_base = uio_curriovbase(uio);
3319
3320 if (io_size > max_io_size) {
3321 io_size = max_io_size;
3322 }
3323
3324 if (useVectorUPL && (iov_base & PAGE_MASK)) {
3325 /*
3326 * We have an iov_base that's not page-aligned.
3327 * Issue all I/O's that have been collected within
3328 * this Vectored UPL.
3329 */
3330 if (vector_upl_index) {
3331 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3332 if (retval == ENOTSUP) {
3333 goto enotsup;
3334 }
3335 reset_vector_run_state();
3336 }
3337
3338 /*
3339 * After this point, if we are using the Vector UPL path and the base is
3340 * not page-aligned then the UPL with that base will be the first in the vector UPL.
3341 */
3342 }
3343
3344 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
3345 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
3346
3347 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
3348 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
3349
3350 vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map;
3351 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
3352 pages_in_pl = 0;
3353 upl_size = (upl_size_t)upl_needed_size;
3354 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
3355 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3356
3357 kret = vm_map_get_upl(map,
3358 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
3359 &upl_size,
3360 &upl,
3361 NULL,
3362 &pages_in_pl,
3363 &upl_flags,
3364 VM_KERN_MEMORY_FILE,
3365 force_data_sync);
3366
3367 if (kret != KERN_SUCCESS) {
3368 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
3369 0, 0, 0, kret, 0);
3370 /*
3371 * failed to get pagelist
3372 *
3373 * we may have already spun some portion of this request
3374 * off as async requests... we need to wait for the I/O
3375 * to complete before returning
3376 */
3377 goto wait_for_dwrites;
3378 }
3379 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
3380 pages_in_pl = upl_size / PAGE_SIZE;
3381
3382 for (i = 0; i < pages_in_pl; i++) {
3383 if (!upl_valid_page(pl, i)) {
3384 break;
3385 }
3386 }
3387 if (i == pages_in_pl) {
3388 break;
3389 }
3390
3391 /*
3392 * didn't get all the pages back that we
3393 * needed... release this upl and try again
3394 */
3395 ubc_upl_abort(upl, 0);
3396 }
3397 if (force_data_sync >= 3) {
3398 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
3399 i, pages_in_pl, upl_size, kret, 0);
3400 /*
3401 * for some reason, we couldn't acquire a hold on all
3402 * the pages needed in the user's address space
3403 *
3404 * we may have already spun some portion of this request
3405 * off as async requests... we need to wait for the I/O
3406 * to complete before returning
3407 */
3408 goto wait_for_dwrites;
3409 }
3410
3411 /*
3412 * Consider the possibility that upl_size wasn't satisfied.
3413 */
3414 if (upl_size < upl_needed_size) {
3415 if (upl_size && upl_offset == 0) {
3416 io_size = upl_size;
3417 } else {
3418 io_size = 0;
3419 }
3420 }
3421 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
3422 (int)upl_offset, upl_size, (int)iov_base, io_size, 0);
3423
3424 if (io_size == 0) {
3425 ubc_upl_abort(upl, 0);
3426 upl = NULL;
3427 /*
3428 * we may have already spun some portion of this request
3429 * off as async requests... we need to wait for the I/O
3430 * to complete before returning
3431 */
3432 goto wait_for_dwrites;
3433 }
3434
3435 if (useVectorUPL) {
3436 vm_offset_t end_off = ((iov_base + io_size) & PAGE_MASK);
3437 if (end_off) {
3438 issueVectorUPL = 1;
3439 }
3440 /*
3441 * After this point, if we are using a vector UPL, then
3442 * either all the UPL elements end on a page boundary OR
3443 * this UPL is the last element because it does not end
3444 * on a page boundary.
3445 */
3446 }
3447
3448 /*
3449 * we want push out these writes asynchronously so that we can overlap
3450 * the preparation of the next I/O
3451 * if there are already too many outstanding writes
3452 * wait until some complete before issuing the next
3453 */
3454 if (vp->v_mount->mnt_minsaturationbytecount) {
3455 bytes_outstanding_limit = vp->v_mount->mnt_minsaturationbytecount;
3456 } else {
3457 if (__improbable(os_mul_overflow(max_upl_size, IO_SCALE(vp, 2),
3458 &bytes_outstanding_limit) ||
3459 (bytes_outstanding_limit > overlapping_write_max))) {
3460 bytes_outstanding_limit = overlapping_write_max;
3461 }
3462 }
3463
3464 cluster_iostate_wait(&iostate, bytes_outstanding_limit, "cluster_write_direct");
3465
3466 if (iostate.io_error) {
3467 /*
3468 * one of the earlier writes we issued ran into a hard error
3469 * don't issue any more writes, cleanup the UPL
3470 * that was just created but not used, then
3471 * go wait for all writes that are part of this stream
3472 * to complete before returning the error to the caller
3473 */
3474 ubc_upl_abort(upl, 0);
3475 upl = NULL;
3476
3477 goto wait_for_dwrites;
3478 }
3479
3480 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
3481 (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
3482
3483 if (!useVectorUPL) {
3484 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
3485 io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3486 } else {
3487 if (!vector_upl_index) {
3488 vector_upl = vector_upl_create(upl_offset, uio->uio_iovcnt);
3489 v_upl_uio_offset = uio->uio_offset;
3490 vector_upl_offset = upl_offset;
3491 }
3492
3493 vector_upl_set_subupl(vector_upl, upl, upl_size);
3494 vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
3495 vector_upl_index++;
3496 vector_upl_iosize += io_size;
3497 vector_upl_size += upl_size;
3498
3499 if (issueVectorUPL || vector_upl_index == vector_upl_max_upls(vector_upl) || vector_upl_size >= max_vector_size) {
3500 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3501 if (retval != ENOTSUP) {
3502 reset_vector_run_state();
3503 }
3504 }
3505 }
3506 if (retval == ENOTSUP) {
3507 enotsup:
3508 /*
3509 * Can't do direct I/O. Try again with cached I/O.
3510 */
3511 // printf("******* FBDP %s:%d ENOTSUP io_size 0%x resid 0x%llx\n", __FUNCTION__, __LINE__, io_size, uio_resid(uio));
3512 io_size = 0;
3513 if (snapshot_uio) {
3514 int restore_error;
3515
3516 /*
3517 * We've been collecting UPLs for this vector UPL and
3518 * moving the uio along. We need to undo that so that
3519 * the I/O can continue where it actually stopped...
3520 */
3521 restore_error = uio_restore(uio, snapshot_uio);
3522 assert(!restore_error);
3523 uio_free(snapshot_uio);
3524 snapshot_uio = NULL;
3525 }
3526 if (vector_upl_index) {
3527 ubc_upl_abort(vector_upl, 0);
3528 vector_upl = NULL;
3529 } else {
3530 ubc_upl_abort(upl, 0);
3531 upl = NULL;
3532 }
3533 goto wait_for_dwrites;
3534 }
3535
3536 /*
3537 * update the uio structure to
3538 * reflect the I/O that we just issued
3539 */
3540 uio_update(uio, (user_size_t)io_size);
3541
3542 /*
3543 * in case we end up calling through to cluster_write_copy to finish
3544 * the tail of this request, we need to update the oldEOF so that we
3545 * don't zero-fill the head of a page if we've successfully written
3546 * data to that area... 'cluster_write_copy' will zero-fill the head of a
3547 * page that is beyond the oldEOF if the write is unaligned... we only
3548 * want that to happen for the very first page of the cluster_write,
3549 * NOT the first page of each vector making up a multi-vector write.
3550 */
3551 if (uio->uio_offset > oldEOF) {
3552 oldEOF = uio->uio_offset;
3553 }
3554
3555 io_req_size -= io_size;
3556
3557 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
3558 (int)upl_offset, (int)uio->uio_offset, io_req_size, retval, 0);
3559 } /* end while */
3560
3561 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0) {
3562 retval = cluster_io_type(uio, write_type, write_length, min_io_size);
3563
3564 if (retval == 0 && *write_type == IO_DIRECT) {
3565 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_NONE,
3566 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
3567
3568 goto next_dwrite;
3569 }
3570 }
3571
3572 wait_for_dwrites:
3573
3574 if (retval == 0 && iostate.io_error == 0 && useVectorUPL && vector_upl_index) {
3575 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3576 reset_vector_run_state();
3577 }
3578 /*
3579 * make sure all async writes issued as part of this stream
3580 * have completed before we return
3581 */
3582 cluster_iostate_wait(&iostate, 0, "cluster_write_direct");
3583
3584 if (iostate.io_error) {
3585 retval = iostate.io_error;
3586 }
3587
3588 lck_mtx_destroy(&iostate.io_mtxp, &cl_mtx_grp);
3589
3590 if (io_throttled == TRUE && retval == 0) {
3591 retval = EAGAIN;
3592 }
3593
3594 if (io_req_size && retval == 0) {
3595 /*
3596 * we couldn't handle the tail of this request in DIRECT mode
3597 * so fire it through the copy path
3598 *
3599 * note that flags will never have IO_HEADZEROFILL or IO_TAILZEROFILL set
3600 * so we can just pass 0 in for the headOff and tailOff
3601 */
3602 if (uio->uio_offset > oldEOF) {
3603 oldEOF = uio->uio_offset;
3604 }
3605
3606 retval = cluster_write_copy(vp, uio, io_req_size, oldEOF, newEOF, (off_t)0, (off_t)0, flags, callback, callback_arg);
3607
3608 *write_type = IO_UNKNOWN;
3609 }
3610
3611 if (snapshot_uio) {
3612 uio_free(snapshot_uio);
3613 snapshot_uio = NULL;
3614 }
3615
3616 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
3617 (int)uio->uio_offset, io_req_size, retval, 4, 0);
3618
3619 return retval;
3620 }
3621
3622
3623 static int
cluster_write_contig(vnode_t vp,struct uio * uio,off_t newEOF,int * write_type,u_int32_t * write_length,int (* callback)(buf_t,void *),void * callback_arg,int bflag)3624 cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF, int *write_type, u_int32_t *write_length,
3625 int (*callback)(buf_t, void *), void *callback_arg, int bflag)
3626 {
3627 upl_page_info_t *pl;
3628 addr64_t src_paddr = 0;
3629 upl_t upl[MAX_VECTS];
3630 vm_offset_t upl_offset;
3631 u_int32_t tail_size = 0;
3632 u_int32_t io_size;
3633 u_int32_t xsize;
3634 upl_size_t upl_size;
3635 vm_size_t upl_needed_size;
3636 mach_msg_type_number_t pages_in_pl;
3637 upl_control_flags_t upl_flags;
3638 kern_return_t kret;
3639 struct clios iostate;
3640 int error = 0;
3641 int cur_upl = 0;
3642 int num_upl = 0;
3643 int n;
3644 user_addr_t iov_base;
3645 u_int32_t devblocksize;
3646 u_int32_t mem_alignment_mask;
3647
3648 /*
3649 * When we enter this routine, we know
3650 * -- the io_req_size will not exceed iov_len
3651 * -- the target address is physically contiguous
3652 */
3653 cluster_syncup(vp, newEOF, callback, callback_arg, callback ? PUSH_SYNC : 0);
3654
3655 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3656 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3657
3658 iostate.io_completed = 0;
3659 iostate.io_issued = 0;
3660 iostate.io_error = 0;
3661 iostate.io_wanted = 0;
3662
3663 lck_mtx_init(&iostate.io_mtxp, &cl_mtx_grp, LCK_ATTR_NULL);
3664
3665 next_cwrite:
3666 io_size = *write_length;
3667
3668 iov_base = uio_curriovbase(uio);
3669
3670 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
3671 upl_needed_size = upl_offset + io_size;
3672
3673 pages_in_pl = 0;
3674 upl_size = (upl_size_t)upl_needed_size;
3675 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
3676 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3677
3678 vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map;
3679 kret = vm_map_get_upl(map,
3680 vm_map_trunc_page(iov_base, vm_map_page_mask(map)),
3681 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0);
3682
3683 if (kret != KERN_SUCCESS) {
3684 /*
3685 * failed to get pagelist
3686 */
3687 error = EINVAL;
3688 goto wait_for_cwrites;
3689 }
3690 num_upl++;
3691
3692 /*
3693 * Consider the possibility that upl_size wasn't satisfied.
3694 */
3695 if (upl_size < upl_needed_size) {
3696 /*
3697 * This is a failure in the physical memory case.
3698 */
3699 error = EINVAL;
3700 goto wait_for_cwrites;
3701 }
3702 pl = ubc_upl_pageinfo(upl[cur_upl]);
3703
3704 src_paddr = ((addr64_t)upl_phys_page(pl, 0) << PAGE_SHIFT) + (addr64_t)upl_offset;
3705
3706 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
3707 u_int32_t head_size;
3708
3709 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
3710
3711 if (head_size > io_size) {
3712 head_size = io_size;
3713 }
3714
3715 error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0, callback, callback_arg);
3716
3717 if (error) {
3718 goto wait_for_cwrites;
3719 }
3720
3721 upl_offset += head_size;
3722 src_paddr += head_size;
3723 io_size -= head_size;
3724
3725 iov_base += head_size;
3726 }
3727 if ((u_int32_t)iov_base & mem_alignment_mask) {
3728 /*
3729 * request doesn't set up on a memory boundary
3730 * the underlying DMA engine can handle...
3731 * return an error instead of going through
3732 * the slow copy path since the intent of this
3733 * path is direct I/O from device memory
3734 */
3735 error = EINVAL;
3736 goto wait_for_cwrites;
3737 }
3738
3739 tail_size = io_size & (devblocksize - 1);
3740 io_size -= tail_size;
3741
3742 while (io_size && error == 0) {
3743 if (io_size > MAX_IO_CONTIG_SIZE) {
3744 xsize = MAX_IO_CONTIG_SIZE;
3745 } else {
3746 xsize = io_size;
3747 }
3748 /*
3749 * request asynchronously so that we can overlap
3750 * the preparation of the next I/O... we'll do
3751 * the commit after all the I/O has completed
3752 * since its all issued against the same UPL
3753 * if there are already too many outstanding writes
3754 * wait until some have completed before issuing the next
3755 */
3756 cluster_iostate_wait(&iostate, MAX_IO_CONTIG_SIZE * IO_SCALE(vp, 2), "cluster_write_contig");
3757
3758 if (iostate.io_error) {
3759 /*
3760 * one of the earlier writes we issued ran into a hard error
3761 * don't issue any more writes...
3762 * go wait for all writes that are part of this stream
3763 * to complete before returning the error to the caller
3764 */
3765 goto wait_for_cwrites;
3766 }
3767 /*
3768 * issue an asynchronous write to cluster_io
3769 */
3770 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
3771 xsize, CL_DEV_MEMORY | CL_ASYNC | bflag, (buf_t)NULL, (struct clios *)&iostate, callback, callback_arg);
3772
3773 if (error == 0) {
3774 /*
3775 * The cluster_io write completed successfully,
3776 * update the uio structure
3777 */
3778 uio_update(uio, (user_size_t)xsize);
3779
3780 upl_offset += xsize;
3781 src_paddr += xsize;
3782 io_size -= xsize;
3783 }
3784 }
3785 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS) {
3786 error = cluster_io_type(uio, write_type, write_length, 0);
3787
3788 if (error == 0 && *write_type == IO_CONTIG) {
3789 cur_upl++;
3790 goto next_cwrite;
3791 }
3792 } else {
3793 *write_type = IO_UNKNOWN;
3794 }
3795
3796 wait_for_cwrites:
3797 /*
3798 * make sure all async writes that are part of this stream
3799 * have completed before we proceed
3800 */
3801 cluster_iostate_wait(&iostate, 0, "cluster_write_contig");
3802
3803 if (iostate.io_error) {
3804 error = iostate.io_error;
3805 }
3806
3807 lck_mtx_destroy(&iostate.io_mtxp, &cl_mtx_grp);
3808
3809 if (error == 0 && tail_size) {
3810 error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0, callback, callback_arg);
3811 }
3812
3813 for (n = 0; n < num_upl; n++) {
3814 /*
3815 * just release our hold on each physically contiguous
3816 * region without changing any state
3817 */
3818 ubc_upl_abort(upl[n], 0);
3819 }
3820
3821 return error;
3822 }
3823
3824
3825 /*
3826 * need to avoid a race between an msync of a range of pages dirtied via mmap
3827 * vs a filesystem such as HFS deciding to write a 'hole' to disk via cluster_write's
3828 * zerofill mechanism before it has seen the VNOP_PAGEOUTs for the pages being msync'd
3829 *
3830 * we should never force-zero-fill pages that are already valid in the cache...
3831 * the entire page contains valid data (either from disk, zero-filled or dirtied
3832 * via an mmap) so we can only do damage by trying to zero-fill
3833 *
3834 */
3835 static int
cluster_zero_range(upl_t upl,upl_page_info_t * pl,int flags,int io_offset,off_t zero_off,off_t upl_f_offset,int bytes_to_zero)3836 cluster_zero_range(upl_t upl, upl_page_info_t *pl, int flags, int io_offset, off_t zero_off, off_t upl_f_offset, int bytes_to_zero)
3837 {
3838 int zero_pg_index;
3839 boolean_t need_cluster_zero = TRUE;
3840
3841 if ((flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
3842 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off & PAGE_MASK_64));
3843 zero_pg_index = (int)((zero_off - upl_f_offset) / PAGE_SIZE_64);
3844
3845 if (upl_valid_page(pl, zero_pg_index)) {
3846 /*
3847 * never force zero valid pages - dirty or clean
3848 * we'll leave these in the UPL for cluster_write_copy to deal with
3849 */
3850 need_cluster_zero = FALSE;
3851 }
3852 }
3853 if (need_cluster_zero == TRUE) {
3854 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
3855 }
3856
3857 return bytes_to_zero;
3858 }
3859
3860
3861 void
cluster_update_state(vnode_t vp,vm_object_offset_t s_offset,vm_object_offset_t e_offset,boolean_t vm_initiated)3862 cluster_update_state(vnode_t vp, vm_object_offset_t s_offset, vm_object_offset_t e_offset, boolean_t vm_initiated)
3863 {
3864 struct cl_extent cl;
3865 boolean_t first_pass = TRUE;
3866
3867 assert(s_offset < e_offset);
3868 assert((s_offset & PAGE_MASK_64) == 0);
3869 assert((e_offset & PAGE_MASK_64) == 0);
3870
3871 cl.b_addr = (daddr64_t)(s_offset / PAGE_SIZE_64);
3872 cl.e_addr = (daddr64_t)(e_offset / PAGE_SIZE_64);
3873
3874 cluster_update_state_internal(vp, &cl, 0, TRUE, &first_pass, s_offset, (int)(e_offset - s_offset),
3875 vp->v_un.vu_ubcinfo->ui_size, NULL, NULL, vm_initiated);
3876 }
3877
3878
3879 static void
cluster_update_state_internal(vnode_t vp,struct cl_extent * cl,int flags,boolean_t defer_writes,boolean_t * first_pass,off_t write_off,int write_cnt,off_t newEOF,int (* callback)(buf_t,void *),void * callback_arg,boolean_t vm_initiated)3880 cluster_update_state_internal(vnode_t vp, struct cl_extent *cl, int flags, boolean_t defer_writes,
3881 boolean_t *first_pass, off_t write_off, int write_cnt, off_t newEOF,
3882 int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated)
3883 {
3884 struct cl_writebehind *wbp;
3885 int cl_index;
3886 int ret_cluster_try_push;
3887 u_int max_cluster_pgcount;
3888
3889
3890 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
3891
3892 /*
3893 * take the lock to protect our accesses
3894 * of the writebehind and sparse cluster state
3895 */
3896 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
3897
3898 if (wbp->cl_scmap) {
3899 if (!(flags & IO_NOCACHE)) {
3900 /*
3901 * we've fallen into the sparse
3902 * cluster method of delaying dirty pages
3903 */
3904 sparse_cluster_add(wbp, &(wbp->cl_scmap), vp, cl, newEOF, callback, callback_arg, vm_initiated);
3905
3906 lck_mtx_unlock(&wbp->cl_lockw);
3907 return;
3908 }
3909 /*
3910 * must have done cached writes that fell into
3911 * the sparse cluster mechanism... we've switched
3912 * to uncached writes on the file, so go ahead
3913 * and push whatever's in the sparse map
3914 * and switch back to normal clustering
3915 */
3916 wbp->cl_number = 0;
3917
3918 sparse_cluster_push(wbp, &(wbp->cl_scmap), vp, newEOF, PUSH_ALL, 0, callback, callback_arg, vm_initiated);
3919 /*
3920 * no clusters of either type present at this point
3921 * so just go directly to start_new_cluster since
3922 * we know we need to delay this I/O since we've
3923 * already released the pages back into the cache
3924 * to avoid the deadlock with sparse_cluster_push
3925 */
3926 goto start_new_cluster;
3927 }
3928 if (*first_pass == TRUE) {
3929 if (write_off == wbp->cl_last_write) {
3930 wbp->cl_seq_written += write_cnt;
3931 } else {
3932 wbp->cl_seq_written = write_cnt;
3933 }
3934
3935 wbp->cl_last_write = write_off + write_cnt;
3936
3937 *first_pass = FALSE;
3938 }
3939 if (wbp->cl_number == 0) {
3940 /*
3941 * no clusters currently present
3942 */
3943 goto start_new_cluster;
3944 }
3945
3946 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
3947 /*
3948 * check each cluster that we currently hold
3949 * try to merge some or all of this write into
3950 * one or more of the existing clusters... if
3951 * any portion of the write remains, start a
3952 * new cluster
3953 */
3954 if (cl->b_addr >= wbp->cl_clusters[cl_index].b_addr) {
3955 /*
3956 * the current write starts at or after the current cluster
3957 */
3958 if (cl->e_addr <= (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
3959 /*
3960 * we have a write that fits entirely
3961 * within the existing cluster limits
3962 */
3963 if (cl->e_addr > wbp->cl_clusters[cl_index].e_addr) {
3964 /*
3965 * update our idea of where the cluster ends
3966 */
3967 wbp->cl_clusters[cl_index].e_addr = cl->e_addr;
3968 }
3969 break;
3970 }
3971 if (cl->b_addr < (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
3972 /*
3973 * we have a write that starts in the middle of the current cluster
3974 * but extends beyond the cluster's limit... we know this because
3975 * of the previous checks
3976 * we'll extend the current cluster to the max
3977 * and update the b_addr for the current write to reflect that
3978 * the head of it was absorbed into this cluster...
3979 * note that we'll always have a leftover tail in this case since
3980 * full absorbtion would have occurred in the clause above
3981 */
3982 wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount;
3983
3984 cl->b_addr = wbp->cl_clusters[cl_index].e_addr;
3985 }
3986 /*
3987 * we come here for the case where the current write starts
3988 * beyond the limit of the existing cluster or we have a leftover
3989 * tail after a partial absorbtion
3990 *
3991 * in either case, we'll check the remaining clusters before
3992 * starting a new one
3993 */
3994 } else {
3995 /*
3996 * the current write starts in front of the cluster we're currently considering
3997 */
3998 if ((wbp->cl_clusters[cl_index].e_addr - cl->b_addr) <= max_cluster_pgcount) {
3999 /*
4000 * we can just merge the new request into
4001 * this cluster and leave it in the cache
4002 * since the resulting cluster is still
4003 * less than the maximum allowable size
4004 */
4005 wbp->cl_clusters[cl_index].b_addr = cl->b_addr;
4006
4007 if (cl->e_addr > wbp->cl_clusters[cl_index].e_addr) {
4008 /*
4009 * the current write completely
4010 * envelops the existing cluster and since
4011 * each write is limited to at most max_cluster_pgcount pages
4012 * we can just use the start and last blocknos of the write
4013 * to generate the cluster limits
4014 */
4015 wbp->cl_clusters[cl_index].e_addr = cl->e_addr;
4016 }
4017 break;
4018 }
4019 /*
4020 * if we were to combine this write with the current cluster
4021 * we would exceed the cluster size limit.... so,
4022 * let's see if there's any overlap of the new I/O with
4023 * the cluster we're currently considering... in fact, we'll
4024 * stretch the cluster out to it's full limit and see if we
4025 * get an intersection with the current write
4026 *
4027 */
4028 if (cl->e_addr > wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount) {
4029 /*
4030 * the current write extends into the proposed cluster
4031 * clip the length of the current write after first combining it's
4032 * tail with the newly shaped cluster
4033 */
4034 wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount;
4035
4036 cl->e_addr = wbp->cl_clusters[cl_index].b_addr;
4037 }
4038 /*
4039 * if we get here, there was no way to merge
4040 * any portion of this write with this cluster
4041 * or we could only merge part of it which
4042 * will leave a tail...
4043 * we'll check the remaining clusters before starting a new one
4044 */
4045 }
4046 }
4047 if (cl_index < wbp->cl_number) {
4048 /*
4049 * we found an existing cluster(s) that we
4050 * could entirely merge this I/O into
4051 */
4052 goto delay_io;
4053 }
4054
4055 if (defer_writes == FALSE &&
4056 wbp->cl_number == MAX_CLUSTERS &&
4057 wbp->cl_seq_written >= (MAX_CLUSTERS * (max_cluster_pgcount * PAGE_SIZE))) {
4058 uint32_t n;
4059
4060 if (vp->v_mount->mnt_minsaturationbytecount) {
4061 n = vp->v_mount->mnt_minsaturationbytecount / MAX_CLUSTER_SIZE(vp);
4062
4063 if (n > MAX_CLUSTERS) {
4064 n = MAX_CLUSTERS;
4065 }
4066 } else {
4067 n = 0;
4068 }
4069
4070 if (n == 0) {
4071 if (disk_conditioner_mount_is_ssd(vp->v_mount)) {
4072 n = WRITE_BEHIND_SSD;
4073 } else {
4074 n = WRITE_BEHIND;
4075 }
4076 }
4077 while (n--) {
4078 cluster_try_push(wbp, vp, newEOF, 0, 0, callback, callback_arg, NULL, vm_initiated);
4079 }
4080 }
4081 if (wbp->cl_number < MAX_CLUSTERS) {
4082 /*
4083 * we didn't find an existing cluster to
4084 * merge into, but there's room to start
4085 * a new one
4086 */
4087 goto start_new_cluster;
4088 }
4089 /*
4090 * no exisitng cluster to merge with and no
4091 * room to start a new one... we'll try
4092 * pushing one of the existing ones... if none of
4093 * them are able to be pushed, we'll switch
4094 * to the sparse cluster mechanism
4095 * cluster_try_push updates cl_number to the
4096 * number of remaining clusters... and
4097 * returns the number of currently unused clusters
4098 */
4099 ret_cluster_try_push = 0;
4100
4101 /*
4102 * if writes are not deferred, call cluster push immediately
4103 */
4104 if (defer_writes == FALSE) {
4105 ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, (flags & IO_NOCACHE) ? 0 : PUSH_DELAY, 0, callback, callback_arg, NULL, vm_initiated);
4106 }
4107 /*
4108 * execute following regardless of writes being deferred or not
4109 */
4110 if (ret_cluster_try_push == 0) {
4111 /*
4112 * no more room in the normal cluster mechanism
4113 * so let's switch to the more expansive but expensive
4114 * sparse mechanism....
4115 */
4116 sparse_cluster_switch(wbp, vp, newEOF, callback, callback_arg, vm_initiated);
4117 sparse_cluster_add(wbp, &(wbp->cl_scmap), vp, cl, newEOF, callback, callback_arg, vm_initiated);
4118
4119 lck_mtx_unlock(&wbp->cl_lockw);
4120 return;
4121 }
4122 start_new_cluster:
4123 wbp->cl_clusters[wbp->cl_number].b_addr = cl->b_addr;
4124 wbp->cl_clusters[wbp->cl_number].e_addr = cl->e_addr;
4125
4126 wbp->cl_clusters[wbp->cl_number].io_flags = 0;
4127
4128 if (flags & IO_NOCACHE) {
4129 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IONOCACHE;
4130 }
4131
4132 if (flags & IO_PASSIVE) {
4133 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IOPASSIVE;
4134 }
4135
4136 wbp->cl_number++;
4137 delay_io:
4138 lck_mtx_unlock(&wbp->cl_lockw);
4139 return;
4140 }
4141
4142
4143 static int
cluster_write_copy(vnode_t vp,struct uio * uio,u_int32_t io_req_size,off_t oldEOF,off_t newEOF,off_t headOff,off_t tailOff,int flags,int (* callback)(buf_t,void *),void * callback_arg)4144 cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF, off_t headOff,
4145 off_t tailOff, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4146 {
4147 upl_page_info_t *pl;
4148 upl_t upl;
4149 vm_offset_t upl_offset = 0;
4150 vm_size_t upl_size;
4151 off_t upl_f_offset;
4152 int pages_in_upl;
4153 int start_offset;
4154 int xfer_resid;
4155 int io_size;
4156 int io_offset;
4157 int bytes_to_zero;
4158 int bytes_to_move;
4159 kern_return_t kret;
4160 int retval = 0;
4161 int io_resid;
4162 long long total_size;
4163 long long zero_cnt;
4164 off_t zero_off;
4165 long long zero_cnt1;
4166 off_t zero_off1;
4167 off_t write_off = 0;
4168 int write_cnt = 0;
4169 boolean_t first_pass = FALSE;
4170 struct cl_extent cl;
4171 int bflag;
4172 u_int max_io_size;
4173
4174 if (uio) {
4175 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
4176 (int)uio->uio_offset, io_req_size, (int)oldEOF, (int)newEOF, 0);
4177
4178 io_resid = io_req_size;
4179 } else {
4180 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
4181 0, 0, (int)oldEOF, (int)newEOF, 0);
4182
4183 io_resid = 0;
4184 }
4185 if (flags & IO_PASSIVE) {
4186 bflag = CL_PASSIVE;
4187 } else {
4188 bflag = 0;
4189 }
4190 if (flags & IO_NOCACHE) {
4191 bflag |= CL_NOCACHE;
4192 }
4193
4194 if (flags & IO_SKIP_ENCRYPTION) {
4195 bflag |= CL_ENCRYPTED;
4196 }
4197
4198 zero_cnt = 0;
4199 zero_cnt1 = 0;
4200 zero_off = 0;
4201 zero_off1 = 0;
4202
4203 max_io_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
4204
4205 if (flags & IO_HEADZEROFILL) {
4206 /*
4207 * some filesystems (HFS is one) don't support unallocated holes within a file...
4208 * so we zero fill the intervening space between the old EOF and the offset
4209 * where the next chunk of real data begins.... ftruncate will also use this
4210 * routine to zero fill to the new EOF when growing a file... in this case, the
4211 * uio structure will not be provided
4212 */
4213 if (uio) {
4214 if (headOff < uio->uio_offset) {
4215 zero_cnt = uio->uio_offset - headOff;
4216 zero_off = headOff;
4217 }
4218 } else if (headOff < newEOF) {
4219 zero_cnt = newEOF - headOff;
4220 zero_off = headOff;
4221 }
4222 } else {
4223 if (uio && uio->uio_offset > oldEOF) {
4224 zero_off = uio->uio_offset & ~PAGE_MASK_64;
4225
4226 if (zero_off >= oldEOF) {
4227 zero_cnt = uio->uio_offset - zero_off;
4228
4229 flags |= IO_HEADZEROFILL;
4230 }
4231 }
4232 }
4233 if (flags & IO_TAILZEROFILL) {
4234 if (uio) {
4235 zero_off1 = uio->uio_offset + io_req_size;
4236
4237 if (zero_off1 < tailOff) {
4238 zero_cnt1 = tailOff - zero_off1;
4239 }
4240 }
4241 } else {
4242 if (uio && newEOF > oldEOF) {
4243 zero_off1 = uio->uio_offset + io_req_size;
4244
4245 if (zero_off1 == newEOF && (zero_off1 & PAGE_MASK_64)) {
4246 zero_cnt1 = PAGE_SIZE_64 - (zero_off1 & PAGE_MASK_64);
4247
4248 flags |= IO_TAILZEROFILL;
4249 }
4250 }
4251 }
4252 if (zero_cnt == 0 && uio == (struct uio *) 0) {
4253 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
4254 retval, 0, 0, 0, 0);
4255 return 0;
4256 }
4257 if (uio) {
4258 write_off = uio->uio_offset;
4259 write_cnt = (int)uio_resid(uio);
4260 /*
4261 * delay updating the sequential write info
4262 * in the control block until we've obtained
4263 * the lock for it
4264 */
4265 first_pass = TRUE;
4266 }
4267 while ((total_size = (io_resid + zero_cnt + zero_cnt1)) && retval == 0) {
4268 /*
4269 * for this iteration of the loop, figure out where our starting point is
4270 */
4271 if (zero_cnt) {
4272 start_offset = (int)(zero_off & PAGE_MASK_64);
4273 upl_f_offset = zero_off - start_offset;
4274 } else if (io_resid) {
4275 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
4276 upl_f_offset = uio->uio_offset - start_offset;
4277 } else {
4278 start_offset = (int)(zero_off1 & PAGE_MASK_64);
4279 upl_f_offset = zero_off1 - start_offset;
4280 }
4281 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 46)) | DBG_FUNC_NONE,
4282 (int)zero_off, (int)zero_cnt, (int)zero_off1, (int)zero_cnt1, 0);
4283
4284 if (total_size > max_io_size) {
4285 total_size = max_io_size;
4286 }
4287
4288 cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
4289
4290 if (uio && ((flags & (IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
4291 /*
4292 * assumption... total_size <= io_resid
4293 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
4294 */
4295 if ((start_offset + total_size) > max_io_size) {
4296 total_size = max_io_size - start_offset;
4297 }
4298 xfer_resid = (int)total_size;
4299
4300 retval = cluster_copy_ubc_data_internal(vp, uio, &xfer_resid, 1, 1);
4301
4302 if (retval) {
4303 break;
4304 }
4305
4306 io_resid -= (total_size - xfer_resid);
4307 total_size = xfer_resid;
4308 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
4309 upl_f_offset = uio->uio_offset - start_offset;
4310
4311 if (total_size == 0) {
4312 if (start_offset) {
4313 /*
4314 * the write did not finish on a page boundary
4315 * which will leave upl_f_offset pointing to the
4316 * beginning of the last page written instead of
4317 * the page beyond it... bump it in this case
4318 * so that the cluster code records the last page
4319 * written as dirty
4320 */
4321 upl_f_offset += PAGE_SIZE_64;
4322 }
4323 upl_size = 0;
4324
4325 goto check_cluster;
4326 }
4327 }
4328 /*
4329 * compute the size of the upl needed to encompass
4330 * the requested write... limit each call to cluster_io
4331 * to the maximum UPL size... cluster_io will clip if
4332 * this exceeds the maximum io_size for the device,
4333 * make sure to account for
4334 * a starting offset that's not page aligned
4335 */
4336 upl_size = (start_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4337
4338 if (upl_size > max_io_size) {
4339 upl_size = max_io_size;
4340 }
4341
4342 pages_in_upl = (int)(upl_size / PAGE_SIZE);
4343 io_size = (int)(upl_size - start_offset);
4344
4345 if ((long long)io_size > total_size) {
4346 io_size = (int)total_size;
4347 }
4348
4349 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, io_size, total_size, 0, 0);
4350
4351
4352 /*
4353 * Gather the pages from the buffer cache.
4354 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
4355 * that we intend to modify these pages.
4356 */
4357 kret = ubc_create_upl_kernel(vp,
4358 upl_f_offset,
4359 (int)upl_size,
4360 &upl,
4361 &pl,
4362 UPL_SET_LITE | ((uio != NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY),
4363 VM_KERN_MEMORY_FILE);
4364 if (kret != KERN_SUCCESS) {
4365 panic("cluster_write_copy: failed to get pagelist");
4366 }
4367
4368 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
4369 upl, (int)upl_f_offset, start_offset, 0, 0);
4370
4371 if (start_offset && upl_f_offset < oldEOF && !upl_valid_page(pl, 0)) {
4372 int read_size;
4373
4374 /*
4375 * we're starting in the middle of the first page of the upl
4376 * and the page isn't currently valid, so we're going to have
4377 * to read it in first... this is a synchronous operation
4378 */
4379 read_size = PAGE_SIZE;
4380
4381 if ((upl_f_offset + read_size) > oldEOF) {
4382 read_size = (int)(oldEOF - upl_f_offset);
4383 }
4384
4385 retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
4386 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4387 if (retval) {
4388 /*
4389 * we had an error during the read which causes us to abort
4390 * the current cluster_write request... before we do, we need
4391 * to release the rest of the pages in the upl without modifying
4392 * there state and mark the failed page in error
4393 */
4394 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
4395
4396 if (upl_size > PAGE_SIZE) {
4397 ubc_upl_abort_range(upl, 0, (upl_size_t)upl_size,
4398 UPL_ABORT_FREE_ON_EMPTY);
4399 }
4400
4401 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
4402 upl, 0, 0, retval, 0);
4403 break;
4404 }
4405 }
4406 if ((start_offset == 0 || upl_size > PAGE_SIZE) && ((start_offset + io_size) & PAGE_MASK)) {
4407 /*
4408 * the last offset we're writing to in this upl does not end on a page
4409 * boundary... if it's not beyond the old EOF, then we'll also need to
4410 * pre-read this page in if it isn't already valid
4411 */
4412 upl_offset = upl_size - PAGE_SIZE;
4413
4414 if ((upl_f_offset + start_offset + io_size) < oldEOF &&
4415 !upl_valid_page(pl, (int)(upl_offset / PAGE_SIZE))) {
4416 int read_size;
4417
4418 read_size = PAGE_SIZE;
4419
4420 if ((off_t)(upl_f_offset + upl_offset + read_size) > oldEOF) {
4421 read_size = (int)(oldEOF - (upl_f_offset + upl_offset));
4422 }
4423
4424 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
4425 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4426 if (retval) {
4427 /*
4428 * we had an error during the read which causes us to abort
4429 * the current cluster_write request... before we do, we
4430 * need to release the rest of the pages in the upl without
4431 * modifying there state and mark the failed page in error
4432 */
4433 ubc_upl_abort_range(upl, (upl_offset_t)upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
4434
4435 if (upl_size > PAGE_SIZE) {
4436 ubc_upl_abort_range(upl, 0, (upl_size_t)upl_size, UPL_ABORT_FREE_ON_EMPTY);
4437 }
4438
4439 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
4440 upl, 0, 0, retval, 0);
4441 break;
4442 }
4443 }
4444 }
4445 xfer_resid = io_size;
4446 io_offset = start_offset;
4447
4448 while (zero_cnt && xfer_resid) {
4449 if (zero_cnt < (long long)xfer_resid) {
4450 bytes_to_zero = (int)zero_cnt;
4451 } else {
4452 bytes_to_zero = xfer_resid;
4453 }
4454
4455 bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off, upl_f_offset, bytes_to_zero);
4456
4457 xfer_resid -= bytes_to_zero;
4458 zero_cnt -= bytes_to_zero;
4459 zero_off += bytes_to_zero;
4460 io_offset += bytes_to_zero;
4461 }
4462 if (xfer_resid && io_resid) {
4463 u_int32_t io_requested;
4464
4465 bytes_to_move = min(io_resid, xfer_resid);
4466 io_requested = bytes_to_move;
4467
4468 retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
4469
4470 if (retval) {
4471 ubc_upl_abort_range(upl, 0, (upl_size_t)upl_size, UPL_ABORT_FREE_ON_EMPTY);
4472
4473 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
4474 upl, 0, 0, retval, 0);
4475 } else {
4476 io_resid -= bytes_to_move;
4477 xfer_resid -= bytes_to_move;
4478 io_offset += bytes_to_move;
4479 }
4480 }
4481 while (xfer_resid && zero_cnt1 && retval == 0) {
4482 if (zero_cnt1 < (long long)xfer_resid) {
4483 bytes_to_zero = (int)zero_cnt1;
4484 } else {
4485 bytes_to_zero = xfer_resid;
4486 }
4487
4488 bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off1, upl_f_offset, bytes_to_zero);
4489
4490 xfer_resid -= bytes_to_zero;
4491 zero_cnt1 -= bytes_to_zero;
4492 zero_off1 += bytes_to_zero;
4493 io_offset += bytes_to_zero;
4494 }
4495 if (retval == 0) {
4496 int do_zeroing = 1;
4497
4498 io_size += start_offset;
4499
4500 /* Force more restrictive zeroing behavior only on APFS */
4501 if ((vnode_tag(vp) == VT_APFS) && (newEOF < oldEOF)) {
4502 do_zeroing = 0;
4503 }
4504
4505 if (do_zeroing && (upl_f_offset + io_size) >= newEOF && (u_int)io_size < upl_size) {
4506 /*
4507 * if we're extending the file with this write
4508 * we'll zero fill the rest of the page so that
4509 * if the file gets extended again in such a way as to leave a
4510 * hole starting at this EOF, we'll have zero's in the correct spot
4511 */
4512 cluster_zero(upl, io_size, (int)(upl_size - io_size), NULL);
4513 }
4514 /*
4515 * release the upl now if we hold one since...
4516 * 1) pages in it may be present in the sparse cluster map
4517 * and may span 2 separate buckets there... if they do and
4518 * we happen to have to flush a bucket to make room and it intersects
4519 * this upl, a deadlock may result on page BUSY
4520 * 2) we're delaying the I/O... from this point forward we're just updating
4521 * the cluster state... no need to hold the pages, so commit them
4522 * 3) IO_SYNC is set...
4523 * because we had to ask for a UPL that provides currenty non-present pages, the
4524 * UPL has been automatically set to clear the dirty flags (both software and hardware)
4525 * upon committing it... this is not the behavior we want since it's possible for
4526 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
4527 * we'll pick these pages back up later with the correct behavior specified.
4528 * 4) we don't want to hold pages busy in a UPL and then block on the cluster lock... if a flush
4529 * of this vnode is in progress, we will deadlock if the pages being flushed intersect the pages
4530 * we hold since the flushing context is holding the cluster lock.
4531 */
4532 ubc_upl_commit_range(upl, 0, (upl_size_t)upl_size,
4533 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
4534 check_cluster:
4535 /*
4536 * calculate the last logical block number
4537 * that this delayed I/O encompassed
4538 */
4539 cl.e_addr = (daddr64_t)((upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64);
4540
4541 if (flags & IO_SYNC) {
4542 /*
4543 * if the IO_SYNC flag is set than we need to bypass
4544 * any clustering and immediately issue the I/O
4545 *
4546 * we don't hold the lock at this point
4547 *
4548 * we've already dropped the current upl, so pick it back up with COPYOUT_FROM set
4549 * so that we correctly deal with a change in state of the hardware modify bit...
4550 * we do this via cluster_push_now... by passing along the IO_SYNC flag, we force
4551 * cluster_push_now to wait until all the I/Os have completed... cluster_push_now is also
4552 * responsible for generating the correct sized I/O(s)
4553 */
4554 retval = cluster_push_now(vp, &cl, newEOF, flags, callback, callback_arg, FALSE);
4555 } else {
4556 boolean_t defer_writes = FALSE;
4557
4558 if (vfs_flags(vp->v_mount) & MNT_DEFWRITE) {
4559 defer_writes = TRUE;
4560 }
4561
4562 cluster_update_state_internal(vp, &cl, flags, defer_writes, &first_pass,
4563 write_off, write_cnt, newEOF, callback, callback_arg, FALSE);
4564 }
4565 }
4566 }
4567 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END, retval, 0, io_resid, 0, 0);
4568
4569 return retval;
4570 }
4571
4572
4573
4574 int
cluster_read(vnode_t vp,struct uio * uio,off_t filesize,int xflags)4575 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
4576 {
4577 return cluster_read_ext(vp, uio, filesize, xflags, NULL, NULL);
4578 }
4579
4580
4581 int
cluster_read_ext(vnode_t vp,struct uio * uio,off_t filesize,int xflags,int (* callback)(buf_t,void *),void * callback_arg)4582 cluster_read_ext(vnode_t vp, struct uio *uio, off_t filesize, int xflags, int (*callback)(buf_t, void *), void *callback_arg)
4583 {
4584 int retval = 0;
4585 int flags;
4586 user_ssize_t cur_resid;
4587 u_int32_t io_size;
4588 u_int32_t read_length = 0;
4589 int read_type = IO_COPY;
4590 bool check_io_type;
4591
4592 flags = xflags;
4593
4594 if (vp->v_flag & VNOCACHE_DATA) {
4595 flags |= IO_NOCACHE;
4596 }
4597 if ((vp->v_flag & VRAOFF) || speculative_reads_disabled) {
4598 flags |= IO_RAOFF;
4599 }
4600
4601 if (flags & IO_SKIP_ENCRYPTION) {
4602 flags |= IO_ENCRYPTED;
4603 }
4604
4605 /*
4606 * do a read through the cache if one of the following is true....
4607 * NOCACHE is not true
4608 * the uio request doesn't target USERSPACE (unless IO_NOCACHE_SYSSPACE is also set)
4609 * Alternatively, if IO_ENCRYPTED is set, then we want to bypass the cache as well.
4610 * Reading encrypted data from a CP filesystem should never result in the data touching
4611 * the UBC.
4612 *
4613 * otherwise, find out if we want the direct or contig variant for
4614 * the first vector in the uio request
4615 */
4616 check_io_type = false;
4617 if (flags & IO_NOCACHE) {
4618 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
4619 /*
4620 * no-cache to user-space: ok to consider IO_DIRECT.
4621 */
4622 check_io_type = true;
4623 } else if (uio->uio_segflg == UIO_SYSSPACE &&
4624 (flags & IO_NOCACHE_SYSSPACE)) {
4625 /*
4626 * no-cache to kernel-space but w/ IO_NOCACHE_SYSSPACE:
4627 * ok to consider IO_DIRECT.
4628 * The caller should make sure to target kernel buffer
4629 * that is backed by regular anonymous memory (i.e.
4630 * not backed by the kernel object or an external
4631 * memory manager like device memory or a file).
4632 */
4633 check_io_type = true;
4634 }
4635 } else if (flags & IO_ENCRYPTED) {
4636 check_io_type = true;
4637 }
4638 if (check_io_type) {
4639 retval = cluster_io_type(uio, &read_type, &read_length, 0);
4640 }
4641
4642 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < filesize && retval == 0) {
4643 switch (read_type) {
4644 case IO_COPY:
4645 /*
4646 * make sure the uio_resid isn't too big...
4647 * internally, we want to handle all of the I/O in
4648 * chunk sizes that fit in a 32 bit int
4649 */
4650 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
4651 io_size = MAX_IO_REQUEST_SIZE;
4652 } else {
4653 io_size = (u_int32_t)cur_resid;
4654 }
4655
4656 retval = cluster_read_copy(vp, uio, io_size, filesize, flags, callback, callback_arg);
4657 break;
4658
4659 case IO_DIRECT:
4660 retval = cluster_read_direct(vp, uio, filesize, &read_type, &read_length, flags, callback, callback_arg);
4661 break;
4662
4663 case IO_CONTIG:
4664 retval = cluster_read_contig(vp, uio, filesize, &read_type, &read_length, callback, callback_arg, flags);
4665 break;
4666
4667 case IO_UNKNOWN:
4668 retval = cluster_io_type(uio, &read_type, &read_length, 0);
4669 break;
4670 }
4671 }
4672 return retval;
4673 }
4674
4675
4676
4677 static void
cluster_read_upl_release(upl_t upl,int start_pg,int last_pg,int take_reference)4678 cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference)
4679 {
4680 int range;
4681 int abort_flags = UPL_ABORT_FREE_ON_EMPTY;
4682
4683 if ((range = last_pg - start_pg)) {
4684 if (take_reference) {
4685 abort_flags |= UPL_ABORT_REFERENCE;
4686 }
4687
4688 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
4689 }
4690 }
4691
4692
4693 static int
cluster_read_copy(vnode_t vp,struct uio * uio,u_int32_t io_req_size,off_t filesize,int flags,int (* callback)(buf_t,void *),void * callback_arg)4694 cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4695 {
4696 upl_page_info_t *pl;
4697 upl_t upl = NULL;
4698 vm_offset_t upl_offset;
4699 u_int32_t upl_size;
4700 off_t upl_f_offset;
4701 int start_offset;
4702 int start_pg;
4703 int last_pg;
4704 int uio_last = 0;
4705 int pages_in_upl;
4706 off_t max_size;
4707 off_t last_ioread_offset;
4708 off_t last_request_offset;
4709 kern_return_t kret;
4710 int error = 0;
4711 int retval = 0;
4712 u_int32_t size_of_prefetch;
4713 u_int32_t xsize;
4714 u_int32_t io_size;
4715 u_int32_t max_rd_size;
4716 u_int32_t max_io_size;
4717 u_int32_t max_prefetch;
4718 u_int rd_ahead_enabled = 1;
4719 u_int prefetch_enabled = 1;
4720 struct cl_readahead * rap;
4721 struct clios iostate;
4722 struct cl_extent extent;
4723 int bflag;
4724 int take_reference = 1;
4725 int policy = IOPOL_DEFAULT;
4726 boolean_t iolock_inited = FALSE;
4727
4728 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
4729 (int)uio->uio_offset, io_req_size, (int)filesize, flags, 0);
4730
4731 if (flags & IO_ENCRYPTED) {
4732 panic("encrypted blocks will hit UBC!");
4733 }
4734
4735 policy = throttle_get_io_policy(NULL);
4736
4737 if (policy == THROTTLE_LEVEL_TIER3 || policy == THROTTLE_LEVEL_TIER2 || (flags & IO_NOCACHE)) {
4738 take_reference = 0;
4739 }
4740
4741 if (flags & IO_PASSIVE) {
4742 bflag = CL_PASSIVE;
4743 } else {
4744 bflag = 0;
4745 }
4746
4747 if (flags & IO_NOCACHE) {
4748 bflag |= CL_NOCACHE;
4749 }
4750
4751 if (flags & IO_SKIP_ENCRYPTION) {
4752 bflag |= CL_ENCRYPTED;
4753 }
4754
4755 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
4756 max_prefetch = cluster_max_prefetch(vp, max_io_size, prefetch_max);
4757 max_rd_size = max_prefetch;
4758
4759 last_request_offset = uio->uio_offset + io_req_size;
4760
4761 if (last_request_offset > filesize) {
4762 last_request_offset = filesize;
4763 }
4764
4765 if ((flags & (IO_RAOFF | IO_NOCACHE)) || ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
4766 rd_ahead_enabled = 0;
4767 rap = NULL;
4768 } else {
4769 if (cluster_is_throttled(vp)) {
4770 /*
4771 * we're in the throttle window, at the very least
4772 * we want to limit the size of the I/O we're about
4773 * to issue
4774 */
4775 rd_ahead_enabled = 0;
4776 prefetch_enabled = 0;
4777
4778 max_rd_size = calculate_max_throttle_size(vp);
4779 }
4780 if ((rap = cluster_get_rap(vp)) == NULL) {
4781 rd_ahead_enabled = 0;
4782 } else {
4783 extent.b_addr = uio->uio_offset / PAGE_SIZE_64;
4784 extent.e_addr = (last_request_offset - 1) / PAGE_SIZE_64;
4785 }
4786 }
4787 if (rap != NULL && rap->cl_ralen && (rap->cl_lastr == extent.b_addr || (rap->cl_lastr + 1) == extent.b_addr)) {
4788 /*
4789 * determine if we already have a read-ahead in the pipe courtesy of the
4790 * last read systemcall that was issued...
4791 * if so, pick up it's extent to determine where we should start
4792 * with respect to any read-ahead that might be necessary to
4793 * garner all the data needed to complete this read systemcall
4794 */
4795 last_ioread_offset = (rap->cl_maxra * PAGE_SIZE_64) + PAGE_SIZE_64;
4796
4797 if (last_ioread_offset < uio->uio_offset) {
4798 last_ioread_offset = (off_t)0;
4799 } else if (last_ioread_offset > last_request_offset) {
4800 last_ioread_offset = last_request_offset;
4801 }
4802 } else {
4803 last_ioread_offset = (off_t)0;
4804 }
4805
4806 while (io_req_size && uio->uio_offset < filesize && retval == 0) {
4807 max_size = filesize - uio->uio_offset;
4808 bool leftover_upl_aborted = false;
4809
4810 if ((off_t)(io_req_size) < max_size) {
4811 io_size = io_req_size;
4812 } else {
4813 io_size = (u_int32_t)max_size;
4814 }
4815
4816 if (!(flags & IO_NOCACHE)) {
4817 while (io_size) {
4818 u_int32_t io_resid;
4819 u_int32_t io_requested;
4820
4821 /*
4822 * if we keep finding the pages we need already in the cache, then
4823 * don't bother to call cluster_read_prefetch since it costs CPU cycles
4824 * to determine that we have all the pages we need... once we miss in
4825 * the cache and have issued an I/O, than we'll assume that we're likely
4826 * to continue to miss in the cache and it's to our advantage to try and prefetch
4827 */
4828 if (last_request_offset && last_ioread_offset && (size_of_prefetch = (u_int32_t)(last_request_offset - last_ioread_offset))) {
4829 if ((last_ioread_offset - uio->uio_offset) <= max_rd_size && prefetch_enabled) {
4830 /*
4831 * we've already issued I/O for this request and
4832 * there's still work to do and
4833 * our prefetch stream is running dry, so issue a
4834 * pre-fetch I/O... the I/O latency will overlap
4835 * with the copying of the data
4836 */
4837 if (size_of_prefetch > max_rd_size) {
4838 size_of_prefetch = max_rd_size;
4839 }
4840
4841 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
4842
4843 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
4844
4845 if (last_ioread_offset > last_request_offset) {
4846 last_ioread_offset = last_request_offset;
4847 }
4848 }
4849 }
4850 /*
4851 * limit the size of the copy we're about to do so that
4852 * we can notice that our I/O pipe is running dry and
4853 * get the next I/O issued before it does go dry
4854 */
4855 if (last_ioread_offset && io_size > (max_io_size / 4)) {
4856 io_resid = (max_io_size / 4);
4857 } else {
4858 io_resid = io_size;
4859 }
4860
4861 io_requested = io_resid;
4862
4863 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_resid, 0, take_reference);
4864
4865 xsize = io_requested - io_resid;
4866
4867 io_size -= xsize;
4868 io_req_size -= xsize;
4869
4870 if (retval || io_resid) {
4871 /*
4872 * if we run into a real error or
4873 * a page that is not in the cache
4874 * we need to leave streaming mode
4875 */
4876 break;
4877 }
4878
4879 if (rd_ahead_enabled && (io_size == 0 || last_ioread_offset == last_request_offset)) {
4880 /*
4881 * we're already finished the I/O for this read request
4882 * let's see if we should do a read-ahead
4883 */
4884 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
4885 }
4886 }
4887 if (retval) {
4888 break;
4889 }
4890 if (io_size == 0) {
4891 if (rap != NULL) {
4892 if (extent.e_addr < rap->cl_lastr) {
4893 rap->cl_maxra = 0;
4894 }
4895 rap->cl_lastr = extent.e_addr;
4896 }
4897 break;
4898 }
4899 /*
4900 * recompute max_size since cluster_copy_ubc_data_internal
4901 * may have advanced uio->uio_offset
4902 */
4903 max_size = filesize - uio->uio_offset;
4904 }
4905
4906 iostate.io_completed = 0;
4907 iostate.io_issued = 0;
4908 iostate.io_error = 0;
4909 iostate.io_wanted = 0;
4910
4911 if ((flags & IO_RETURN_ON_THROTTLE)) {
4912 if (cluster_is_throttled(vp) == THROTTLE_NOW) {
4913 if (!cluster_io_present_in_BC(vp, uio->uio_offset)) {
4914 /*
4915 * we're in the throttle window and at least 1 I/O
4916 * has already been issued by a throttleable thread
4917 * in this window, so return with EAGAIN to indicate
4918 * to the FS issuing the cluster_read call that it
4919 * should now throttle after dropping any locks
4920 */
4921 throttle_info_update_by_mount(vp->v_mount);
4922
4923 retval = EAGAIN;
4924 break;
4925 }
4926 }
4927 }
4928
4929 /*
4930 * compute the size of the upl needed to encompass
4931 * the requested read... limit each call to cluster_io
4932 * to the maximum UPL size... cluster_io will clip if
4933 * this exceeds the maximum io_size for the device,
4934 * make sure to account for
4935 * a starting offset that's not page aligned
4936 */
4937 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
4938 upl_f_offset = uio->uio_offset - (off_t)start_offset;
4939
4940 if (io_size > max_rd_size) {
4941 io_size = max_rd_size;
4942 }
4943
4944 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4945
4946 if (flags & IO_NOCACHE) {
4947 if (upl_size > max_io_size) {
4948 upl_size = max_io_size;
4949 }
4950 } else {
4951 if (upl_size > max_io_size / 4) {
4952 upl_size = max_io_size / 4;
4953 upl_size &= ~PAGE_MASK;
4954
4955 if (upl_size == 0) {
4956 upl_size = PAGE_SIZE;
4957 }
4958 }
4959 }
4960 pages_in_upl = upl_size / PAGE_SIZE;
4961
4962 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
4963 upl, (int)upl_f_offset, upl_size, start_offset, 0);
4964
4965 kret = ubc_create_upl_kernel(vp,
4966 upl_f_offset,
4967 upl_size,
4968 &upl,
4969 &pl,
4970 UPL_FILE_IO | UPL_SET_LITE,
4971 VM_KERN_MEMORY_FILE);
4972 if (kret != KERN_SUCCESS) {
4973 panic("cluster_read_copy: failed to get pagelist");
4974 }
4975
4976 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
4977 upl, (int)upl_f_offset, upl_size, start_offset, 0);
4978
4979 /*
4980 * scan from the beginning of the upl looking for the first
4981 * non-valid page.... this will become the first page in
4982 * the request we're going to make to 'cluster_io'... if all
4983 * of the pages are valid, we won't call through to 'cluster_io'
4984 */
4985 for (start_pg = 0; start_pg < pages_in_upl; start_pg++) {
4986 if (!upl_valid_page(pl, start_pg)) {
4987 break;
4988 }
4989 }
4990
4991 /*
4992 * scan from the starting invalid page looking for a valid
4993 * page before the end of the upl is reached, if we
4994 * find one, then it will be the last page of the request to
4995 * 'cluster_io'
4996 */
4997 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4998 if (upl_valid_page(pl, last_pg)) {
4999 break;
5000 }
5001 }
5002
5003 if (start_pg < last_pg) {
5004 /*
5005 * we found a range of 'invalid' pages that must be filled
5006 * if the last page in this range is the last page of the file
5007 * we may have to clip the size of it to keep from reading past
5008 * the end of the last physical block associated with the file
5009 */
5010 if (iolock_inited == FALSE) {
5011 lck_mtx_init(&iostate.io_mtxp, &cl_mtx_grp, LCK_ATTR_NULL);
5012
5013 iolock_inited = TRUE;
5014 }
5015 upl_offset = start_pg * PAGE_SIZE;
5016 io_size = (last_pg - start_pg) * PAGE_SIZE;
5017
5018 if ((off_t)(upl_f_offset + upl_offset + io_size) > filesize) {
5019 io_size = (u_int32_t)(filesize - (upl_f_offset + upl_offset));
5020 }
5021
5022 /*
5023 * Find out if this needs verification, we'll have to manage the UPL
5024 * diffrently if so. Note that this call only lets us know if
5025 * verification is enabled on this mount point, the actual verification
5026 * is performed in the File system.
5027 */
5028 size_t verify_block_size = 0;
5029 if ((VNOP_VERIFY(vp, start_offset, NULL, 0, &verify_block_size, NULL, VNODE_VERIFY_DEFAULT, NULL, NULL) == 0) /* && verify_block_size */) {
5030 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
5031 if (!upl_valid_page(pl, uio_last)) {
5032 break;
5033 }
5034 }
5035 if (uio_last < pages_in_upl) {
5036 /*
5037 * there were some invalid pages beyond the valid pages
5038 * that we didn't issue an I/O for, just release them
5039 * unchanged now, so that any prefetch/readahed can
5040 * include them
5041 */
5042 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
5043 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
5044 leftover_upl_aborted = true;
5045 }
5046 }
5047
5048 /*
5049 * issue an asynchronous read to cluster_io
5050 */
5051
5052 error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
5053 io_size, CL_READ | CL_ASYNC | bflag, (buf_t)NULL, &iostate, callback, callback_arg);
5054
5055 if (rap) {
5056 if (extent.e_addr < rap->cl_maxra) {
5057 /*
5058 * we've just issued a read for a block that should have been
5059 * in the cache courtesy of the read-ahead engine... something
5060 * has gone wrong with the pipeline, so reset the read-ahead
5061 * logic which will cause us to restart from scratch
5062 */
5063 rap->cl_maxra = 0;
5064 }
5065 }
5066 }
5067 if (error == 0) {
5068 /*
5069 * if the read completed successfully, or there was no I/O request
5070 * issued, than copy the data into user land via 'cluster_upl_copy_data'
5071 * we'll first add on any 'valid'
5072 * pages that were present in the upl when we acquired it.
5073 */
5074 u_int val_size;
5075
5076 if (!leftover_upl_aborted) {
5077 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
5078 if (!upl_valid_page(pl, uio_last)) {
5079 break;
5080 }
5081 }
5082 if (uio_last < pages_in_upl) {
5083 /*
5084 * there were some invalid pages beyond the valid pages
5085 * that we didn't issue an I/O for, just release them
5086 * unchanged now, so that any prefetch/readahed can
5087 * include them
5088 */
5089 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
5090 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
5091 }
5092 }
5093
5094 /*
5095 * compute size to transfer this round, if io_req_size is
5096 * still non-zero after this attempt, we'll loop around and
5097 * set up for another I/O.
5098 */
5099 val_size = (uio_last * PAGE_SIZE) - start_offset;
5100
5101 if (val_size > max_size) {
5102 val_size = (u_int)max_size;
5103 }
5104
5105 if (val_size > io_req_size) {
5106 val_size = io_req_size;
5107 }
5108
5109 if ((uio->uio_offset + val_size) > last_ioread_offset) {
5110 last_ioread_offset = uio->uio_offset + val_size;
5111 }
5112
5113 if ((size_of_prefetch = (u_int32_t)(last_request_offset - last_ioread_offset)) && prefetch_enabled) {
5114 if ((last_ioread_offset - (uio->uio_offset + val_size)) <= upl_size) {
5115 /*
5116 * if there's still I/O left to do for this request, and...
5117 * we're not in hard throttle mode, and...
5118 * we're close to using up the previous prefetch, then issue a
5119 * new pre-fetch I/O... the I/O latency will overlap
5120 * with the copying of the data
5121 */
5122 if (size_of_prefetch > max_rd_size) {
5123 size_of_prefetch = max_rd_size;
5124 }
5125
5126 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
5127
5128 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
5129
5130 if (last_ioread_offset > last_request_offset) {
5131 last_ioread_offset = last_request_offset;
5132 }
5133 }
5134 } else if ((uio->uio_offset + val_size) == last_request_offset) {
5135 /*
5136 * this transfer will finish this request, so...
5137 * let's try to read ahead if we're in
5138 * a sequential access pattern and we haven't
5139 * explicitly disabled it
5140 */
5141 if (rd_ahead_enabled) {
5142 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
5143 }
5144
5145 if (rap != NULL) {
5146 if (extent.e_addr < rap->cl_lastr) {
5147 rap->cl_maxra = 0;
5148 }
5149 rap->cl_lastr = extent.e_addr;
5150 }
5151 }
5152 if (iolock_inited == TRUE) {
5153 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
5154 }
5155
5156 if (iostate.io_error) {
5157 error = iostate.io_error;
5158 } else {
5159 u_int32_t io_requested;
5160
5161 io_requested = val_size;
5162
5163 retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
5164
5165 io_req_size -= (val_size - io_requested);
5166 }
5167 } else {
5168 if (iolock_inited == TRUE) {
5169 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
5170 }
5171 }
5172 if (start_pg < last_pg) {
5173 /*
5174 * compute the range of pages that we actually issued an I/O for
5175 * and either commit them as valid if the I/O succeeded
5176 * or abort them if the I/O failed or we're not supposed to
5177 * keep them in the cache
5178 */
5179 io_size = (last_pg - start_pg) * PAGE_SIZE;
5180
5181 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, upl, start_pg * PAGE_SIZE, io_size, error, 0);
5182
5183 if (error || (flags & IO_NOCACHE)) {
5184 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
5185 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
5186 } else {
5187 int commit_flags = UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY;
5188
5189 if (take_reference) {
5190 commit_flags |= UPL_COMMIT_INACTIVATE;
5191 } else {
5192 commit_flags |= UPL_COMMIT_SPECULATE;
5193 }
5194
5195 ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size, commit_flags);
5196 }
5197 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, start_pg * PAGE_SIZE, io_size, error, 0);
5198 }
5199 if ((last_pg - start_pg) < pages_in_upl) {
5200 /*
5201 * the set of pages that we issued an I/O for did not encompass
5202 * the entire upl... so just release these without modifying
5203 * their state
5204 */
5205 if (error) {
5206 if (leftover_upl_aborted) {
5207 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, (uio_last - start_pg) * PAGE_SIZE,
5208 UPL_ABORT_FREE_ON_EMPTY);
5209 } else {
5210 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
5211 }
5212 } else {
5213 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
5214 upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
5215
5216 /*
5217 * handle any valid pages at the beginning of
5218 * the upl... release these appropriately
5219 */
5220 cluster_read_upl_release(upl, 0, start_pg, take_reference);
5221
5222 /*
5223 * handle any valid pages immediately after the
5224 * pages we issued I/O for... ... release these appropriately
5225 */
5226 cluster_read_upl_release(upl, last_pg, uio_last, take_reference);
5227
5228 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, -1, -1, 0, 0);
5229 }
5230 }
5231 if (retval == 0) {
5232 retval = error;
5233 }
5234
5235 if (io_req_size) {
5236 uint32_t max_throttle_size = calculate_max_throttle_size(vp);
5237
5238 if (cluster_is_throttled(vp)) {
5239 /*
5240 * we're in the throttle window, at the very least
5241 * we want to limit the size of the I/O we're about
5242 * to issue
5243 */
5244 rd_ahead_enabled = 0;
5245 prefetch_enabled = 0;
5246 max_rd_size = max_throttle_size;
5247 } else {
5248 if (max_rd_size == max_throttle_size) {
5249 /*
5250 * coming out of throttled state
5251 */
5252 if (policy != THROTTLE_LEVEL_TIER3 && policy != THROTTLE_LEVEL_TIER2) {
5253 if (rap != NULL) {
5254 rd_ahead_enabled = 1;
5255 }
5256 prefetch_enabled = 1;
5257 }
5258 max_rd_size = max_prefetch;
5259 last_ioread_offset = 0;
5260 }
5261 }
5262 }
5263 }
5264 if (iolock_inited == TRUE) {
5265 /*
5266 * cluster_io returned an error after it
5267 * had already issued some I/O. we need
5268 * to wait for that I/O to complete before
5269 * we can destroy the iostate mutex...
5270 * 'retval' already contains the early error
5271 * so no need to pick it up from iostate.io_error
5272 */
5273 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
5274
5275 lck_mtx_destroy(&iostate.io_mtxp, &cl_mtx_grp);
5276 }
5277 if (rap != NULL) {
5278 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
5279 (int)uio->uio_offset, io_req_size, rap->cl_lastr, retval, 0);
5280
5281 lck_mtx_unlock(&rap->cl_lockr);
5282 } else {
5283 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
5284 (int)uio->uio_offset, io_req_size, 0, retval, 0);
5285 }
5286
5287 return retval;
5288 }
5289
5290 /*
5291 * We don't want another read/write lock for every vnode in the system
5292 * so we keep a hash of them here. There should never be very many of
5293 * these around at any point in time.
5294 */
5295 cl_direct_read_lock_t *
cluster_lock_direct_read(vnode_t vp,lck_rw_type_t type)5296 cluster_lock_direct_read(vnode_t vp, lck_rw_type_t type)
5297 {
5298 struct cl_direct_read_locks *head
5299 = &cl_direct_read_locks[(uintptr_t)vp / sizeof(*vp)
5300 % CL_DIRECT_READ_LOCK_BUCKETS];
5301
5302 struct cl_direct_read_lock *lck, *new_lck = NULL;
5303
5304 for (;;) {
5305 lck_spin_lock(&cl_direct_read_spin_lock);
5306
5307 LIST_FOREACH(lck, head, chain) {
5308 if (lck->vp == vp) {
5309 ++lck->ref_count;
5310 lck_spin_unlock(&cl_direct_read_spin_lock);
5311 if (new_lck) {
5312 // Someone beat us to it, ditch the allocation
5313 lck_rw_destroy(&new_lck->rw_lock, &cl_mtx_grp);
5314 kfree_type(cl_direct_read_lock_t, new_lck);
5315 }
5316 lck_rw_lock(&lck->rw_lock, type);
5317 return lck;
5318 }
5319 }
5320
5321 if (new_lck) {
5322 // Use the lock we allocated
5323 LIST_INSERT_HEAD(head, new_lck, chain);
5324 lck_spin_unlock(&cl_direct_read_spin_lock);
5325 lck_rw_lock(&new_lck->rw_lock, type);
5326 return new_lck;
5327 }
5328
5329 lck_spin_unlock(&cl_direct_read_spin_lock);
5330
5331 // Allocate a new lock
5332 new_lck = kalloc_type(cl_direct_read_lock_t, Z_WAITOK);
5333 lck_rw_init(&new_lck->rw_lock, &cl_mtx_grp, LCK_ATTR_NULL);
5334 new_lck->vp = vp;
5335 new_lck->ref_count = 1;
5336
5337 // Got to go round again
5338 }
5339 }
5340
5341 void
cluster_unlock_direct_read(cl_direct_read_lock_t * lck)5342 cluster_unlock_direct_read(cl_direct_read_lock_t *lck)
5343 {
5344 lck_rw_done(&lck->rw_lock);
5345
5346 lck_spin_lock(&cl_direct_read_spin_lock);
5347 if (lck->ref_count == 1) {
5348 LIST_REMOVE(lck, chain);
5349 lck_spin_unlock(&cl_direct_read_spin_lock);
5350 lck_rw_destroy(&lck->rw_lock, &cl_mtx_grp);
5351 kfree_type(cl_direct_read_lock_t, lck);
5352 } else {
5353 --lck->ref_count;
5354 lck_spin_unlock(&cl_direct_read_spin_lock);
5355 }
5356 }
5357
5358 static int
cluster_read_direct(vnode_t vp,struct uio * uio,off_t filesize,int * read_type,u_int32_t * read_length,int flags,int (* callback)(buf_t,void *),void * callback_arg)5359 cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
5360 int flags, int (*callback)(buf_t, void *), void *callback_arg)
5361 {
5362 upl_t upl = NULL;
5363 upl_page_info_t *pl;
5364 off_t max_io_size;
5365 size_t verify_block_size = 0;
5366 vm_offset_t upl_offset, vector_upl_offset = 0;
5367 upl_size_t upl_size = 0, vector_upl_size = 0;
5368 vm_size_t upl_needed_size;
5369 unsigned int pages_in_pl;
5370 upl_control_flags_t upl_flags;
5371 kern_return_t kret = KERN_SUCCESS;
5372 unsigned int i;
5373 int force_data_sync;
5374 int retval = 0;
5375 int no_zero_fill = 0;
5376 int io_flag = 0;
5377 int misaligned = 0;
5378 struct clios iostate;
5379 user_addr_t iov_base;
5380 u_int32_t io_req_size;
5381 u_int32_t offset_in_file;
5382 u_int32_t offset_in_iovbase;
5383 u_int32_t io_size;
5384 u_int32_t io_min;
5385 u_int32_t xsize;
5386 u_int32_t devblocksize;
5387 u_int32_t mem_alignment_mask;
5388 u_int32_t max_upl_size;
5389 u_int32_t max_rd_size;
5390 u_int32_t max_rd_ahead;
5391 u_int32_t max_vector_size;
5392 boolean_t io_throttled = FALSE;
5393
5394 u_int32_t vector_upl_iosize = 0;
5395 int issueVectorUPL = 0, useVectorUPL = (uio->uio_iovcnt > 1);
5396 off_t v_upl_uio_offset = 0;
5397 int vector_upl_index = 0;
5398 upl_t vector_upl = NULL;
5399 cl_direct_read_lock_t *lock = NULL;
5400 uint32_t verify_mask = 0;
5401
5402 assert(vm_map_page_shift(current_map()) >= PAGE_SHIFT);
5403
5404 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
5405 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
5406
5407 max_upl_size = cluster_max_io_size(vp->v_mount, CL_READ);
5408
5409 max_rd_size = max_upl_size;
5410
5411 if (__improbable(os_mul_overflow(max_rd_size, IO_SCALE(vp, 2),
5412 &max_rd_ahead) || (max_rd_ahead > overlapping_read_max))) {
5413 max_rd_ahead = overlapping_read_max;
5414 }
5415
5416 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO;
5417
5418 if (flags & IO_PASSIVE) {
5419 io_flag |= CL_PASSIVE;
5420 }
5421
5422 if (flags & IO_ENCRYPTED) {
5423 io_flag |= CL_RAW_ENCRYPTED;
5424 }
5425
5426 if (flags & IO_NOCACHE) {
5427 io_flag |= CL_NOCACHE;
5428 }
5429
5430 if (flags & IO_SKIP_ENCRYPTION) {
5431 io_flag |= CL_ENCRYPTED;
5432 }
5433
5434 iostate.io_completed = 0;
5435 iostate.io_issued = 0;
5436 iostate.io_error = 0;
5437 iostate.io_wanted = 0;
5438
5439 lck_mtx_init(&iostate.io_mtxp, &cl_mtx_grp, LCK_ATTR_NULL);
5440
5441 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
5442 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
5443
5444 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
5445 (int)devblocksize, (int)mem_alignment_mask, 0, 0, 0);
5446
5447 if (devblocksize == 1) {
5448 /*
5449 * the AFP client advertises a devblocksize of 1
5450 * however, its BLOCKMAP routine maps to physical
5451 * blocks that are PAGE_SIZE in size...
5452 * therefore we can't ask for I/Os that aren't page aligned
5453 * or aren't multiples of PAGE_SIZE in size
5454 * by setting devblocksize to PAGE_SIZE, we re-instate
5455 * the old behavior we had before the mem_alignment_mask
5456 * changes went in...
5457 */
5458 devblocksize = PAGE_SIZE;
5459 }
5460
5461 /*
5462 * We are going to need this uio for the prefaulting later
5463 * especially for the cases where multiple non-contiguous
5464 * iovs are passed into this routine.
5465 *
5466 * Note that we only want to prefault for direct IOs to userspace buffers,
5467 * not kernel buffers.
5468 */
5469 uio_t uio_acct = NULL;
5470 if (uio->uio_segflg != UIO_SYSSPACE) {
5471 uio_acct = uio_duplicate(uio);
5472 }
5473
5474 retval = VNOP_VERIFY(vp, 0, NULL, 0, &verify_block_size, NULL, VNODE_VERIFY_DEFAULT, NULL, NULL);
5475 if (retval) {
5476 verify_block_size = 0;
5477 } else if (verify_block_size) {
5478 assert((verify_block_size & (verify_block_size - 1)) == 0);
5479 verify_mask = verify_block_size - 1;
5480 }
5481
5482 next_dread:
5483 io_req_size = *read_length;
5484 iov_base = uio_curriovbase(uio);
5485
5486 offset_in_file = (u_int32_t)uio->uio_offset & (devblocksize - 1);
5487 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
5488
5489 if (vm_map_page_mask(current_map()) < PAGE_MASK) {
5490 /*
5491 * XXX TODO4K
5492 * Direct I/O might not work as expected from a 16k kernel space
5493 * to a 4k user space because each 4k chunk might point to
5494 * a different 16k physical page...
5495 * Let's go the "misaligned" way.
5496 */
5497 if (!misaligned) {
5498 DEBUG4K_VFS("forcing misaligned\n");
5499 }
5500 misaligned = 1;
5501 }
5502
5503 if (offset_in_file || offset_in_iovbase) {
5504 /*
5505 * one of the 2 important offsets is misaligned
5506 * so fire an I/O through the cache for this entire vector
5507 */
5508 misaligned = 1;
5509 }
5510 if (iov_base & (devblocksize - 1)) {
5511 /*
5512 * the offset in memory must be on a device block boundary
5513 * so that we can guarantee that we can generate an
5514 * I/O that ends on a page boundary in cluster_io
5515 */
5516 misaligned = 1;
5517 }
5518
5519 if (verify_block_size && !misaligned && ((uio_offset(uio) & verify_mask) || (uio_resid(uio) & verify_mask))) {
5520 /*
5521 * If the offset is not aligned to the verification block size
5522 * or the size is not aligned to the verification block size,
5523 * we simply send this through the cached i/o path as that is
5524 * what the Filesystem will end up doing anyway i.e. it will
5525 * read all the remaining data in order to verify it and then
5526 * discard the data it has read.
5527 */
5528 misaligned = 1;
5529 }
5530
5531 max_io_size = filesize - uio->uio_offset;
5532
5533 /*
5534 * The user must request IO in aligned chunks. If the
5535 * offset into the file is bad, or the userland pointer
5536 * is non-aligned, then we cannot service the encrypted IO request.
5537 */
5538 if (flags & IO_ENCRYPTED) {
5539 if (misaligned || (io_req_size & (devblocksize - 1))) {
5540 retval = EINVAL;
5541 }
5542
5543 max_io_size = roundup(max_io_size, devblocksize);
5544 }
5545
5546 if ((off_t)io_req_size > max_io_size) {
5547 io_req_size = (u_int32_t)max_io_size;
5548 }
5549
5550 /*
5551 * When we get to this point, we know...
5552 * -- the offset into the file is on a devblocksize boundary
5553 */
5554
5555 while (io_req_size && retval == 0) {
5556 u_int32_t io_start;
5557
5558 if (cluster_is_throttled(vp)) {
5559 uint32_t max_throttle_size = calculate_max_throttle_size(vp);
5560
5561 /*
5562 * we're in the throttle window, at the very least
5563 * we want to limit the size of the I/O we're about
5564 * to issue
5565 */
5566 max_rd_size = max_throttle_size;
5567 max_rd_ahead = max_throttle_size - 1;
5568 max_vector_size = max_throttle_size;
5569 } else {
5570 max_rd_size = max_upl_size;
5571 max_rd_ahead = max_rd_size * IO_SCALE(vp, 2);
5572 max_vector_size = MAX_VECTOR_UPL_SIZE;
5573 }
5574 io_start = io_size = io_req_size;
5575
5576 /*
5577 * First look for pages already in the cache
5578 * and move them to user space. But only do this
5579 * check if we are not retrieving encrypted data directly
5580 * from the filesystem; those blocks should never
5581 * be in the UBC.
5582 *
5583 * cluster_copy_ubc_data returns the resid
5584 * in io_size
5585 */
5586 if ((flags & IO_ENCRYPTED) == 0) {
5587 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_size, 0, 0);
5588 }
5589 /*
5590 * calculate the number of bytes actually copied
5591 * starting size - residual
5592 */
5593 xsize = io_start - io_size;
5594
5595 io_req_size -= xsize;
5596
5597 if (useVectorUPL && (xsize || (iov_base & PAGE_MASK))) {
5598 /*
5599 * We found something in the cache or we have an iov_base that's not
5600 * page-aligned.
5601 *
5602 * Issue all I/O's that have been collected within this Vectored UPL.
5603 */
5604 if (vector_upl_index) {
5605 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
5606 reset_vector_run_state();
5607 }
5608
5609 if (xsize) {
5610 useVectorUPL = 0;
5611 }
5612
5613 /*
5614 * After this point, if we are using the Vector UPL path and the base is
5615 * not page-aligned then the UPL with that base will be the first in the vector UPL.
5616 */
5617 }
5618
5619 /*
5620 * check to see if we are finished with this request.
5621 *
5622 * If we satisfied this IO already, then io_req_size will be 0.
5623 * Otherwise, see if the IO was mis-aligned and needs to go through
5624 * the UBC to deal with the 'tail'.
5625 *
5626 */
5627 if (io_req_size == 0 || (misaligned)) {
5628 /*
5629 * see if there's another uio vector to
5630 * process that's of type IO_DIRECT
5631 *
5632 * break out of while loop to get there
5633 */
5634 break;
5635 }
5636 /*
5637 * assume the request ends on a device block boundary
5638 */
5639 io_min = devblocksize;
5640
5641 /*
5642 * we can handle I/O's in multiples of the device block size
5643 * however, if io_size isn't a multiple of devblocksize we
5644 * want to clip it back to the nearest page boundary since
5645 * we are going to have to go through cluster_read_copy to
5646 * deal with the 'overhang'... by clipping it to a PAGE_SIZE
5647 * multiple, we avoid asking the drive for the same physical
5648 * blocks twice.. once for the partial page at the end of the
5649 * request and a 2nd time for the page we read into the cache
5650 * (which overlaps the end of the direct read) in order to
5651 * get at the overhang bytes
5652 */
5653 if (io_size & (devblocksize - 1)) {
5654 assert(!(flags & IO_ENCRYPTED));
5655 /*
5656 * Clip the request to the previous page size boundary
5657 * since request does NOT end on a device block boundary
5658 */
5659 io_size &= ~PAGE_MASK;
5660 io_min = PAGE_SIZE;
5661 }
5662 if (retval || io_size < io_min) {
5663 /*
5664 * either an error or we only have the tail left to
5665 * complete via the copy path...
5666 * we may have already spun some portion of this request
5667 * off as async requests... we need to wait for the I/O
5668 * to complete before returning
5669 */
5670 goto wait_for_dreads;
5671 }
5672
5673 /*
5674 * Don't re-check the UBC data if we are looking for uncached IO
5675 * or asking for encrypted blocks.
5676 */
5677 if ((flags & IO_ENCRYPTED) == 0) {
5678 if ((xsize = io_size) > max_rd_size) {
5679 xsize = max_rd_size;
5680 }
5681
5682 io_size = 0;
5683
5684 if (!lock) {
5685 /*
5686 * We hold a lock here between the time we check the
5687 * cache and the time we issue I/O. This saves us
5688 * from having to lock the pages in the cache. Not
5689 * all clients will care about this lock but some
5690 * clients may want to guarantee stability between
5691 * here and when the I/O is issued in which case they
5692 * will take the lock exclusively.
5693 */
5694 lock = cluster_lock_direct_read(vp, LCK_RW_TYPE_SHARED);
5695 }
5696
5697 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + xsize, UPL_ROP_ABSENT, (int *)&io_size);
5698
5699 if (io_size == 0) {
5700 /*
5701 * a page must have just come into the cache
5702 * since the first page in this range is no
5703 * longer absent, go back and re-evaluate
5704 */
5705 continue;
5706 }
5707 }
5708 if ((flags & IO_RETURN_ON_THROTTLE)) {
5709 if (cluster_is_throttled(vp) == THROTTLE_NOW) {
5710 if (!cluster_io_present_in_BC(vp, uio->uio_offset)) {
5711 /*
5712 * we're in the throttle window and at least 1 I/O
5713 * has already been issued by a throttleable thread
5714 * in this window, so return with EAGAIN to indicate
5715 * to the FS issuing the cluster_read call that it
5716 * should now throttle after dropping any locks
5717 */
5718 throttle_info_update_by_mount(vp->v_mount);
5719
5720 io_throttled = TRUE;
5721 goto wait_for_dreads;
5722 }
5723 }
5724 }
5725 if (io_size > max_rd_size) {
5726 io_size = max_rd_size;
5727 }
5728
5729 iov_base = uio_curriovbase(uio);
5730
5731 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
5732 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
5733
5734 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
5735 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
5736
5737 if (upl_offset == 0 && ((io_size & PAGE_MASK) == 0)) {
5738 no_zero_fill = 1;
5739 } else {
5740 no_zero_fill = 0;
5741 }
5742
5743 vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map;
5744 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
5745 pages_in_pl = 0;
5746 upl_size = (upl_size_t)upl_needed_size;
5747 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
5748 if (no_zero_fill) {
5749 upl_flags |= UPL_NOZEROFILL;
5750 }
5751 if (force_data_sync) {
5752 upl_flags |= UPL_FORCE_DATA_SYNC;
5753 }
5754
5755 kret = vm_map_create_upl(map,
5756 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
5757 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE);
5758
5759 if (kret != KERN_SUCCESS) {
5760 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
5761 (int)upl_offset, upl_size, io_size, kret, 0);
5762 /*
5763 * failed to get pagelist
5764 *
5765 * we may have already spun some portion of this request
5766 * off as async requests... we need to wait for the I/O
5767 * to complete before returning
5768 */
5769 goto wait_for_dreads;
5770 }
5771 pages_in_pl = upl_size / PAGE_SIZE;
5772 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
5773
5774 for (i = 0; i < pages_in_pl; i++) {
5775 if (!upl_page_present(pl, i)) {
5776 break;
5777 }
5778 }
5779 if (i == pages_in_pl) {
5780 break;
5781 }
5782
5783 ubc_upl_abort(upl, 0);
5784 }
5785 if (force_data_sync >= 3) {
5786 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
5787 (int)upl_offset, upl_size, io_size, kret, 0);
5788
5789 goto wait_for_dreads;
5790 }
5791 /*
5792 * Consider the possibility that upl_size wasn't satisfied.
5793 */
5794 if (upl_size < upl_needed_size) {
5795 if (upl_size && upl_offset == 0) {
5796 io_size = upl_size;
5797 } else {
5798 io_size = 0;
5799 }
5800 }
5801 if (io_size == 0) {
5802 ubc_upl_abort(upl, 0);
5803 goto wait_for_dreads;
5804 }
5805 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
5806 (int)upl_offset, upl_size, io_size, kret, 0);
5807
5808 if (useVectorUPL) {
5809 vm_offset_t end_off = ((iov_base + io_size) & PAGE_MASK);
5810 if (end_off) {
5811 issueVectorUPL = 1;
5812 }
5813 /*
5814 * After this point, if we are using a vector UPL, then
5815 * either all the UPL elements end on a page boundary OR
5816 * this UPL is the last element because it does not end
5817 * on a page boundary.
5818 */
5819 }
5820
5821 /*
5822 * request asynchronously so that we can overlap
5823 * the preparation of the next I/O
5824 * if there are already too many outstanding reads
5825 * wait until some have completed before issuing the next read
5826 */
5827 cluster_iostate_wait(&iostate, max_rd_ahead, "cluster_read_direct");
5828
5829 if (iostate.io_error) {
5830 /*
5831 * one of the earlier reads we issued ran into a hard error
5832 * don't issue any more reads, cleanup the UPL
5833 * that was just created but not used, then
5834 * go wait for any other reads to complete before
5835 * returning the error to the caller
5836 */
5837 ubc_upl_abort(upl, 0);
5838
5839 goto wait_for_dreads;
5840 }
5841 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
5842 upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
5843
5844 if (!useVectorUPL) {
5845 if (no_zero_fill) {
5846 io_flag &= ~CL_PRESERVE;
5847 } else {
5848 io_flag |= CL_PRESERVE;
5849 }
5850
5851 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
5852 } else {
5853 if (!vector_upl_index) {
5854 vector_upl = vector_upl_create(upl_offset, uio->uio_iovcnt);
5855 v_upl_uio_offset = uio->uio_offset;
5856 vector_upl_offset = upl_offset;
5857 }
5858
5859 vector_upl_set_subupl(vector_upl, upl, upl_size);
5860 vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
5861 vector_upl_index++;
5862 vector_upl_size += upl_size;
5863 vector_upl_iosize += io_size;
5864
5865 if (issueVectorUPL || vector_upl_index == vector_upl_max_upls(vector_upl) || vector_upl_size >= max_vector_size) {
5866 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
5867 reset_vector_run_state();
5868 }
5869 }
5870
5871 if (lock) {
5872 // We don't need to wait for the I/O to complete
5873 cluster_unlock_direct_read(lock);
5874 lock = NULL;
5875 }
5876
5877 /*
5878 * update the uio structure
5879 */
5880 if ((flags & IO_ENCRYPTED) && (max_io_size < io_size)) {
5881 uio_update(uio, (user_size_t)max_io_size);
5882 } else {
5883 uio_update(uio, (user_size_t)io_size);
5884 }
5885
5886 io_req_size -= io_size;
5887
5888 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
5889 upl, (int)uio->uio_offset, io_req_size, retval, 0);
5890 } /* end while */
5891
5892 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0 && uio->uio_offset < filesize) {
5893 retval = cluster_io_type(uio, read_type, read_length, 0);
5894
5895 if (retval == 0 && *read_type == IO_DIRECT) {
5896 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
5897 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
5898
5899 goto next_dread;
5900 }
5901 }
5902
5903 wait_for_dreads:
5904
5905 if (retval == 0 && iostate.io_error == 0 && useVectorUPL && vector_upl_index) {
5906 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
5907 reset_vector_run_state();
5908 }
5909
5910 // We don't need to wait for the I/O to complete
5911 if (lock) {
5912 cluster_unlock_direct_read(lock);
5913 }
5914
5915 /*
5916 * make sure all async reads that are part of this stream
5917 * have completed before we return
5918 */
5919 cluster_iostate_wait(&iostate, 0, "cluster_read_direct");
5920
5921 if (iostate.io_error) {
5922 retval = iostate.io_error;
5923 }
5924
5925 lck_mtx_destroy(&iostate.io_mtxp, &cl_mtx_grp);
5926
5927 if (io_throttled == TRUE && retval == 0) {
5928 retval = EAGAIN;
5929 }
5930
5931 vm_map_offset_t current_page_size, current_page_mask;
5932 current_page_size = vm_map_page_size(current_map());
5933 current_page_mask = vm_map_page_mask(current_map());
5934 if (uio_acct) {
5935 assert(uio_acct->uio_segflg != UIO_SYSSPACE);
5936 off_t bytes_to_prefault = 0, bytes_prefaulted = 0;
5937 user_addr_t curr_iov_base = 0;
5938 user_addr_t curr_iov_end = 0;
5939 user_size_t curr_iov_len = 0;
5940
5941 bytes_to_prefault = uio_offset(uio) - uio_offset(uio_acct);
5942
5943 for (; bytes_prefaulted < bytes_to_prefault;) {
5944 curr_iov_base = uio_curriovbase(uio_acct);
5945 curr_iov_len = MIN(uio_curriovlen(uio_acct), bytes_to_prefault - bytes_prefaulted);
5946 curr_iov_end = curr_iov_base + curr_iov_len;
5947
5948 for (; curr_iov_base < curr_iov_end;) {
5949 /*
5950 * This is specifically done for pmap accounting purposes.
5951 * vm_pre_fault() will call vm_fault() to enter the page into
5952 * the pmap if there isn't _a_ physical page for that VA already.
5953 */
5954 vm_pre_fault(vm_map_trunc_page(curr_iov_base, current_page_mask), VM_PROT_READ);
5955 curr_iov_base += current_page_size;
5956 bytes_prefaulted += current_page_size;
5957 }
5958 /*
5959 * Use update instead of advance so we can see how many iovs we processed.
5960 */
5961 uio_update(uio_acct, curr_iov_len);
5962 }
5963 uio_free(uio_acct);
5964 uio_acct = NULL;
5965 }
5966
5967 if (io_req_size && retval == 0) {
5968 /*
5969 * we couldn't handle the tail of this request in DIRECT mode
5970 * so fire it through the copy path
5971 */
5972 if (flags & IO_ENCRYPTED) {
5973 /*
5974 * We cannot fall back to the copy path for encrypted I/O. If this
5975 * happens, there is something wrong with the user buffer passed
5976 * down.
5977 */
5978 retval = EFAULT;
5979 } else {
5980 retval = cluster_read_copy(vp, uio, io_req_size, filesize, flags, callback, callback_arg);
5981 }
5982
5983 *read_type = IO_UNKNOWN;
5984 }
5985 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
5986 (int)uio->uio_offset, (int)uio_resid(uio), io_req_size, retval, 0);
5987
5988 return retval;
5989 }
5990
5991
5992 static int
cluster_read_contig(vnode_t vp,struct uio * uio,off_t filesize,int * read_type,u_int32_t * read_length,int (* callback)(buf_t,void *),void * callback_arg,int flags)5993 cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
5994 int (*callback)(buf_t, void *), void *callback_arg, int flags)
5995 {
5996 upl_page_info_t *pl;
5997 upl_t upl[MAX_VECTS];
5998 vm_offset_t upl_offset;
5999 addr64_t dst_paddr = 0;
6000 user_addr_t iov_base;
6001 off_t max_size;
6002 upl_size_t upl_size;
6003 vm_size_t upl_needed_size;
6004 mach_msg_type_number_t pages_in_pl;
6005 upl_control_flags_t upl_flags;
6006 kern_return_t kret;
6007 struct clios iostate;
6008 int error = 0;
6009 int cur_upl = 0;
6010 int num_upl = 0;
6011 int n;
6012 u_int32_t xsize;
6013 u_int32_t io_size;
6014 u_int32_t devblocksize;
6015 u_int32_t mem_alignment_mask;
6016 u_int32_t tail_size = 0;
6017 int bflag;
6018
6019 if (flags & IO_PASSIVE) {
6020 bflag = CL_PASSIVE;
6021 } else {
6022 bflag = 0;
6023 }
6024
6025 if (flags & IO_NOCACHE) {
6026 bflag |= CL_NOCACHE;
6027 }
6028
6029 /*
6030 * When we enter this routine, we know
6031 * -- the read_length will not exceed the current iov_len
6032 * -- the target address is physically contiguous for read_length
6033 */
6034 cluster_syncup(vp, filesize, callback, callback_arg, PUSH_SYNC);
6035
6036 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
6037 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
6038
6039 iostate.io_completed = 0;
6040 iostate.io_issued = 0;
6041 iostate.io_error = 0;
6042 iostate.io_wanted = 0;
6043
6044 lck_mtx_init(&iostate.io_mtxp, &cl_mtx_grp, LCK_ATTR_NULL);
6045
6046 next_cread:
6047 io_size = *read_length;
6048
6049 max_size = filesize - uio->uio_offset;
6050
6051 if (io_size > max_size) {
6052 io_size = (u_int32_t)max_size;
6053 }
6054
6055 iov_base = uio_curriovbase(uio);
6056
6057 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
6058 upl_needed_size = upl_offset + io_size;
6059
6060 pages_in_pl = 0;
6061 upl_size = (upl_size_t)upl_needed_size;
6062 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
6063
6064
6065 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START,
6066 (int)upl_offset, (int)upl_size, (int)iov_base, io_size, 0);
6067
6068 vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map;
6069 kret = vm_map_get_upl(map,
6070 vm_map_trunc_page(iov_base, vm_map_page_mask(map)),
6071 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0);
6072
6073 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END,
6074 (int)upl_offset, upl_size, io_size, kret, 0);
6075
6076 if (kret != KERN_SUCCESS) {
6077 /*
6078 * failed to get pagelist
6079 */
6080 error = EINVAL;
6081 goto wait_for_creads;
6082 }
6083 num_upl++;
6084
6085 if (upl_size < upl_needed_size) {
6086 /*
6087 * The upl_size wasn't satisfied.
6088 */
6089 error = EINVAL;
6090 goto wait_for_creads;
6091 }
6092 pl = ubc_upl_pageinfo(upl[cur_upl]);
6093
6094 dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << PAGE_SHIFT) + (addr64_t)upl_offset;
6095
6096 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
6097 u_int32_t head_size;
6098
6099 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
6100
6101 if (head_size > io_size) {
6102 head_size = io_size;
6103 }
6104
6105 error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ, callback, callback_arg);
6106
6107 if (error) {
6108 goto wait_for_creads;
6109 }
6110
6111 upl_offset += head_size;
6112 dst_paddr += head_size;
6113 io_size -= head_size;
6114
6115 iov_base += head_size;
6116 }
6117 if ((u_int32_t)iov_base & mem_alignment_mask) {
6118 /*
6119 * request doesn't set up on a memory boundary
6120 * the underlying DMA engine can handle...
6121 * return an error instead of going through
6122 * the slow copy path since the intent of this
6123 * path is direct I/O to device memory
6124 */
6125 error = EINVAL;
6126 goto wait_for_creads;
6127 }
6128
6129 tail_size = io_size & (devblocksize - 1);
6130
6131 io_size -= tail_size;
6132
6133 while (io_size && error == 0) {
6134 if (io_size > MAX_IO_CONTIG_SIZE) {
6135 xsize = MAX_IO_CONTIG_SIZE;
6136 } else {
6137 xsize = io_size;
6138 }
6139 /*
6140 * request asynchronously so that we can overlap
6141 * the preparation of the next I/O... we'll do
6142 * the commit after all the I/O has completed
6143 * since its all issued against the same UPL
6144 * if there are already too many outstanding reads
6145 * wait until some have completed before issuing the next
6146 */
6147 cluster_iostate_wait(&iostate, MAX_IO_CONTIG_SIZE * IO_SCALE(vp, 2), "cluster_read_contig");
6148
6149 if (iostate.io_error) {
6150 /*
6151 * one of the earlier reads we issued ran into a hard error
6152 * don't issue any more reads...
6153 * go wait for any other reads to complete before
6154 * returning the error to the caller
6155 */
6156 goto wait_for_creads;
6157 }
6158 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize,
6159 CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC | bflag,
6160 (buf_t)NULL, &iostate, callback, callback_arg);
6161 /*
6162 * The cluster_io read was issued successfully,
6163 * update the uio structure
6164 */
6165 if (error == 0) {
6166 uio_update(uio, (user_size_t)xsize);
6167
6168 dst_paddr += xsize;
6169 upl_offset += xsize;
6170 io_size -= xsize;
6171 }
6172 }
6173 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS && uio->uio_offset < filesize) {
6174 error = cluster_io_type(uio, read_type, read_length, 0);
6175
6176 if (error == 0 && *read_type == IO_CONTIG) {
6177 cur_upl++;
6178 goto next_cread;
6179 }
6180 } else {
6181 *read_type = IO_UNKNOWN;
6182 }
6183
6184 wait_for_creads:
6185 /*
6186 * make sure all async reads that are part of this stream
6187 * have completed before we proceed
6188 */
6189 cluster_iostate_wait(&iostate, 0, "cluster_read_contig");
6190
6191 if (iostate.io_error) {
6192 error = iostate.io_error;
6193 }
6194
6195 lck_mtx_destroy(&iostate.io_mtxp, &cl_mtx_grp);
6196
6197 if (error == 0 && tail_size) {
6198 error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ, callback, callback_arg);
6199 }
6200
6201 for (n = 0; n < num_upl; n++) {
6202 /*
6203 * just release our hold on each physically contiguous
6204 * region without changing any state
6205 */
6206 ubc_upl_abort(upl[n], 0);
6207 }
6208
6209 return error;
6210 }
6211
6212
6213 static int
cluster_io_type(struct uio * uio,int * io_type,u_int32_t * io_length,u_int32_t min_length)6214 cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length)
6215 {
6216 user_size_t iov_len;
6217 user_addr_t iov_base = 0;
6218 upl_t upl;
6219 upl_size_t upl_size;
6220 upl_control_flags_t upl_flags;
6221 int retval = 0;
6222
6223 /*
6224 * skip over any emtpy vectors
6225 */
6226 uio_update(uio, (user_size_t)0);
6227
6228 iov_len = MIN(uio_curriovlen(uio), uio_resid(uio));
6229
6230 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_START, uio, (int)iov_len, 0, 0, 0);
6231
6232 if (iov_len) {
6233 iov_base = uio_curriovbase(uio);
6234 /*
6235 * make sure the size of the vector isn't too big...
6236 * internally, we want to handle all of the I/O in
6237 * chunk sizes that fit in a 32 bit int
6238 */
6239 if (iov_len > (user_size_t)MAX_IO_REQUEST_SIZE) {
6240 upl_size = MAX_IO_REQUEST_SIZE;
6241 } else {
6242 upl_size = (u_int32_t)iov_len;
6243 }
6244
6245 upl_flags = UPL_QUERY_OBJECT_TYPE;
6246
6247 vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map;
6248 if ((vm_map_get_upl(map,
6249 vm_map_trunc_page(iov_base, vm_map_page_mask(map)),
6250 &upl_size, &upl, NULL, NULL, &upl_flags, VM_KERN_MEMORY_FILE, 0)) != KERN_SUCCESS) {
6251 /*
6252 * the user app must have passed in an invalid address
6253 */
6254 retval = EFAULT;
6255 }
6256 if (upl_size == 0) {
6257 retval = EFAULT;
6258 }
6259
6260 *io_length = upl_size;
6261
6262 if (upl_flags & UPL_PHYS_CONTIG) {
6263 *io_type = IO_CONTIG;
6264 } else if (iov_len >= min_length) {
6265 *io_type = IO_DIRECT;
6266 } else {
6267 *io_type = IO_COPY;
6268 }
6269 } else {
6270 /*
6271 * nothing left to do for this uio
6272 */
6273 *io_length = 0;
6274 *io_type = IO_UNKNOWN;
6275 }
6276 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_END, iov_base, *io_type, *io_length, retval, 0);
6277
6278 if (*io_type == IO_DIRECT &&
6279 vm_map_page_shift(current_map()) < PAGE_SHIFT) {
6280 /* no direct I/O for sub-page-size address spaces */
6281 DEBUG4K_VFS("io_type IO_DIRECT -> IO_COPY\n");
6282 *io_type = IO_COPY;
6283 }
6284
6285 return retval;
6286 }
6287
6288
6289 /*
6290 * generate advisory I/O's in the largest chunks possible
6291 * the completed pages will be released into the VM cache
6292 */
6293 int
advisory_read(vnode_t vp,off_t filesize,off_t f_offset,int resid)6294 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
6295 {
6296 return advisory_read_ext(vp, filesize, f_offset, resid, NULL, NULL, CL_PASSIVE);
6297 }
6298
6299 int
advisory_read_ext(vnode_t vp,off_t filesize,off_t f_offset,int resid,int (* callback)(buf_t,void *),void * callback_arg,int bflag)6300 advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
6301 {
6302 upl_page_info_t *pl;
6303 upl_t upl = NULL;
6304 vm_offset_t upl_offset;
6305 int upl_size;
6306 off_t upl_f_offset;
6307 int start_offset;
6308 int start_pg;
6309 int last_pg;
6310 int pages_in_upl;
6311 off_t max_size;
6312 int io_size;
6313 kern_return_t kret;
6314 int retval = 0;
6315 int issued_io;
6316 int skip_range;
6317 uint32_t max_io_size;
6318
6319
6320 if (!UBCINFOEXISTS(vp)) {
6321 return EINVAL;
6322 }
6323
6324 if (f_offset < 0 || resid < 0) {
6325 return EINVAL;
6326 }
6327
6328 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
6329
6330 if (disk_conditioner_mount_is_ssd(vp->v_mount)) {
6331 if (max_io_size > speculative_prefetch_max_iosize) {
6332 max_io_size = speculative_prefetch_max_iosize;
6333 }
6334 }
6335
6336 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START,
6337 (int)f_offset, resid, (int)filesize, 0, 0);
6338
6339 while (resid && f_offset < filesize && retval == 0) {
6340 /*
6341 * compute the size of the upl needed to encompass
6342 * the requested read... limit each call to cluster_io
6343 * to the maximum UPL size... cluster_io will clip if
6344 * this exceeds the maximum io_size for the device,
6345 * make sure to account for
6346 * a starting offset that's not page aligned
6347 */
6348 start_offset = (int)(f_offset & PAGE_MASK_64);
6349 upl_f_offset = f_offset - (off_t)start_offset;
6350 max_size = filesize - f_offset;
6351
6352 if (resid < max_size) {
6353 io_size = resid;
6354 } else {
6355 io_size = (int)max_size;
6356 }
6357
6358 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
6359 if ((uint32_t)upl_size > max_io_size) {
6360 upl_size = max_io_size;
6361 }
6362
6363 skip_range = 0;
6364 /*
6365 * return the number of contiguously present pages in the cache
6366 * starting at upl_f_offset within the file
6367 */
6368 ubc_range_op(vp, upl_f_offset, upl_f_offset + upl_size, UPL_ROP_PRESENT, &skip_range);
6369
6370 if (skip_range) {
6371 /*
6372 * skip over pages already present in the cache
6373 */
6374 io_size = skip_range - start_offset;
6375
6376 f_offset += io_size;
6377 resid -= io_size;
6378
6379 if (skip_range == upl_size) {
6380 continue;
6381 }
6382 /*
6383 * have to issue some real I/O
6384 * at this point, we know it's starting on a page boundary
6385 * because we've skipped over at least the first page in the request
6386 */
6387 start_offset = 0;
6388 upl_f_offset += skip_range;
6389 upl_size -= skip_range;
6390 }
6391 pages_in_upl = upl_size / PAGE_SIZE;
6392
6393 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START,
6394 upl, (int)upl_f_offset, upl_size, start_offset, 0);
6395
6396 kret = ubc_create_upl_kernel(vp,
6397 upl_f_offset,
6398 upl_size,
6399 &upl,
6400 &pl,
6401 UPL_RET_ONLY_ABSENT | UPL_SET_LITE,
6402 VM_KERN_MEMORY_FILE);
6403 if (kret != KERN_SUCCESS) {
6404 return retval;
6405 }
6406 issued_io = 0;
6407
6408 /*
6409 * before we start marching forward, we must make sure we end on
6410 * a present page, otherwise we will be working with a freed
6411 * upl
6412 */
6413 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
6414 if (upl_page_present(pl, last_pg)) {
6415 break;
6416 }
6417 }
6418 pages_in_upl = last_pg + 1;
6419
6420
6421 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_END,
6422 upl, (int)upl_f_offset, upl_size, start_offset, 0);
6423
6424
6425 for (last_pg = 0; last_pg < pages_in_upl;) {
6426 /*
6427 * scan from the beginning of the upl looking for the first
6428 * page that is present.... this will become the first page in
6429 * the request we're going to make to 'cluster_io'... if all
6430 * of the pages are absent, we won't call through to 'cluster_io'
6431 */
6432 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
6433 if (upl_page_present(pl, start_pg)) {
6434 break;
6435 }
6436 }
6437
6438 /*
6439 * scan from the starting present page looking for an absent
6440 * page before the end of the upl is reached, if we
6441 * find one, then it will terminate the range of pages being
6442 * presented to 'cluster_io'
6443 */
6444 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
6445 if (!upl_page_present(pl, last_pg)) {
6446 break;
6447 }
6448 }
6449
6450 if (last_pg > start_pg) {
6451 /*
6452 * we found a range of pages that must be filled
6453 * if the last page in this range is the last page of the file
6454 * we may have to clip the size of it to keep from reading past
6455 * the end of the last physical block associated with the file
6456 */
6457 upl_offset = start_pg * PAGE_SIZE;
6458 io_size = (last_pg - start_pg) * PAGE_SIZE;
6459
6460 if ((off_t)(upl_f_offset + upl_offset + io_size) > filesize) {
6461 io_size = (int)(filesize - (upl_f_offset + upl_offset));
6462 }
6463
6464 /*
6465 * issue an asynchronous read to cluster_io
6466 */
6467 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
6468 CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
6469
6470 issued_io = 1;
6471 }
6472 }
6473 if (issued_io == 0) {
6474 ubc_upl_abort(upl, 0);
6475 }
6476
6477 io_size = upl_size - start_offset;
6478
6479 if (io_size > resid) {
6480 io_size = resid;
6481 }
6482 f_offset += io_size;
6483 resid -= io_size;
6484 }
6485
6486 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_END,
6487 (int)f_offset, resid, retval, 0, 0);
6488
6489 return retval;
6490 }
6491
6492
6493 int
cluster_push(vnode_t vp,int flags)6494 cluster_push(vnode_t vp, int flags)
6495 {
6496 return cluster_push_ext(vp, flags, NULL, NULL);
6497 }
6498
6499
6500 int
cluster_push_ext(vnode_t vp,int flags,int (* callback)(buf_t,void *),void * callback_arg)6501 cluster_push_ext(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg)
6502 {
6503 return cluster_push_err(vp, flags, callback, callback_arg, NULL);
6504 }
6505
6506 /* write errors via err, but return the number of clusters written */
6507 extern uint32_t system_inshutdown;
6508 uint32_t cl_sparse_push_error = 0;
6509 int
cluster_push_err(vnode_t vp,int flags,int (* callback)(buf_t,void *),void * callback_arg,int * err)6510 cluster_push_err(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg, int *err)
6511 {
6512 int retval;
6513 int my_sparse_wait = 0;
6514 struct cl_writebehind *wbp;
6515 int local_err = 0;
6516
6517 if (err) {
6518 *err = 0;
6519 }
6520
6521 if (!UBCINFOEXISTS(vp)) {
6522 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, kdebug_vnode(vp), flags, 0, -1, 0);
6523 return 0;
6524 }
6525 /* return if deferred write is set */
6526 if (((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) && (flags & IO_DEFWRITE)) {
6527 return 0;
6528 }
6529 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) == NULL) {
6530 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, kdebug_vnode(vp), flags, 0, -2, 0);
6531 return 0;
6532 }
6533 if (!ISSET(flags, IO_SYNC) && wbp->cl_number == 0 && wbp->cl_scmap == NULL) {
6534 lck_mtx_unlock(&wbp->cl_lockw);
6535
6536 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, kdebug_vnode(vp), flags, 0, -3, 0);
6537 return 0;
6538 }
6539 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
6540 wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
6541
6542 /*
6543 * if we have an fsync in progress, we don't want to allow any additional
6544 * sync/fsync/close(s) to occur until it finishes.
6545 * note that its possible for writes to continue to occur to this file
6546 * while we're waiting and also once the fsync starts to clean if we're
6547 * in the sparse map case
6548 */
6549 while (wbp->cl_sparse_wait) {
6550 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START, kdebug_vnode(vp), 0, 0, 0, 0);
6551
6552 msleep((caddr_t)&wbp->cl_sparse_wait, &wbp->cl_lockw, PRIBIO + 1, "cluster_push_ext", NULL);
6553
6554 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END, kdebug_vnode(vp), 0, 0, 0, 0);
6555 }
6556 if (flags & IO_SYNC) {
6557 my_sparse_wait = 1;
6558 wbp->cl_sparse_wait = 1;
6559
6560 /*
6561 * this is an fsync (or equivalent)... we must wait for any existing async
6562 * cleaning operations to complete before we evaulate the current state
6563 * and finish cleaning... this insures that all writes issued before this
6564 * fsync actually get cleaned to the disk before this fsync returns
6565 */
6566 while (wbp->cl_sparse_pushes) {
6567 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 98)) | DBG_FUNC_START, kdebug_vnode(vp), 0, 0, 0, 0);
6568
6569 msleep((caddr_t)&wbp->cl_sparse_pushes, &wbp->cl_lockw, PRIBIO + 1, "cluster_push_ext", NULL);
6570
6571 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 98)) | DBG_FUNC_END, kdebug_vnode(vp), 0, 0, 0, 0);
6572 }
6573 }
6574 if (wbp->cl_scmap) {
6575 void *scmap;
6576
6577 if (wbp->cl_sparse_pushes < SPARSE_PUSH_LIMIT) {
6578 scmap = wbp->cl_scmap;
6579 wbp->cl_scmap = NULL;
6580
6581 wbp->cl_sparse_pushes++;
6582
6583 lck_mtx_unlock(&wbp->cl_lockw);
6584
6585 retval = sparse_cluster_push(wbp, &scmap, vp, ubc_getsize(vp), PUSH_ALL, flags, callback, callback_arg, FALSE);
6586
6587 lck_mtx_lock(&wbp->cl_lockw);
6588
6589 wbp->cl_sparse_pushes--;
6590
6591 if (retval) {
6592 if (wbp->cl_scmap != NULL) {
6593 /*
6594 * panic("cluster_push_err: Expected NULL cl_scmap\n");
6595 *
6596 * This can happen if we get an error from the underlying FS
6597 * e.g. ENOSPC, EPERM or EIO etc. We hope that these errors
6598 * are transient and the I/Os will succeed at a later point.
6599 *
6600 * The tricky part here is that a new sparse cluster has been
6601 * allocated and tracking a different set of dirty pages. So these
6602 * pages are not going to be pushed out with the next sparse_cluster_push.
6603 * An explicit msync or file close will, however, push the pages out.
6604 *
6605 * What if those calls still don't work? And so, during shutdown we keep
6606 * trying till we succeed...
6607 */
6608
6609 if (system_inshutdown) {
6610 if ((retval == ENOSPC) && (vp->v_mount->mnt_flag & (MNT_LOCAL | MNT_REMOVABLE)) == MNT_LOCAL) {
6611 os_atomic_inc(&cl_sparse_push_error, relaxed);
6612 }
6613 } else {
6614 vfs_drt_control(&scmap, 0); /* emit stats and free this memory. Dirty pages stay intact. */
6615 scmap = NULL;
6616 }
6617 } else {
6618 wbp->cl_scmap = scmap;
6619 }
6620 }
6621
6622 if (wbp->cl_sparse_wait && wbp->cl_sparse_pushes == 0) {
6623 wakeup((caddr_t)&wbp->cl_sparse_pushes);
6624 }
6625 } else {
6626 retval = sparse_cluster_push(wbp, &(wbp->cl_scmap), vp, ubc_getsize(vp), PUSH_ALL, flags, callback, callback_arg, FALSE);
6627 }
6628
6629 local_err = retval;
6630
6631 if (err) {
6632 *err = retval;
6633 }
6634 retval = 1;
6635 } else {
6636 retval = cluster_try_push(wbp, vp, ubc_getsize(vp), PUSH_ALL, flags, callback, callback_arg, &local_err, FALSE);
6637 if (err) {
6638 *err = local_err;
6639 }
6640 }
6641 lck_mtx_unlock(&wbp->cl_lockw);
6642
6643 if (flags & IO_SYNC) {
6644 (void)vnode_waitforwrites(vp, 0, 0, 0, "cluster_push");
6645 }
6646
6647 if (my_sparse_wait) {
6648 /*
6649 * I'm the owner of the serialization token
6650 * clear it and wakeup anyone that is waiting
6651 * for me to finish
6652 */
6653 lck_mtx_lock(&wbp->cl_lockw);
6654
6655 wbp->cl_sparse_wait = 0;
6656 wakeup((caddr_t)&wbp->cl_sparse_wait);
6657
6658 lck_mtx_unlock(&wbp->cl_lockw);
6659 }
6660 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
6661 wbp->cl_scmap, wbp->cl_number, retval, local_err, 0);
6662
6663 return retval;
6664 }
6665
6666
6667 __private_extern__ void
cluster_release(struct ubc_info * ubc)6668 cluster_release(struct ubc_info *ubc)
6669 {
6670 struct cl_writebehind *wbp;
6671 struct cl_readahead *rap;
6672
6673 if ((wbp = ubc->cl_wbehind)) {
6674 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, ubc, wbp->cl_scmap, 0, 0, 0);
6675
6676 if (wbp->cl_scmap) {
6677 vfs_drt_control(&(wbp->cl_scmap), 0);
6678 }
6679 lck_mtx_destroy(&wbp->cl_lockw, &cl_mtx_grp);
6680 zfree(cl_wr_zone, wbp);
6681 ubc->cl_wbehind = NULL;
6682 } else {
6683 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, ubc, 0, 0, 0, 0);
6684 }
6685
6686 if ((rap = ubc->cl_rahead)) {
6687 lck_mtx_destroy(&rap->cl_lockr, &cl_mtx_grp);
6688 zfree(cl_rd_zone, rap);
6689 ubc->cl_rahead = NULL;
6690 }
6691
6692 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_END, ubc, rap, wbp, 0, 0);
6693 }
6694
6695
6696 static int
cluster_try_push(struct cl_writebehind * wbp,vnode_t vp,off_t EOF,int push_flag,int io_flags,int (* callback)(buf_t,void *),void * callback_arg,int * err,boolean_t vm_initiated)6697 cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int io_flags, int (*callback)(buf_t, void *), void *callback_arg, int *err, boolean_t vm_initiated)
6698 {
6699 int cl_index;
6700 int cl_index1;
6701 int min_index;
6702 int cl_len;
6703 int cl_pushed = 0;
6704 struct cl_wextent l_clusters[MAX_CLUSTERS];
6705 u_int max_cluster_pgcount;
6706 int error = 0;
6707
6708 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
6709 /*
6710 * the write behind context exists and has
6711 * already been locked...
6712 */
6713 if (wbp->cl_number == 0) {
6714 /*
6715 * no clusters to push
6716 * return number of empty slots
6717 */
6718 return MAX_CLUSTERS;
6719 }
6720
6721 /*
6722 * make a local 'sorted' copy of the clusters
6723 * and clear wbp->cl_number so that new clusters can
6724 * be developed
6725 */
6726 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
6727 for (min_index = -1, cl_index1 = 0; cl_index1 < wbp->cl_number; cl_index1++) {
6728 if (wbp->cl_clusters[cl_index1].b_addr == wbp->cl_clusters[cl_index1].e_addr) {
6729 continue;
6730 }
6731 if (min_index == -1) {
6732 min_index = cl_index1;
6733 } else if (wbp->cl_clusters[cl_index1].b_addr < wbp->cl_clusters[min_index].b_addr) {
6734 min_index = cl_index1;
6735 }
6736 }
6737 if (min_index == -1) {
6738 break;
6739 }
6740
6741 l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
6742 l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
6743 l_clusters[cl_index].io_flags = wbp->cl_clusters[min_index].io_flags;
6744
6745 wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
6746 }
6747 wbp->cl_number = 0;
6748
6749 cl_len = cl_index;
6750
6751 /* skip switching to the sparse cluster mechanism if on diskimage */
6752 if (((push_flag & PUSH_DELAY) && cl_len == MAX_CLUSTERS) &&
6753 !(vp->v_mount->mnt_kern_flag & MNTK_VIRTUALDEV)) {
6754 int i;
6755
6756 /*
6757 * determine if we appear to be writing the file sequentially
6758 * if not, by returning without having pushed any clusters
6759 * we will cause this vnode to be pushed into the sparse cluster mechanism
6760 * used for managing more random I/O patterns
6761 *
6762 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
6763 * that's why we're in try_push with PUSH_DELAY...
6764 *
6765 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
6766 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
6767 * so we can just make a simple pass through, up to, but not including the last one...
6768 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
6769 * are sequential
6770 *
6771 * we let the last one be partial as long as it was adjacent to the previous one...
6772 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
6773 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
6774 */
6775 for (i = 0; i < MAX_CLUSTERS - 1; i++) {
6776 if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != max_cluster_pgcount) {
6777 goto dont_try;
6778 }
6779 if (l_clusters[i].e_addr != l_clusters[i + 1].b_addr) {
6780 goto dont_try;
6781 }
6782 }
6783 }
6784 if (vm_initiated == TRUE) {
6785 lck_mtx_unlock(&wbp->cl_lockw);
6786 }
6787
6788 for (cl_index = 0; cl_index < cl_len; cl_index++) {
6789 int flags;
6790 struct cl_extent cl;
6791 int retval;
6792
6793 flags = io_flags & (IO_PASSIVE | IO_CLOSE);
6794
6795 /*
6796 * try to push each cluster in turn...
6797 */
6798 if (l_clusters[cl_index].io_flags & CLW_IONOCACHE) {
6799 flags |= IO_NOCACHE;
6800 }
6801
6802 if (l_clusters[cl_index].io_flags & CLW_IOPASSIVE) {
6803 flags |= IO_PASSIVE;
6804 }
6805
6806 if (push_flag & PUSH_SYNC) {
6807 flags |= IO_SYNC;
6808 }
6809
6810 cl.b_addr = l_clusters[cl_index].b_addr;
6811 cl.e_addr = l_clusters[cl_index].e_addr;
6812
6813 retval = cluster_push_now(vp, &cl, EOF, flags, callback, callback_arg, vm_initiated);
6814
6815 if (retval == 0) {
6816 cl_pushed++;
6817
6818 l_clusters[cl_index].b_addr = 0;
6819 l_clusters[cl_index].e_addr = 0;
6820 } else if (error == 0) {
6821 error = retval;
6822 }
6823
6824 if (!(push_flag & PUSH_ALL)) {
6825 break;
6826 }
6827 }
6828 if (vm_initiated == TRUE) {
6829 lck_mtx_lock(&wbp->cl_lockw);
6830 }
6831
6832 if (err) {
6833 *err = error;
6834 }
6835
6836 dont_try:
6837 if (cl_len > cl_pushed) {
6838 /*
6839 * we didn't push all of the clusters, so
6840 * lets try to merge them back in to the vnode
6841 */
6842 if ((MAX_CLUSTERS - wbp->cl_number) < (cl_len - cl_pushed)) {
6843 /*
6844 * we picked up some new clusters while we were trying to
6845 * push the old ones... this can happen because I've dropped
6846 * the vnode lock... the sum of the
6847 * leftovers plus the new cluster count exceeds our ability
6848 * to represent them, so switch to the sparse cluster mechanism
6849 *
6850 * collect the active public clusters...
6851 */
6852 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg, vm_initiated);
6853
6854 for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
6855 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr) {
6856 continue;
6857 }
6858 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
6859 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
6860 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
6861
6862 cl_index1++;
6863 }
6864 /*
6865 * update the cluster count
6866 */
6867 wbp->cl_number = cl_index1;
6868
6869 /*
6870 * and collect the original clusters that were moved into the
6871 * local storage for sorting purposes
6872 */
6873 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg, vm_initiated);
6874 } else {
6875 /*
6876 * we've got room to merge the leftovers back in
6877 * just append them starting at the next 'hole'
6878 * represented by wbp->cl_number
6879 */
6880 for (cl_index = 0, cl_index1 = wbp->cl_number; cl_index < cl_len; cl_index++) {
6881 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr) {
6882 continue;
6883 }
6884
6885 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
6886 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
6887 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
6888
6889 cl_index1++;
6890 }
6891 /*
6892 * update the cluster count
6893 */
6894 wbp->cl_number = cl_index1;
6895 }
6896 }
6897 return MAX_CLUSTERS - wbp->cl_number;
6898 }
6899
6900
6901
6902 static int
cluster_push_now(vnode_t vp,struct cl_extent * cl,off_t EOF,int flags,int (* callback)(buf_t,void *),void * callback_arg,boolean_t vm_initiated)6903 cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags,
6904 int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated)
6905 {
6906 upl_page_info_t *pl;
6907 upl_t upl;
6908 vm_offset_t upl_offset;
6909 int upl_size;
6910 off_t upl_f_offset;
6911 int pages_in_upl;
6912 int start_pg;
6913 int last_pg;
6914 int io_size;
6915 int io_flags;
6916 int upl_flags;
6917 int bflag;
6918 int size;
6919 int error = 0;
6920 int retval;
6921 kern_return_t kret;
6922
6923 if (flags & IO_PASSIVE) {
6924 bflag = CL_PASSIVE;
6925 } else {
6926 bflag = 0;
6927 }
6928
6929 if (flags & IO_SKIP_ENCRYPTION) {
6930 bflag |= CL_ENCRYPTED;
6931 }
6932
6933 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
6934 (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
6935
6936 if ((pages_in_upl = (int)(cl->e_addr - cl->b_addr)) == 0) {
6937 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 0, 0, 0, 0);
6938
6939 return 0;
6940 }
6941 upl_size = pages_in_upl * PAGE_SIZE;
6942 upl_f_offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
6943
6944 if (upl_f_offset + upl_size >= EOF) {
6945 if (upl_f_offset >= EOF) {
6946 /*
6947 * must have truncated the file and missed
6948 * clearing a dangling cluster (i.e. it's completely
6949 * beyond the new EOF
6950 */
6951 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 1, 0, 0, 0);
6952
6953 return 0;
6954 }
6955 size = (int)(EOF - upl_f_offset);
6956
6957 upl_size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
6958 pages_in_upl = upl_size / PAGE_SIZE;
6959 } else {
6960 size = upl_size;
6961 }
6962
6963
6964 if (vm_initiated) {
6965 vnode_pageout(vp, NULL, (upl_offset_t)0, upl_f_offset, (upl_size_t)upl_size,
6966 UPL_MSYNC | UPL_VNODE_PAGER | UPL_KEEPCACHED, &error);
6967
6968 return error;
6969 }
6970 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, size, 0, 0, 0);
6971
6972 /*
6973 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
6974 *
6975 * - only pages that are currently dirty are returned... these are the ones we need to clean
6976 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
6977 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
6978 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
6979 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
6980 *
6981 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
6982 */
6983
6984 if ((vp->v_flag & VNOCACHE_DATA) || (flags & IO_NOCACHE)) {
6985 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE | UPL_WILL_BE_DUMPED;
6986 } else {
6987 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE;
6988 }
6989
6990 kret = ubc_create_upl_kernel(vp,
6991 upl_f_offset,
6992 upl_size,
6993 &upl,
6994 &pl,
6995 upl_flags,
6996 VM_KERN_MEMORY_FILE);
6997 if (kret != KERN_SUCCESS) {
6998 panic("cluster_push: failed to get pagelist");
6999 }
7000
7001 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, upl, upl_f_offset, 0, 0, 0);
7002
7003 /*
7004 * since we only asked for the dirty pages back
7005 * it's possible that we may only get a few or even none, so...
7006 * before we start marching forward, we must make sure we know
7007 * where the last present page is in the UPL, otherwise we could
7008 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
7009 * employed by commit_range and abort_range.
7010 */
7011 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
7012 if (upl_page_present(pl, last_pg)) {
7013 break;
7014 }
7015 }
7016 pages_in_upl = last_pg + 1;
7017
7018 if (pages_in_upl == 0) {
7019 ubc_upl_abort(upl, 0);
7020
7021 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 2, 0, 0, 0);
7022 return 0;
7023 }
7024
7025 for (last_pg = 0; last_pg < pages_in_upl;) {
7026 /*
7027 * find the next dirty page in the UPL
7028 * this will become the first page in the
7029 * next I/O to generate
7030 */
7031 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
7032 if (upl_dirty_page(pl, start_pg)) {
7033 break;
7034 }
7035 if (upl_page_present(pl, start_pg)) {
7036 /*
7037 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
7038 * just release these unchanged since we're not going
7039 * to steal them or change their state
7040 */
7041 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
7042 }
7043 }
7044 if (start_pg >= pages_in_upl) {
7045 /*
7046 * done... no more dirty pages to push
7047 */
7048 break;
7049 }
7050 if (start_pg > last_pg) {
7051 /*
7052 * skipped over some non-dirty pages
7053 */
7054 size -= ((start_pg - last_pg) * PAGE_SIZE);
7055 }
7056
7057 /*
7058 * find a range of dirty pages to write
7059 */
7060 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
7061 if (!upl_dirty_page(pl, last_pg)) {
7062 break;
7063 }
7064 }
7065 upl_offset = start_pg * PAGE_SIZE;
7066
7067 io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
7068
7069 io_flags = CL_THROTTLE | CL_COMMIT | CL_AGE | bflag;
7070
7071 if (!(flags & IO_SYNC)) {
7072 io_flags |= CL_ASYNC;
7073 }
7074
7075 if (flags & IO_CLOSE) {
7076 io_flags |= CL_CLOSE;
7077 }
7078
7079 if (flags & IO_NOCACHE) {
7080 io_flags |= CL_NOCACHE;
7081 }
7082
7083 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
7084 io_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
7085
7086 if (error == 0 && retval) {
7087 error = retval;
7088 }
7089
7090 size -= io_size;
7091 }
7092 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 3, error, 0, 0);
7093
7094 return error;
7095 }
7096
7097
7098 /*
7099 * sparse_cluster_switch is called with the write behind lock held
7100 */
7101 static int
sparse_cluster_switch(struct cl_writebehind * wbp,vnode_t vp,off_t EOF,int (* callback)(buf_t,void *),void * callback_arg,boolean_t vm_initiated)7102 sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated)
7103 {
7104 int cl_index;
7105 int error = 0;
7106
7107 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_START, kdebug_vnode(vp), wbp->cl_scmap, wbp->cl_number, 0, 0);
7108
7109 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
7110 int flags;
7111 struct cl_extent cl;
7112
7113 for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
7114 if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, NULL, &flags) == KERN_SUCCESS) {
7115 if (flags & UPL_POP_DIRTY) {
7116 cl.e_addr = cl.b_addr + 1;
7117
7118 error = sparse_cluster_add(wbp, &(wbp->cl_scmap), vp, &cl, EOF, callback, callback_arg, vm_initiated);
7119
7120 if (error) {
7121 break;
7122 }
7123 }
7124 }
7125 }
7126 }
7127 wbp->cl_number -= cl_index;
7128
7129 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_END, kdebug_vnode(vp), wbp->cl_scmap, wbp->cl_number, error, 0);
7130
7131 return error;
7132 }
7133
7134
7135 /*
7136 * sparse_cluster_push must be called with the write-behind lock held if the scmap is
7137 * still associated with the write-behind context... however, if the scmap has been disassociated
7138 * from the write-behind context (the cluster_push case), the wb lock is not held
7139 */
7140 static int
sparse_cluster_push(struct cl_writebehind * wbp,void ** scmap,vnode_t vp,off_t EOF,int push_flag,int io_flags,int (* callback)(buf_t,void *),void * callback_arg,boolean_t vm_initiated)7141 sparse_cluster_push(struct cl_writebehind *wbp, void **scmap, vnode_t vp, off_t EOF, int push_flag,
7142 int io_flags, int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated)
7143 {
7144 struct cl_extent cl;
7145 off_t offset;
7146 u_int length;
7147 void *l_scmap;
7148 int error = 0;
7149
7150 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, kdebug_vnode(vp), (*scmap), 0, push_flag, 0);
7151
7152 if (push_flag & PUSH_ALL) {
7153 vfs_drt_control(scmap, 1);
7154 }
7155
7156 l_scmap = *scmap;
7157
7158 for (;;) {
7159 int retval;
7160
7161 if (vfs_drt_get_cluster(scmap, &offset, &length) != KERN_SUCCESS) {
7162 /*
7163 * Not finding anything to push will return KERN_FAILURE.
7164 * Confusing since it isn't really a failure. But that's the
7165 * reason we don't set 'error' here like we do below.
7166 */
7167 break;
7168 }
7169
7170 if (vm_initiated == TRUE) {
7171 lck_mtx_unlock(&wbp->cl_lockw);
7172 }
7173
7174 cl.b_addr = (daddr64_t)(offset / PAGE_SIZE_64);
7175 cl.e_addr = (daddr64_t)((offset + length) / PAGE_SIZE_64);
7176
7177 retval = cluster_push_now(vp, &cl, EOF, io_flags, callback, callback_arg, vm_initiated);
7178 if (error == 0 && retval) {
7179 error = retval;
7180 }
7181
7182 if (vm_initiated == TRUE) {
7183 lck_mtx_lock(&wbp->cl_lockw);
7184
7185 if (*scmap != l_scmap) {
7186 break;
7187 }
7188 }
7189
7190 if (error) {
7191 if (vfs_drt_mark_pages(scmap, offset, length, NULL) != KERN_SUCCESS) {
7192 panic("Failed to restore dirty state on failure");
7193 }
7194
7195 break;
7196 }
7197
7198 if (!(push_flag & PUSH_ALL)) {
7199 break;
7200 }
7201 }
7202 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, kdebug_vnode(vp), (*scmap), error, 0, 0);
7203
7204 return error;
7205 }
7206
7207
7208 /*
7209 * sparse_cluster_add is called with the write behind lock held
7210 */
7211 static int
sparse_cluster_add(struct cl_writebehind * wbp,void ** scmap,vnode_t vp,struct cl_extent * cl,off_t EOF,int (* callback)(buf_t,void *),void * callback_arg,boolean_t vm_initiated)7212 sparse_cluster_add(struct cl_writebehind *wbp, void **scmap, vnode_t vp, struct cl_extent *cl, off_t EOF,
7213 int (*callback)(buf_t, void *), void *callback_arg, boolean_t vm_initiated)
7214 {
7215 u_int new_dirty;
7216 u_int length;
7217 off_t offset;
7218 int error = 0;
7219 int push_flag = 0; /* Is this a valid value? */
7220
7221 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_START, (*scmap), 0, cl->b_addr, (int)cl->e_addr, 0);
7222
7223 offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
7224 length = ((u_int)(cl->e_addr - cl->b_addr)) * PAGE_SIZE;
7225
7226 while (vfs_drt_mark_pages(scmap, offset, length, &new_dirty) != KERN_SUCCESS) {
7227 /*
7228 * no room left in the map
7229 * only a partial update was done
7230 * push out some pages and try again
7231 */
7232
7233 if (vfs_get_scmap_push_behavior_internal(scmap, &push_flag)) {
7234 push_flag = 0;
7235 }
7236
7237 error = sparse_cluster_push(wbp, scmap, vp, EOF, push_flag, 0, callback, callback_arg, vm_initiated);
7238
7239 if (error) {
7240 break;
7241 }
7242
7243 offset += (new_dirty * PAGE_SIZE_64);
7244 length -= (new_dirty * PAGE_SIZE);
7245 }
7246 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_END, kdebug_vnode(vp), (*scmap), error, 0, 0);
7247
7248 return error;
7249 }
7250
7251
7252 static int
cluster_align_phys_io(vnode_t vp,struct uio * uio,addr64_t usr_paddr,u_int32_t xsize,int flags,int (* callback)(buf_t,void *),void * callback_arg)7253 cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
7254 {
7255 upl_page_info_t *pl;
7256 upl_t upl;
7257 addr64_t ubc_paddr;
7258 kern_return_t kret;
7259 int error = 0;
7260 int did_read = 0;
7261 int abort_flags;
7262 int upl_flags;
7263 int bflag;
7264
7265 if (flags & IO_PASSIVE) {
7266 bflag = CL_PASSIVE;
7267 } else {
7268 bflag = 0;
7269 }
7270
7271 if (flags & IO_NOCACHE) {
7272 bflag |= CL_NOCACHE;
7273 }
7274
7275 upl_flags = UPL_SET_LITE;
7276
7277 if (!(flags & CL_READ)) {
7278 /*
7279 * "write" operation: let the UPL subsystem know
7280 * that we intend to modify the buffer cache pages
7281 * we're gathering.
7282 */
7283 upl_flags |= UPL_WILL_MODIFY;
7284 } else {
7285 /*
7286 * indicate that there is no need to pull the
7287 * mapping for this page... we're only going
7288 * to read from it, not modify it.
7289 */
7290 upl_flags |= UPL_FILE_IO;
7291 }
7292 kret = ubc_create_upl_kernel(vp,
7293 uio->uio_offset & ~PAGE_MASK_64,
7294 PAGE_SIZE,
7295 &upl,
7296 &pl,
7297 upl_flags,
7298 VM_KERN_MEMORY_FILE);
7299
7300 if (kret != KERN_SUCCESS) {
7301 return EINVAL;
7302 }
7303
7304 if (!upl_valid_page(pl, 0)) {
7305 /*
7306 * issue a synchronous read to cluster_io
7307 */
7308 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
7309 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
7310 if (error) {
7311 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
7312
7313 return error;
7314 }
7315 did_read = 1;
7316 }
7317 ubc_paddr = ((addr64_t)upl_phys_page(pl, 0) << PAGE_SHIFT) + (addr64_t)(uio->uio_offset & PAGE_MASK_64);
7318
7319 /*
7320 * NOTE: There is no prototype for the following in BSD. It, and the definitions
7321 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
7322 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
7323 * way to do so without exporting them to kexts as well.
7324 */
7325 if (flags & CL_READ) {
7326 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
7327 copypv(ubc_paddr, usr_paddr, xsize, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
7328 } else {
7329 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
7330 copypv(usr_paddr, ubc_paddr, xsize, 2 | 1 | 8); /* Copy physical to physical and flush the source */
7331 }
7332 if (!(flags & CL_READ) || (upl_valid_page(pl, 0) && upl_dirty_page(pl, 0))) {
7333 /*
7334 * issue a synchronous write to cluster_io
7335 */
7336 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
7337 bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
7338 }
7339 if (error == 0) {
7340 uio_update(uio, (user_size_t)xsize);
7341 }
7342
7343 if (did_read) {
7344 abort_flags = UPL_ABORT_FREE_ON_EMPTY;
7345 } else {
7346 abort_flags = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
7347 }
7348
7349 ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
7350
7351 return error;
7352 }
7353
7354 int
cluster_copy_upl_data(struct uio * uio,upl_t upl,int upl_offset,int * io_resid)7355 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
7356 {
7357 int pg_offset;
7358 int pg_index;
7359 int csize;
7360 int segflg;
7361 int retval = 0;
7362 int xsize;
7363 upl_page_info_t *pl;
7364 int dirty_count;
7365
7366 xsize = *io_resid;
7367
7368 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
7369 (int)uio->uio_offset, upl_offset, xsize, 0, 0);
7370
7371 segflg = uio->uio_segflg;
7372
7373 switch (segflg) {
7374 case UIO_USERSPACE32:
7375 case UIO_USERISPACE32:
7376 uio->uio_segflg = UIO_PHYS_USERSPACE32;
7377 break;
7378
7379 case UIO_USERSPACE:
7380 case UIO_USERISPACE:
7381 uio->uio_segflg = UIO_PHYS_USERSPACE;
7382 break;
7383
7384 case UIO_USERSPACE64:
7385 case UIO_USERISPACE64:
7386 uio->uio_segflg = UIO_PHYS_USERSPACE64;
7387 break;
7388
7389 case UIO_SYSSPACE:
7390 uio->uio_segflg = UIO_PHYS_SYSSPACE;
7391 break;
7392 }
7393 pl = ubc_upl_pageinfo(upl);
7394
7395 pg_index = upl_offset / PAGE_SIZE;
7396 pg_offset = upl_offset & PAGE_MASK;
7397 csize = min(PAGE_SIZE - pg_offset, xsize);
7398
7399 dirty_count = 0;
7400 while (xsize && retval == 0) {
7401 addr64_t paddr;
7402 ppnum_t pn = upl_phys_page(pl, pg_index);
7403
7404 paddr = ((addr64_t)pn << PAGE_SHIFT) + pg_offset;
7405 if ((uio->uio_rw == UIO_WRITE) && (upl_dirty_page(pl, pg_index) == FALSE)) {
7406 dirty_count++;
7407 }
7408
7409 /* such phyiscal pages should never be restricted pages */
7410 if (pmap_is_page_restricted(pn)) {
7411 panic("%s: cannot uiomove64 into a restricted page", __func__);
7412 }
7413
7414 retval = uiomove64(paddr, csize, uio);
7415
7416 pg_index += 1;
7417 pg_offset = 0;
7418 xsize -= csize;
7419 csize = min(PAGE_SIZE, xsize);
7420 }
7421 *io_resid = xsize;
7422
7423 uio->uio_segflg = segflg;
7424
7425 if (dirty_count) {
7426 task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_DEFERRED, upl_lookup_vnode(upl));
7427 }
7428
7429 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
7430 (int)uio->uio_offset, xsize, retval, segflg, 0);
7431
7432 return retval;
7433 }
7434
7435
7436 int
cluster_copy_ubc_data(vnode_t vp,struct uio * uio,int * io_resid,int mark_dirty)7437 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
7438 {
7439 return cluster_copy_ubc_data_internal(vp, uio, io_resid, mark_dirty, 1);
7440 }
7441
7442
7443 static int
cluster_copy_ubc_data_internal(vnode_t vp,struct uio * uio,int * io_resid,int mark_dirty,int take_reference)7444 cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference)
7445 {
7446 int segflg;
7447 int io_size;
7448 int xsize;
7449 int start_offset;
7450 int retval = 0;
7451 memory_object_control_t control;
7452
7453 io_size = *io_resid;
7454
7455 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
7456 (int)uio->uio_offset, io_size, mark_dirty, take_reference, 0);
7457
7458 control = ubc_getobject(vp, UBC_FLAGS_NONE);
7459
7460 if (control == MEMORY_OBJECT_CONTROL_NULL) {
7461 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
7462 (int)uio->uio_offset, io_size, retval, 3, 0);
7463
7464 return 0;
7465 }
7466 segflg = uio->uio_segflg;
7467
7468 switch (segflg) {
7469 case UIO_USERSPACE32:
7470 case UIO_USERISPACE32:
7471 uio->uio_segflg = UIO_PHYS_USERSPACE32;
7472 break;
7473
7474 case UIO_USERSPACE64:
7475 case UIO_USERISPACE64:
7476 uio->uio_segflg = UIO_PHYS_USERSPACE64;
7477 break;
7478
7479 case UIO_USERSPACE:
7480 case UIO_USERISPACE:
7481 uio->uio_segflg = UIO_PHYS_USERSPACE;
7482 break;
7483
7484 case UIO_SYSSPACE:
7485 uio->uio_segflg = UIO_PHYS_SYSSPACE;
7486 break;
7487 }
7488
7489 if ((io_size = *io_resid)) {
7490 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
7491 xsize = (int)uio_resid(uio);
7492
7493 retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset, uio,
7494 start_offset, io_size, mark_dirty, take_reference);
7495 xsize -= uio_resid(uio);
7496
7497 int num_bytes_copied = xsize;
7498 if (num_bytes_copied && uio_rw(uio)) {
7499 task_update_logical_writes(current_task(), num_bytes_copied, TASK_WRITE_DEFERRED, vp);
7500 }
7501 io_size -= xsize;
7502 }
7503 uio->uio_segflg = segflg;
7504 *io_resid = io_size;
7505
7506 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
7507 (int)uio->uio_offset, io_size, retval, 0x80000000 | segflg, 0);
7508
7509 return retval;
7510 }
7511
7512
7513 int
is_file_clean(vnode_t vp,off_t filesize)7514 is_file_clean(vnode_t vp, off_t filesize)
7515 {
7516 off_t f_offset;
7517 int flags;
7518 int total_dirty = 0;
7519
7520 for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
7521 if (ubc_page_op(vp, f_offset, 0, NULL, &flags) == KERN_SUCCESS) {
7522 if (flags & UPL_POP_DIRTY) {
7523 total_dirty++;
7524 }
7525 }
7526 }
7527 if (total_dirty) {
7528 return EINVAL;
7529 }
7530
7531 return 0;
7532 }
7533
7534
7535
7536 /*
7537 * Dirty region tracking/clustering mechanism.
7538 *
7539 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
7540 * dirty regions within a larger space (file). It is primarily intended to
7541 * support clustering in large files with many dirty areas.
7542 *
7543 * The implementation assumes that the dirty regions are pages.
7544 *
7545 * To represent dirty pages within the file, we store bit vectors in a
7546 * variable-size circular hash.
7547 */
7548
7549 /*
7550 * Bitvector size. This determines the number of pages we group in a
7551 * single hashtable entry. Each hashtable entry is aligned to this
7552 * size within the file.
7553 */
7554 #define DRT_BITVECTOR_PAGES ((1024 * 256) / PAGE_SIZE)
7555
7556 /*
7557 * File offset handling.
7558 *
7559 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
7560 * the correct formula is (~((DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1))
7561 */
7562 #define DRT_ADDRESS_MASK (~((DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1))
7563 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
7564
7565 /*
7566 * Hashtable address field handling.
7567 *
7568 * The low-order bits of the hashtable address are used to conserve
7569 * space.
7570 *
7571 * DRT_HASH_COUNT_MASK must be large enough to store the range
7572 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
7573 * to indicate that the bucket is actually unoccupied.
7574 */
7575 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
7576 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
7577 do { \
7578 (scm)->scm_hashtable[(i)].dhe_control = \
7579 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
7580 } while (0)
7581 #define DRT_HASH_COUNT_MASK 0x1ff
7582 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
7583 #define DRT_HASH_SET_COUNT(scm, i, c) \
7584 do { \
7585 (scm)->scm_hashtable[(i)].dhe_control = \
7586 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
7587 } while (0)
7588 #define DRT_HASH_CLEAR(scm, i) \
7589 do { \
7590 (scm)->scm_hashtable[(i)].dhe_control = 0; \
7591 } while (0)
7592 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
7593 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
7594 #define DRT_HASH_COPY(oscm, oi, scm, i) \
7595 do { \
7596 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
7597 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
7598 } while(0);
7599
7600
7601 #if !defined(XNU_TARGET_OS_OSX)
7602 /*
7603 * Hash table moduli.
7604 *
7605 * Since the hashtable entry's size is dependent on the size of
7606 * the bitvector, and since the hashtable size is constrained to
7607 * both being prime and fitting within the desired allocation
7608 * size, these values need to be manually determined.
7609 *
7610 * For DRT_BITVECTOR_SIZE = 64, the entry size is 16 bytes.
7611 *
7612 * The small hashtable allocation is 4096 bytes, so the modulus is 251.
7613 * The large hashtable allocation is 32768 bytes, so the modulus is 2039.
7614 * The xlarge hashtable allocation is 131072 bytes, so the modulus is 8179.
7615 */
7616
7617 #define DRT_HASH_SMALL_MODULUS 251
7618 #define DRT_HASH_LARGE_MODULUS 2039
7619 #define DRT_HASH_XLARGE_MODULUS 8179
7620
7621 /*
7622 * Physical memory required before the large hash modulus is permitted.
7623 *
7624 * On small memory systems, the large hash modulus can lead to phsyical
7625 * memory starvation, so we avoid using it there.
7626 */
7627 #define DRT_HASH_LARGE_MEMORY_REQUIRED (1024LL * 1024LL * 1024LL) /* 1GiB */
7628 #define DRT_HASH_XLARGE_MEMORY_REQUIRED (8 * 1024LL * 1024LL * 1024LL) /* 8GiB */
7629
7630 #define DRT_SMALL_ALLOCATION 4096 /* 80 bytes spare */
7631 #define DRT_LARGE_ALLOCATION 32768 /* 144 bytes spare */
7632 #define DRT_XLARGE_ALLOCATION 131072 /* 208 bytes spare */
7633
7634 #else /* XNU_TARGET_OS_OSX */
7635 /*
7636 * Hash table moduli.
7637 *
7638 * Since the hashtable entry's size is dependent on the size of
7639 * the bitvector, and since the hashtable size is constrained to
7640 * both being prime and fitting within the desired allocation
7641 * size, these values need to be manually determined.
7642 *
7643 * For DRT_BITVECTOR_SIZE = 64, the entry size is 16 bytes.
7644 *
7645 * The small hashtable allocation is 16384 bytes, so the modulus is 1019.
7646 * The large hashtable allocation is 131072 bytes, so the modulus is 8179.
7647 * The xlarge hashtable allocation is 524288 bytes, so the modulus is 32749.
7648 */
7649
7650 #define DRT_HASH_SMALL_MODULUS 1019
7651 #define DRT_HASH_LARGE_MODULUS 8179
7652 #define DRT_HASH_XLARGE_MODULUS 32749
7653
7654 /*
7655 * Physical memory required before the large hash modulus is permitted.
7656 *
7657 * On small memory systems, the large hash modulus can lead to phsyical
7658 * memory starvation, so we avoid using it there.
7659 */
7660 #define DRT_HASH_LARGE_MEMORY_REQUIRED (4 * 1024LL * 1024LL * 1024LL) /* 4GiB */
7661 #define DRT_HASH_XLARGE_MEMORY_REQUIRED (32 * 1024LL * 1024LL * 1024LL) /* 32GiB */
7662
7663 #define DRT_SMALL_ALLOCATION 16384 /* 80 bytes spare */
7664 #define DRT_LARGE_ALLOCATION 131072 /* 208 bytes spare */
7665 #define DRT_XLARGE_ALLOCATION 524288 /* 304 bytes spare */
7666
7667 #endif /* ! XNU_TARGET_OS_OSX */
7668
7669 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
7670
7671 /*
7672 * Hashtable entry.
7673 */
7674 struct vfs_drt_hashentry {
7675 u_int64_t dhe_control;
7676 /*
7677 * dhe_bitvector was declared as dhe_bitvector[DRT_BITVECTOR_PAGES / 32];
7678 * DRT_BITVECTOR_PAGES is defined as ((1024 * 256) / PAGE_SIZE)
7679 * Since PAGE_SIZE is only known at boot time,
7680 * -define MAX_DRT_BITVECTOR_PAGES for smallest supported page size (4k)
7681 * -declare dhe_bitvector array for largest possible length
7682 */
7683 #define MAX_DRT_BITVECTOR_PAGES (1024 * 256)/( 4 * 1024)
7684 u_int32_t dhe_bitvector[MAX_DRT_BITVECTOR_PAGES / 32];
7685 };
7686
7687 /*
7688 * Hashtable bitvector handling.
7689 *
7690 * Bitvector fields are 32 bits long.
7691 */
7692
7693 #define DRT_HASH_SET_BIT(scm, i, bit) \
7694 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
7695
7696 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
7697 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
7698
7699 #define DRT_HASH_TEST_BIT(scm, i, bit) \
7700 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
7701
7702 #define DRT_BITVECTOR_CLEAR(scm, i) \
7703 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (MAX_DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
7704
7705 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
7706 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
7707 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
7708 (MAX_DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
7709
7710 /*
7711 * Dirty Region Tracking structure.
7712 *
7713 * The hashtable is allocated entirely inside the DRT structure.
7714 *
7715 * The hash is a simple circular prime modulus arrangement, the structure
7716 * is resized from small to large if it overflows.
7717 */
7718
7719 struct vfs_drt_clustermap {
7720 u_int32_t scm_magic; /* sanity/detection */
7721 #define DRT_SCM_MAGIC 0x12020003
7722 u_int32_t scm_modulus; /* current ring size */
7723 u_int32_t scm_buckets; /* number of occupied buckets */
7724 u_int32_t scm_lastclean; /* last entry we cleaned */
7725 u_int32_t scm_iskips; /* number of slot skips */
7726
7727 struct vfs_drt_hashentry scm_hashtable[0];
7728 };
7729
7730
7731 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
7732 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
7733
7734 /*
7735 * Debugging codes and arguments.
7736 */
7737 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
7738 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
7739 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
7740 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
7741 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
7742 * dirty */
7743 /* 0, setcount */
7744 /* 1 (clean, no map) */
7745 /* 2 (map alloc fail) */
7746 /* 3, resid (partial) */
7747 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
7748 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
7749 * lastclean, iskips */
7750
7751
7752 static kern_return_t vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp);
7753 static kern_return_t vfs_drt_free_map(struct vfs_drt_clustermap *cmap);
7754 static kern_return_t vfs_drt_search_index(struct vfs_drt_clustermap *cmap,
7755 u_int64_t offset, int *indexp);
7756 static kern_return_t vfs_drt_get_index(struct vfs_drt_clustermap **cmapp,
7757 u_int64_t offset,
7758 int *indexp,
7759 int recursed);
7760 static kern_return_t vfs_drt_do_mark_pages(
7761 void **cmapp,
7762 u_int64_t offset,
7763 u_int length,
7764 u_int *setcountp,
7765 int dirty);
7766 static void vfs_drt_trace(
7767 struct vfs_drt_clustermap *cmap,
7768 int code,
7769 int arg1,
7770 int arg2,
7771 int arg3,
7772 int arg4);
7773
7774
7775 /*
7776 * Allocate and initialise a sparse cluster map.
7777 *
7778 * Will allocate a new map, resize or compact an existing map.
7779 *
7780 * XXX we should probably have at least one intermediate map size,
7781 * as the 1:16 ratio seems a bit drastic.
7782 */
7783 static kern_return_t
vfs_drt_alloc_map(struct vfs_drt_clustermap ** cmapp)7784 vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
7785 {
7786 struct vfs_drt_clustermap *cmap = NULL, *ocmap = NULL;
7787 kern_return_t kret = KERN_SUCCESS;
7788 u_int64_t offset = 0;
7789 u_int32_t i = 0;
7790 int modulus_size = 0, map_size = 0, active_buckets = 0, index = 0, copycount = 0;
7791
7792 ocmap = NULL;
7793 if (cmapp != NULL) {
7794 ocmap = *cmapp;
7795 }
7796
7797 /*
7798 * Decide on the size of the new map.
7799 */
7800 if (ocmap == NULL) {
7801 modulus_size = DRT_HASH_SMALL_MODULUS;
7802 map_size = DRT_SMALL_ALLOCATION;
7803 } else {
7804 /* count the number of active buckets in the old map */
7805 active_buckets = 0;
7806 for (i = 0; i < ocmap->scm_modulus; i++) {
7807 if (!DRT_HASH_VACANT(ocmap, i) &&
7808 (DRT_HASH_GET_COUNT(ocmap, i) != 0)) {
7809 active_buckets++;
7810 }
7811 }
7812 /*
7813 * If we're currently using the small allocation, check to
7814 * see whether we should grow to the large one.
7815 */
7816 if (ocmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
7817 /*
7818 * If the ring is nearly full and we are allowed to
7819 * use the large modulus, upgrade.
7820 */
7821 if ((active_buckets > (DRT_HASH_SMALL_MODULUS - 5)) &&
7822 (max_mem >= DRT_HASH_LARGE_MEMORY_REQUIRED)) {
7823 modulus_size = DRT_HASH_LARGE_MODULUS;
7824 map_size = DRT_LARGE_ALLOCATION;
7825 } else {
7826 modulus_size = DRT_HASH_SMALL_MODULUS;
7827 map_size = DRT_SMALL_ALLOCATION;
7828 }
7829 } else if (ocmap->scm_modulus == DRT_HASH_LARGE_MODULUS) {
7830 if ((active_buckets > (DRT_HASH_LARGE_MODULUS - 5)) &&
7831 (max_mem >= DRT_HASH_XLARGE_MEMORY_REQUIRED)) {
7832 modulus_size = DRT_HASH_XLARGE_MODULUS;
7833 map_size = DRT_XLARGE_ALLOCATION;
7834 } else {
7835 /*
7836 * If the ring is completely full and we can't
7837 * expand, there's nothing useful for us to do.
7838 * Behave as though we had compacted into the new
7839 * array and return.
7840 */
7841 return KERN_SUCCESS;
7842 }
7843 } else {
7844 /* already using the xlarge modulus */
7845 modulus_size = DRT_HASH_XLARGE_MODULUS;
7846 map_size = DRT_XLARGE_ALLOCATION;
7847
7848 /*
7849 * If the ring is completely full, there's
7850 * nothing useful for us to do. Behave as
7851 * though we had compacted into the new
7852 * array and return.
7853 */
7854 if (active_buckets >= DRT_HASH_XLARGE_MODULUS) {
7855 return KERN_SUCCESS;
7856 }
7857 }
7858 }
7859
7860 /*
7861 * Allocate and initialise the new map.
7862 */
7863
7864 kret = kmem_alloc(kernel_map, (vm_offset_t *)&cmap, map_size,
7865 KMA_DATA, VM_KERN_MEMORY_FILE);
7866 if (kret != KERN_SUCCESS) {
7867 return kret;
7868 }
7869 cmap->scm_magic = DRT_SCM_MAGIC;
7870 cmap->scm_modulus = modulus_size;
7871 cmap->scm_buckets = 0;
7872 cmap->scm_lastclean = 0;
7873 cmap->scm_iskips = 0;
7874 for (i = 0; i < cmap->scm_modulus; i++) {
7875 DRT_HASH_CLEAR(cmap, i);
7876 DRT_HASH_VACATE(cmap, i);
7877 DRT_BITVECTOR_CLEAR(cmap, i);
7878 }
7879
7880 /*
7881 * If there's an old map, re-hash entries from it into the new map.
7882 */
7883 copycount = 0;
7884 if (ocmap != NULL) {
7885 for (i = 0; i < ocmap->scm_modulus; i++) {
7886 /* skip empty buckets */
7887 if (DRT_HASH_VACANT(ocmap, i) ||
7888 (DRT_HASH_GET_COUNT(ocmap, i) == 0)) {
7889 continue;
7890 }
7891 /* get new index */
7892 offset = DRT_HASH_GET_ADDRESS(ocmap, i);
7893 kret = vfs_drt_get_index(&cmap, offset, &index, 1);
7894 if (kret != KERN_SUCCESS) {
7895 /* XXX need to bail out gracefully here */
7896 panic("vfs_drt: new cluster map mysteriously too small");
7897 index = 0;
7898 }
7899 /* copy */
7900 DRT_HASH_COPY(ocmap, i, cmap, index);
7901 copycount++;
7902 }
7903 }
7904
7905 /* log what we've done */
7906 vfs_drt_trace(cmap, DRT_DEBUG_ALLOC, copycount, 0, 0, 0);
7907
7908 /*
7909 * It's important to ensure that *cmapp always points to
7910 * a valid map, so we must overwrite it before freeing
7911 * the old map.
7912 */
7913 *cmapp = cmap;
7914 if (ocmap != NULL) {
7915 /* emit stats into trace buffer */
7916 vfs_drt_trace(ocmap, DRT_DEBUG_SCMDATA,
7917 ocmap->scm_modulus,
7918 ocmap->scm_buckets,
7919 ocmap->scm_lastclean,
7920 ocmap->scm_iskips);
7921
7922 vfs_drt_free_map(ocmap);
7923 }
7924 return KERN_SUCCESS;
7925 }
7926
7927
7928 /*
7929 * Free a sparse cluster map.
7930 */
7931 static kern_return_t
vfs_drt_free_map(struct vfs_drt_clustermap * cmap)7932 vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
7933 {
7934 vm_size_t map_size = 0;
7935
7936 if (cmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
7937 map_size = DRT_SMALL_ALLOCATION;
7938 } else if (cmap->scm_modulus == DRT_HASH_LARGE_MODULUS) {
7939 map_size = DRT_LARGE_ALLOCATION;
7940 } else if (cmap->scm_modulus == DRT_HASH_XLARGE_MODULUS) {
7941 map_size = DRT_XLARGE_ALLOCATION;
7942 } else {
7943 panic("vfs_drt_free_map: Invalid modulus %d", cmap->scm_modulus);
7944 }
7945
7946 kmem_free(kernel_map, (vm_offset_t)cmap, map_size);
7947 return KERN_SUCCESS;
7948 }
7949
7950
7951 /*
7952 * Find the hashtable slot currently occupied by an entry for the supplied offset.
7953 */
7954 static kern_return_t
vfs_drt_search_index(struct vfs_drt_clustermap * cmap,u_int64_t offset,int * indexp)7955 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
7956 {
7957 int index;
7958 u_int32_t i;
7959
7960 offset = DRT_ALIGN_ADDRESS(offset);
7961 index = DRT_HASH(cmap, offset);
7962
7963 /* traverse the hashtable */
7964 for (i = 0; i < cmap->scm_modulus; i++) {
7965 /*
7966 * If the slot is vacant, we can stop.
7967 */
7968 if (DRT_HASH_VACANT(cmap, index)) {
7969 break;
7970 }
7971
7972 /*
7973 * If the address matches our offset, we have success.
7974 */
7975 if (DRT_HASH_GET_ADDRESS(cmap, index) == offset) {
7976 *indexp = index;
7977 return KERN_SUCCESS;
7978 }
7979
7980 /*
7981 * Move to the next slot, try again.
7982 */
7983 index = DRT_HASH_NEXT(cmap, index);
7984 }
7985 /*
7986 * It's not there.
7987 */
7988 return KERN_FAILURE;
7989 }
7990
7991 /*
7992 * Find the hashtable slot for the supplied offset. If we haven't allocated
7993 * one yet, allocate one and populate the address field. Note that it will
7994 * not have a nonzero page count and thus will still technically be free, so
7995 * in the case where we are called to clean pages, the slot will remain free.
7996 */
7997 static kern_return_t
vfs_drt_get_index(struct vfs_drt_clustermap ** cmapp,u_int64_t offset,int * indexp,int recursed)7998 vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *indexp, int recursed)
7999 {
8000 struct vfs_drt_clustermap *cmap;
8001 kern_return_t kret;
8002 u_int32_t index;
8003 u_int32_t i;
8004
8005 cmap = *cmapp;
8006
8007 /* look for an existing entry */
8008 kret = vfs_drt_search_index(cmap, offset, indexp);
8009 if (kret == KERN_SUCCESS) {
8010 return kret;
8011 }
8012
8013 /* need to allocate an entry */
8014 offset = DRT_ALIGN_ADDRESS(offset);
8015 index = DRT_HASH(cmap, offset);
8016
8017 /* scan from the index forwards looking for a vacant slot */
8018 for (i = 0; i < cmap->scm_modulus; i++) {
8019 /* slot vacant? */
8020 if (DRT_HASH_VACANT(cmap, index) || DRT_HASH_GET_COUNT(cmap, index) == 0) {
8021 cmap->scm_buckets++;
8022 if (index < cmap->scm_lastclean) {
8023 cmap->scm_lastclean = index;
8024 }
8025 DRT_HASH_SET_ADDRESS(cmap, index, offset);
8026 DRT_HASH_SET_COUNT(cmap, index, 0);
8027 DRT_BITVECTOR_CLEAR(cmap, index);
8028 *indexp = index;
8029 vfs_drt_trace(cmap, DRT_DEBUG_INSERT, (int)offset, i, 0, 0);
8030 return KERN_SUCCESS;
8031 }
8032 cmap->scm_iskips += i;
8033 index = DRT_HASH_NEXT(cmap, index);
8034 }
8035
8036 /*
8037 * We haven't found a vacant slot, so the map is full. If we're not
8038 * already recursed, try reallocating/compacting it.
8039 */
8040 if (recursed) {
8041 return KERN_FAILURE;
8042 }
8043 kret = vfs_drt_alloc_map(cmapp);
8044 if (kret == KERN_SUCCESS) {
8045 /* now try to insert again */
8046 kret = vfs_drt_get_index(cmapp, offset, indexp, 1);
8047 }
8048 return kret;
8049 }
8050
8051 /*
8052 * Implementation of set dirty/clean.
8053 *
8054 * In the 'clean' case, not finding a map is OK.
8055 */
8056 static kern_return_t
vfs_drt_do_mark_pages(void ** private,u_int64_t offset,u_int length,u_int * setcountp,int dirty)8057 vfs_drt_do_mark_pages(
8058 void **private,
8059 u_int64_t offset,
8060 u_int length,
8061 u_int *setcountp,
8062 int dirty)
8063 {
8064 struct vfs_drt_clustermap *cmap, **cmapp;
8065 kern_return_t kret;
8066 int i, index, pgoff, pgcount, setcount, ecount;
8067
8068 cmapp = (struct vfs_drt_clustermap **)private;
8069 cmap = *cmapp;
8070
8071 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_START, (int)offset, (int)length, dirty, 0);
8072
8073 if (setcountp != NULL) {
8074 *setcountp = 0;
8075 }
8076
8077 /* allocate a cluster map if we don't already have one */
8078 if (cmap == NULL) {
8079 /* no cluster map, nothing to clean */
8080 if (!dirty) {
8081 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 1, 0, 0, 0);
8082 return KERN_SUCCESS;
8083 }
8084 kret = vfs_drt_alloc_map(cmapp);
8085 if (kret != KERN_SUCCESS) {
8086 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 2, 0, 0, 0);
8087 return kret;
8088 }
8089 }
8090 setcount = 0;
8091
8092 /*
8093 * Iterate over the length of the region.
8094 */
8095 while (length > 0) {
8096 /*
8097 * Get the hashtable index for this offset.
8098 *
8099 * XXX this will add blank entries if we are clearing a range
8100 * that hasn't been dirtied.
8101 */
8102 kret = vfs_drt_get_index(cmapp, offset, &index, 0);
8103 cmap = *cmapp; /* may have changed! */
8104 /* this may be a partial-success return */
8105 if (kret != KERN_SUCCESS) {
8106 if (setcountp != NULL) {
8107 *setcountp = setcount;
8108 }
8109 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 3, (int)length, 0, 0);
8110
8111 return kret;
8112 }
8113
8114 /*
8115 * Work out how many pages we're modifying in this
8116 * hashtable entry.
8117 */
8118 pgoff = (int)((offset - DRT_ALIGN_ADDRESS(offset)) / PAGE_SIZE);
8119 pgcount = min((length / PAGE_SIZE), (DRT_BITVECTOR_PAGES - pgoff));
8120
8121 /*
8122 * Iterate over pages, dirty/clearing as we go.
8123 */
8124 ecount = DRT_HASH_GET_COUNT(cmap, index);
8125 for (i = 0; i < pgcount; i++) {
8126 if (dirty) {
8127 if (!DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
8128 if (ecount >= DRT_BITVECTOR_PAGES) {
8129 panic("ecount >= DRT_BITVECTOR_PAGES, cmap = %p, index = %d, bit = %d", cmap, index, pgoff + i);
8130 }
8131 DRT_HASH_SET_BIT(cmap, index, pgoff + i);
8132 ecount++;
8133 setcount++;
8134 }
8135 } else {
8136 if (DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
8137 if (ecount <= 0) {
8138 panic("ecount <= 0, cmap = %p, index = %d, bit = %d", cmap, index, pgoff + i);
8139 }
8140 assert(ecount > 0);
8141 DRT_HASH_CLEAR_BIT(cmap, index, pgoff + i);
8142 ecount--;
8143 setcount++;
8144 }
8145 }
8146 }
8147 DRT_HASH_SET_COUNT(cmap, index, ecount);
8148
8149 offset += pgcount * PAGE_SIZE;
8150 length -= pgcount * PAGE_SIZE;
8151 }
8152 if (setcountp != NULL) {
8153 *setcountp = setcount;
8154 }
8155
8156 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 0, setcount, 0, 0);
8157
8158 return KERN_SUCCESS;
8159 }
8160
8161 /*
8162 * Mark a set of pages as dirty/clean.
8163 *
8164 * This is a public interface.
8165 *
8166 * cmapp
8167 * Pointer to storage suitable for holding a pointer. Note that
8168 * this must either be NULL or a value set by this function.
8169 *
8170 * size
8171 * Current file size in bytes.
8172 *
8173 * offset
8174 * Offset of the first page to be marked as dirty, in bytes. Must be
8175 * page-aligned.
8176 *
8177 * length
8178 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
8179 *
8180 * setcountp
8181 * Number of pages newly marked dirty by this call (optional).
8182 *
8183 * Returns KERN_SUCCESS if all the pages were successfully marked.
8184 */
8185 static kern_return_t
vfs_drt_mark_pages(void ** cmapp,off_t offset,u_int length,u_int * setcountp)8186 vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp)
8187 {
8188 /* XXX size unused, drop from interface */
8189 return vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1);
8190 }
8191
8192 #if 0
8193 static kern_return_t
8194 vfs_drt_unmark_pages(void **cmapp, off_t offset, u_int length)
8195 {
8196 return vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
8197 }
8198 #endif
8199
8200 /*
8201 * Get a cluster of dirty pages.
8202 *
8203 * This is a public interface.
8204 *
8205 * cmapp
8206 * Pointer to storage managed by drt_mark_pages. Note that this must
8207 * be NULL or a value set by drt_mark_pages.
8208 *
8209 * offsetp
8210 * Returns the byte offset into the file of the first page in the cluster.
8211 *
8212 * lengthp
8213 * Returns the length in bytes of the cluster of dirty pages.
8214 *
8215 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
8216 * are no dirty pages meeting the minmum size criteria. Private storage will
8217 * be released if there are no more dirty pages left in the map
8218 *
8219 */
8220 static kern_return_t
vfs_drt_get_cluster(void ** cmapp,off_t * offsetp,u_int * lengthp)8221 vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
8222 {
8223 struct vfs_drt_clustermap *cmap;
8224 u_int64_t offset;
8225 u_int length;
8226 u_int32_t j;
8227 int index, i, fs, ls;
8228
8229 /* sanity */
8230 if ((cmapp == NULL) || (*cmapp == NULL)) {
8231 return KERN_FAILURE;
8232 }
8233 cmap = *cmapp;
8234
8235 /* walk the hashtable */
8236 for (offset = 0, j = 0; j < cmap->scm_modulus; offset += (DRT_BITVECTOR_PAGES * PAGE_SIZE), j++) {
8237 index = DRT_HASH(cmap, offset);
8238
8239 if (DRT_HASH_VACANT(cmap, index) || (DRT_HASH_GET_COUNT(cmap, index) == 0)) {
8240 continue;
8241 }
8242
8243 /* scan the bitfield for a string of bits */
8244 fs = -1;
8245
8246 for (i = 0; i < DRT_BITVECTOR_PAGES; i++) {
8247 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
8248 fs = i;
8249 break;
8250 }
8251 }
8252 if (fs == -1) {
8253 /* didn't find any bits set */
8254 panic("vfs_drt: entry summary count > 0 but no bits set in map, cmap = %p, index = %d, count = %lld",
8255 cmap, index, DRT_HASH_GET_COUNT(cmap, index));
8256 }
8257 for (ls = 0; i < DRT_BITVECTOR_PAGES; i++, ls++) {
8258 if (!DRT_HASH_TEST_BIT(cmap, index, i)) {
8259 break;
8260 }
8261 }
8262
8263 /* compute offset and length, mark pages clean */
8264 offset = DRT_HASH_GET_ADDRESS(cmap, index) + (PAGE_SIZE * fs);
8265 length = ls * PAGE_SIZE;
8266 vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
8267 cmap->scm_lastclean = index;
8268
8269 /* return successful */
8270 *offsetp = (off_t)offset;
8271 *lengthp = length;
8272
8273 vfs_drt_trace(cmap, DRT_DEBUG_RETCLUSTER, (int)offset, (int)length, 0, 0);
8274 return KERN_SUCCESS;
8275 }
8276 /*
8277 * We didn't find anything... hashtable is empty
8278 * emit stats into trace buffer and
8279 * then free it
8280 */
8281 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
8282 cmap->scm_modulus,
8283 cmap->scm_buckets,
8284 cmap->scm_lastclean,
8285 cmap->scm_iskips);
8286
8287 vfs_drt_free_map(cmap);
8288 *cmapp = NULL;
8289
8290 return KERN_FAILURE;
8291 }
8292
8293
8294 static kern_return_t
vfs_drt_control(void ** cmapp,int op_type)8295 vfs_drt_control(void **cmapp, int op_type)
8296 {
8297 struct vfs_drt_clustermap *cmap;
8298
8299 /* sanity */
8300 if ((cmapp == NULL) || (*cmapp == NULL)) {
8301 return KERN_FAILURE;
8302 }
8303 cmap = *cmapp;
8304
8305 switch (op_type) {
8306 case 0:
8307 /* emit stats into trace buffer */
8308 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
8309 cmap->scm_modulus,
8310 cmap->scm_buckets,
8311 cmap->scm_lastclean,
8312 cmap->scm_iskips);
8313
8314 vfs_drt_free_map(cmap);
8315 *cmapp = NULL;
8316 break;
8317
8318 case 1:
8319 cmap->scm_lastclean = 0;
8320 break;
8321 }
8322 return KERN_SUCCESS;
8323 }
8324
8325
8326
8327 /*
8328 * Emit a summary of the state of the clustermap into the trace buffer
8329 * along with some caller-provided data.
8330 */
8331 #if KDEBUG
8332 static void
vfs_drt_trace(__unused struct vfs_drt_clustermap * cmap,int code,int arg1,int arg2,int arg3,int arg4)8333 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, int code, int arg1, int arg2, int arg3, int arg4)
8334 {
8335 KERNEL_DEBUG(code, arg1, arg2, arg3, arg4, 0);
8336 }
8337 #else
8338 static void
vfs_drt_trace(__unused struct vfs_drt_clustermap * cmap,__unused int code,__unused int arg1,__unused int arg2,__unused int arg3,__unused int arg4)8339 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, __unused int code,
8340 __unused int arg1, __unused int arg2, __unused int arg3,
8341 __unused int arg4)
8342 {
8343 }
8344 #endif
8345
8346 #if 0
8347 /*
8348 * Perform basic sanity check on the hash entry summary count
8349 * vs. the actual bits set in the entry.
8350 */
8351 static void
8352 vfs_drt_sanity(struct vfs_drt_clustermap *cmap)
8353 {
8354 int index, i;
8355 int bits_on;
8356
8357 for (index = 0; index < cmap->scm_modulus; index++) {
8358 if (DRT_HASH_VACANT(cmap, index)) {
8359 continue;
8360 }
8361
8362 for (bits_on = 0, i = 0; i < DRT_BITVECTOR_PAGES; i++) {
8363 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
8364 bits_on++;
8365 }
8366 }
8367 if (bits_on != DRT_HASH_GET_COUNT(cmap, index)) {
8368 panic("bits_on = %d, index = %d", bits_on, index);
8369 }
8370 }
8371 }
8372 #endif
8373
8374 /*
8375 * Internal interface only.
8376 */
8377 static kern_return_t
vfs_get_scmap_push_behavior_internal(void ** cmapp,int * push_flag)8378 vfs_get_scmap_push_behavior_internal(void **cmapp, int *push_flag)
8379 {
8380 struct vfs_drt_clustermap *cmap;
8381
8382 /* sanity */
8383 if ((cmapp == NULL) || (*cmapp == NULL) || (push_flag == NULL)) {
8384 return KERN_FAILURE;
8385 }
8386 cmap = *cmapp;
8387
8388 if (cmap->scm_modulus == DRT_HASH_XLARGE_MODULUS) {
8389 /*
8390 * If we have a full xlarge sparse cluster,
8391 * we push it out all at once so the cluster
8392 * map can be available to absorb more I/Os.
8393 * This is done on large memory configs so
8394 * the small I/Os don't interfere with the
8395 * pro workloads.
8396 */
8397 *push_flag = PUSH_ALL;
8398 }
8399 return KERN_SUCCESS;
8400 }
8401