1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * File: vnode_pager.c
36 *
37 * "Swap" pager that pages to/from vnodes. Also
38 * handles demand paging from files.
39 *
40 */
41
42 #include <mach/boolean.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/user.h>
46 #include <sys/proc.h>
47 #include <sys/kauth.h>
48 #include <sys/buf.h>
49 #include <sys/uio.h>
50 #include <sys/vnode_internal.h>
51 #include <sys/namei.h>
52 #include <sys/mount_internal.h> /* needs internal due to fhandle_t */
53 #include <sys/ubc_internal.h>
54 #include <sys/lock.h>
55 #include <sys/disk.h> /* For DKIOC calls */
56
57 #include <mach/mach_types.h>
58 #include <mach/memory_object_types.h>
59 #include <mach/vm_map.h>
60 #include <mach/mach_vm.h>
61 #include <mach/upl.h>
62 #include <mach/sdt.h>
63
64 #include <vm/vm_map.h>
65 #include <vm/vm_kern.h>
66 #include <kern/zalloc.h>
67 #include <libkern/libkern.h>
68
69 #include <vm/vnode_pager.h>
70 #include <vm/vm_pageout.h>
71 #include <vm/vm_ubc.h>
72
73 #include <kern/assert.h>
74 #include <sys/kdebug.h>
75 #include <nfs/nfs.h>
76
77 #include <vm/vm_protos_internal.h>
78
79 #include <sys/kdebug.h>
80 #include <sys/kdebug_triage.h>
81 #include <vfs/vfs_disk_conditioner.h>
82
83 void
vnode_pager_throttle(void)84 vnode_pager_throttle(void)
85 {
86 if (current_uthread()->uu_lowpri_window) {
87 throttle_lowpri_io(1);
88 }
89 }
90
91 boolean_t
vnode_pager_isSSD(vnode_t vp)92 vnode_pager_isSSD(vnode_t vp)
93 {
94 return disk_conditioner_mount_is_ssd(vp->v_mount);
95 }
96
97 #if FBDP_DEBUG_OBJECT_NO_PAGER
98 bool
vnode_pager_forced_unmount(vnode_t vp)99 vnode_pager_forced_unmount(vnode_t vp)
100 {
101 mount_t mnt;
102 mnt = vnode_mount(vp);
103 if (!mnt) {
104 return false;
105 }
106 return vfs_isforce(mnt);
107 }
108 #endif /* FBDP_DEBUG_OBJECT_NO_PAGER */
109
110 #if CONFIG_IOSCHED
111 void
vnode_pager_issue_reprioritize_io(struct vnode * devvp,uint64_t blkno,uint32_t len,int priority)112 vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
113 {
114 u_int32_t blocksize = 0;
115 dk_extent_t extent;
116 dk_set_tier_t set_tier;
117 int error = 0;
118
119 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
120 if (error) {
121 return;
122 }
123
124 memset(&extent, 0, sizeof(dk_extent_t));
125 memset(&set_tier, 0, sizeof(dk_set_tier_t));
126
127 extent.offset = blkno * (u_int64_t) blocksize;
128 extent.length = len;
129
130 set_tier.extents = &extent;
131 set_tier.extentsCount = 1;
132 set_tier.tier = (uint8_t)priority;
133
134 error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
135 return;
136 }
137 #endif
138
139 void
vnode_pager_was_dirtied(struct vnode * vp,vm_object_offset_t s_offset,vm_object_offset_t e_offset)140 vnode_pager_was_dirtied(
141 struct vnode *vp,
142 vm_object_offset_t s_offset,
143 vm_object_offset_t e_offset)
144 {
145 cluster_update_state(vp, s_offset, e_offset, TRUE);
146 }
147
148 uint32_t
vnode_pager_isinuse(struct vnode * vp)149 vnode_pager_isinuse(struct vnode *vp)
150 {
151 if (vp->v_usecount > vp->v_kusecount) {
152 return 1;
153 }
154 return 0;
155 }
156
157 uint32_t
vnode_pager_return_throttle_io_limit(struct vnode * vp,uint32_t * limit)158 vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)
159 {
160 return cluster_throttle_io_limit(vp, limit);
161 }
162
163 vm_object_offset_t
vnode_pager_get_filesize(struct vnode * vp)164 vnode_pager_get_filesize(struct vnode *vp)
165 {
166 return (vm_object_offset_t) ubc_getsize(vp);
167 }
168
169 extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
170
171 kern_return_t
vnode_pager_get_name(struct vnode * vp,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)172 vnode_pager_get_name(
173 struct vnode *vp,
174 char *pathname,
175 vm_size_t pathname_len,
176 char *filename,
177 vm_size_t filename_len,
178 boolean_t *truncated_path_p)
179 {
180 *truncated_path_p = FALSE;
181 if (pathname != NULL) {
182 /* get the path name */
183 safe_getpath(vp, NULL,
184 pathname, (int) pathname_len,
185 truncated_path_p);
186 }
187 if ((pathname == NULL || *truncated_path_p) &&
188 filename != NULL) {
189 /* get the file name */
190 const char *name;
191
192 name = vnode_getname_printable(vp);
193 strlcpy(filename, name, (size_t) filename_len);
194 vnode_putname_printable(name);
195 }
196 return KERN_SUCCESS;
197 }
198
199 kern_return_t
vnode_pager_get_mtime(struct vnode * vp,struct timespec * current_mtime,struct timespec * cs_mtime)200 vnode_pager_get_mtime(
201 struct vnode *vp,
202 struct timespec *current_mtime,
203 struct timespec *cs_mtime)
204 {
205 vnode_mtime(vp, current_mtime, vfs_context_current());
206 if (cs_mtime != NULL) {
207 ubc_get_cs_mtime(vp, cs_mtime);
208 }
209 return KERN_SUCCESS;
210 }
211
212 kern_return_t
vnode_pager_get_cs_blobs(struct vnode * vp,void ** blobs)213 vnode_pager_get_cs_blobs(
214 struct vnode *vp,
215 void **blobs)
216 {
217 *blobs = ubc_get_cs_blobs(vp);
218 return KERN_SUCCESS;
219 }
220
221 /*
222 * vnode_trim:
223 * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
224 * Trims the region at offset bytes into the file, for length bytes.
225 *
226 * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this
227 * function is called; no iocounts or usecounts are taken on the vnode.
228 * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them
229 * are successfully discarded.
230 */
231 u_int32_t
vnode_trim(struct vnode * vp,off_t offset,size_t length)232 vnode_trim(
233 struct vnode *vp,
234 off_t offset,
235 size_t length)
236 {
237 daddr64_t io_blockno; /* Block number corresponding to the start of the extent */
238 size_t io_bytecount; /* Number of bytes in current extent for the specified range */
239 size_t trimmed = 0;
240 off_t current_offset = offset;
241 size_t remaining_length = length;
242 int error = 0;
243 u_int32_t blocksize = 0;
244 struct vnode *devvp;
245 dk_extent_t extent;
246 dk_unmap_t unmap;
247
248
249 /* Get the underlying device vnode */
250 devvp = vp->v_mount->mnt_devvp;
251
252 /* Figure out the underlying device block size */
253 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
254 if (error) {
255 goto trim_exit;
256 }
257
258 /*
259 * We may not get the entire range from offset -> offset+length in a single
260 * extent from the blockmap call. Keep looping/going until we are sure we've hit
261 * the whole range or if we encounter an error.
262 */
263 while (trimmed < length) {
264 /*
265 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
266 * specified offset. It returns blocks in contiguous chunks, so if the logical range is
267 * broken into multiple extents, it must be called multiple times, increasing the offset
268 * in each call to ensure that the entire range is covered.
269 */
270 error = VNOP_BLOCKMAP(vp, current_offset, remaining_length,
271 &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL);
272
273 if (error) {
274 goto trim_exit;
275 }
276 /*
277 * We have a contiguous run. Prepare & issue the ioctl for the device.
278 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
279 */
280 memset(&extent, 0, sizeof(dk_extent_t));
281 memset(&unmap, 0, sizeof(dk_unmap_t));
282 extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
283 extent.length = io_bytecount;
284 unmap.extents = &extent;
285 unmap.extentsCount = 1;
286 error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel());
287
288 if (error) {
289 goto trim_exit;
290 }
291 remaining_length = remaining_length - io_bytecount;
292 trimmed = trimmed + io_bytecount;
293 current_offset = current_offset + io_bytecount;
294 }
295 trim_exit:
296
297 return error;
298 }
299
300 pager_return_t
vnode_pageout(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)301 vnode_pageout(struct vnode *vp,
302 upl_t upl,
303 upl_offset_t upl_offset,
304 vm_object_offset_t f_offset,
305 upl_size_t size,
306 int flags,
307 int *errorp)
308 {
309 int result = PAGER_SUCCESS;
310 int error = 0;
311 int error_ret = 0;
312 daddr64_t blkno;
313 int isize;
314 int pg_index;
315 int base_index;
316 upl_offset_t offset;
317 upl_page_info_t *pl;
318 vfs_context_t ctx = vfs_context_current(); /* pager context */
319
320 isize = (int)size;
321
322 /*
323 * This call is non-blocking and does not ever fail but it can
324 * only be made when there is other explicit synchronization
325 * with reclaiming of the vnode which, in this path, is provided
326 * by the paging in progress counter.
327 *
328 * In addition, this may also be entered via explicit ubc_msync
329 * calls or vm_swapfile_io where the existing iocount provides
330 * the necessary synchronization. Ideally we would not take an
331 * additional iocount here in the cases where an explcit iocount
332 * has already been taken but this call doesn't cause a deadlock
333 * as other forms of vnode_get* might if this thread has already
334 * taken an iocount.
335 */
336 error = vnode_getalways_from_pager(vp);
337 if (error != 0) {
338 /* This can't happen */
339 panic("vnode_getalways returned %d for vp %p", error, vp);
340 }
341
342 if (isize <= 0) {
343 result = PAGER_ERROR;
344 error_ret = EINVAL;
345 goto out;
346 }
347
348 if (UBCINFOEXISTS(vp) == 0) {
349 result = PAGER_ERROR;
350 error_ret = EINVAL;
351
352 if (upl && !(flags & UPL_NOCOMMIT)) {
353 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
354 }
355 goto out;
356 }
357 if (!(flags & UPL_VNODE_PAGER)) {
358 /*
359 * This is a pageout from the default pager,
360 * just go ahead and call vnop_pageout since
361 * it has already sorted out the dirty ranges
362 */
363 KDBG_RELEASE(
364 VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_START,
365 size, 1);
366
367 if ((error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
368 (size_t)size, flags, ctx))) {
369 result = PAGER_ERROR;
370 }
371
372 KDBG_RELEASE(
373 VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_END,
374 size, 1);
375
376 goto out;
377 }
378 if (upl == NULL) {
379 int request_flags;
380
381 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
382 /*
383 * filesystem has requested the new form of VNOP_PAGEOUT for file
384 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT...
385 * it is the fileystem's responsibility to grab the range we're denoting
386 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
387 * take any locks it needs, before effectively locking the pages into a UPL...
388 */
389 KDBG_RELEASE(VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_START,
390 size, (int)f_offset);
391
392 if ((error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
393 size, flags, ctx))) {
394 result = PAGER_ERROR;
395 }
396 KDBG_RELEASE(VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_END,
397 size);
398
399 goto out;
400 }
401 if (flags & UPL_MSYNC) {
402 request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
403 } else {
404 request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
405 }
406
407 if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) {
408 result = PAGER_ERROR;
409 error_ret = EINVAL;
410 goto out;
411 }
412 upl_offset = 0;
413 } else {
414 pl = ubc_upl_pageinfo(upl);
415 }
416
417 /*
418 * Ignore any non-present pages at the end of the
419 * UPL so that we aren't looking at a upl that
420 * may already have been freed by the preceeding
421 * aborts/completions.
422 */
423 base_index = upl_offset / PAGE_SIZE;
424
425 for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
426 if (upl_page_present(pl, --pg_index)) {
427 break;
428 }
429 if (pg_index == base_index) {
430 /*
431 * no pages were returned, so release
432 * our hold on the upl and leave
433 */
434 if (!(flags & UPL_NOCOMMIT)) {
435 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
436 }
437
438 goto out;
439 }
440 }
441 isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
442
443 /*
444 * we come here for pageouts to 'real' files and
445 * for msyncs... the upl may not contain any
446 * dirty pages.. it's our responsibility to sort
447 * through it and find the 'runs' of dirty pages
448 * to call VNOP_PAGEOUT on...
449 */
450
451 if (ubc_getsize(vp) == 0) {
452 /*
453 * if the file has been effectively deleted, then
454 * we need to go through the UPL and invalidate any
455 * buffer headers we might have that reference any
456 * of it's pages
457 */
458 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
459 if (vp->v_tag == VT_NFS) {
460 /* check with nfs if page is OK to drop */
461 error = nfs_buf_page_inval(vp, (off_t)f_offset);
462 } else {
463 blkno = ubc_offtoblk(vp, (off_t)f_offset);
464 error = buf_invalblkno(vp, blkno, 0);
465 }
466 if (error) {
467 if (!(flags & UPL_NOCOMMIT)) {
468 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
469 }
470 if (error_ret == 0) {
471 error_ret = error;
472 }
473 result = PAGER_ERROR;
474 } else if (!(flags & UPL_NOCOMMIT)) {
475 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
476 }
477 f_offset += PAGE_SIZE;
478 }
479 goto out;
480 }
481
482 offset = upl_offset;
483 pg_index = base_index;
484
485 while (isize) {
486 int xsize;
487 int num_of_pages;
488
489 if (!upl_page_present(pl, pg_index)) {
490 /*
491 * we asked for RET_ONLY_DIRTY, so it's possible
492 * to get back empty slots in the UPL
493 * just skip over them
494 */
495 f_offset += PAGE_SIZE;
496 offset += PAGE_SIZE;
497 isize -= PAGE_SIZE;
498 pg_index++;
499
500 continue;
501 }
502 if (!upl_dirty_page(pl, pg_index)) {
503 /*
504 * if the page is not dirty and reached here it is
505 * marked precious or it is due to invalidation in
506 * memory_object_lock request as part of truncation
507 * We also get here from vm_object_terminate()
508 * So all you need to do in these
509 * cases is to invalidate incore buffer if it is there
510 * Note we must not sleep here if the buffer is busy - that is
511 * a lock inversion which causes deadlock.
512 */
513 if (vp->v_tag == VT_NFS) {
514 /* check with nfs if page is OK to drop */
515 error = nfs_buf_page_inval(vp, (off_t)f_offset);
516 } else {
517 blkno = ubc_offtoblk(vp, (off_t)f_offset);
518 error = buf_invalblkno(vp, blkno, 0);
519 }
520 if (error) {
521 if (!(flags & UPL_NOCOMMIT)) {
522 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
523 }
524 if (error_ret == 0) {
525 error_ret = error;
526 }
527 result = PAGER_ERROR;
528 } else if (!(flags & UPL_NOCOMMIT)) {
529 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
530 }
531 f_offset += PAGE_SIZE;
532 offset += PAGE_SIZE;
533 isize -= PAGE_SIZE;
534 pg_index++;
535
536 continue;
537 }
538 num_of_pages = 1;
539 xsize = isize - PAGE_SIZE;
540
541 while (xsize) {
542 if (!upl_dirty_page(pl, pg_index + num_of_pages)) {
543 break;
544 }
545 num_of_pages++;
546 xsize -= PAGE_SIZE;
547 }
548 xsize = num_of_pages * PAGE_SIZE;
549
550 KDBG_RELEASE(VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_START,
551 xsize, (int)f_offset);
552
553 if ((error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
554 xsize, flags, ctx))) {
555 if (error_ret == 0) {
556 error_ret = error;
557 }
558 result = PAGER_ERROR;
559 }
560 KDBG_RELEASE(VMDBG_CODE(DBG_VM_VNODE_PAGEOUT) | DBG_FUNC_END,
561 xsize, error);
562
563 f_offset += xsize;
564 offset += xsize;
565 isize -= xsize;
566 pg_index += num_of_pages;
567 }
568 out:
569 vnode_put_from_pager(vp);
570
571 if (errorp) {
572 *errorp = error_ret;
573 }
574
575 return result;
576 }
577
578
579 pager_return_t
vnode_pagein(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)580 vnode_pagein(
581 struct vnode *vp,
582 upl_t upl,
583 upl_offset_t upl_offset,
584 vm_object_offset_t f_offset,
585 upl_size_t size,
586 int flags,
587 int *errorp)
588 {
589 upl_page_info_t *pl;
590 int result = PAGER_SUCCESS;
591 int error = 0;
592 int pages_in_upl;
593 int start_pg;
594 int last_pg;
595 int first_pg;
596 int xsize;
597 int must_commit = 1;
598 int ignore_valid_page_check = 0;
599
600 if (flags & UPL_NOCOMMIT) {
601 must_commit = 0;
602 }
603
604 if (flags & UPL_IGNORE_VALID_PAGE_CHECK) {
605 ignore_valid_page_check = 1;
606 }
607
608 /*
609 * This call is non-blocking and does not ever fail but it can
610 * only be made when there is other explicit synchronization
611 * with reclaiming of the vnode which, in this path, is provided
612 * by the paging in progress counter.
613 *
614 * In addition, this may also be entered via vm_swapfile_io
615 * where the existing iocount provides the necessary synchronization.
616 * Ideally we would not take an additional iocount here in the cases
617 * where an explcit iocount has already been taken but this call
618 * doesn't cause a deadlock as other forms of vnode_get* might if
619 * this thread has already taken an iocount.
620 */
621 error = vnode_getalways_from_pager(vp);
622 if (error != 0) {
623 /* This can't happen */
624 panic("vnode_getalways returned %d for vp %p", error, vp);
625 }
626
627 if (UBCINFOEXISTS(vp) == 0) {
628 result = PAGER_ERROR;
629 error = PAGER_ERROR;
630
631 if (upl && must_commit) {
632 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
633 }
634
635 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UBCINFO), 0 /* arg */);
636 goto out;
637 }
638 if (upl == (upl_t)NULL) {
639 flags &= ~UPL_NOCOMMIT;
640
641 if (size > MAX_UPL_SIZE_BYTES) {
642 result = PAGER_ERROR;
643 error = PAGER_ERROR;
644 goto out;
645 }
646 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) {
647 /*
648 * filesystem has requested the new form of VNOP_PAGEIN for file
649 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN...
650 * it is the fileystem's responsibility to grab the range we're denoting
651 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
652 * take any locks it needs, before effectively locking the pages into a UPL...
653 * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
654 * is used to identify the "must have" page in the extent... the filesystem is free
655 * to clip the extent to better fit the underlying FS blocksize if it desires as
656 * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
657 * identifies that page
658 */
659 if ((error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
660 size, flags, vfs_context_current()))) {
661 set_thread_pagein_error(current_thread(), error);
662 result = PAGER_ERROR;
663 error = PAGER_ERROR;
664 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
665 }
666 goto out;
667 }
668 ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE);
669
670 if (upl == (upl_t)NULL) {
671 result = PAGER_ABSENT;
672 error = PAGER_ABSENT;
673 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UPL), 0 /* arg */);
674 goto out;
675 }
676 ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
677
678 upl_offset = 0;
679 first_pg = 0;
680
681 /*
682 * if we get here, we've created the upl and
683 * are responsible for commiting/aborting it
684 * regardless of what the caller has passed in
685 */
686 must_commit = 1;
687 } else {
688 pl = ubc_upl_pageinfo(upl);
689 first_pg = upl_offset / PAGE_SIZE;
690 }
691 pages_in_upl = size / PAGE_SIZE;
692 DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
693
694 /*
695 * before we start marching forward, we must make sure we end on
696 * a present page, otherwise we will be working with a freed
697 * upl
698 */
699 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
700 if (upl_page_present(pl, last_pg)) {
701 break;
702 }
703 if (last_pg == first_pg) {
704 /*
705 * empty UPL, no pages are present
706 */
707 if (must_commit) {
708 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
709 }
710 goto out;
711 }
712 }
713 pages_in_upl = last_pg + 1;
714 last_pg = first_pg;
715
716 while (last_pg < pages_in_upl) {
717 /*
718 * skip over missing pages...
719 */
720 for (; last_pg < pages_in_upl; last_pg++) {
721 if (upl_page_present(pl, last_pg)) {
722 break;
723 }
724 }
725
726 if (ignore_valid_page_check == 1) {
727 start_pg = last_pg;
728 } else {
729 /*
730 * skip over 'valid' pages... we don't want to issue I/O for these
731 */
732 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
733 if (!upl_valid_page(pl, last_pg)) {
734 break;
735 }
736 }
737 }
738
739 if (last_pg > start_pg) {
740 /*
741 * we've found a range of valid pages
742 * if we've got COMMIT responsibility
743 * commit this range of pages back to the
744 * cache unchanged
745 */
746 xsize = (last_pg - start_pg) * PAGE_SIZE;
747
748 if (must_commit) {
749 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
750 }
751 }
752 if (last_pg == pages_in_upl) {
753 /*
754 * we're done... all pages that were present
755 * have either had I/O issued on them or
756 * were aborted unchanged...
757 */
758 break;
759 }
760
761 if (!upl_page_present(pl, last_pg)) {
762 /*
763 * we found a range of valid pages
764 * terminated by a missing page...
765 * bump index to the next page and continue on
766 */
767 last_pg++;
768 continue;
769 }
770 /*
771 * scan from the found invalid page looking for a valid
772 * or non-present page before the end of the upl is reached, if we
773 * find one, then it will be the last page of the request to
774 * 'cluster_io'
775 */
776 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
777 if ((!ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) {
778 break;
779 }
780 }
781 if (last_pg > start_pg) {
782 int xoff;
783 xsize = (last_pg - start_pg) * PAGE_SIZE;
784 xoff = start_pg * PAGE_SIZE;
785
786 if ((error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
787 (off_t)f_offset + xoff,
788 xsize, flags, vfs_context_current()))) {
789 /*
790 * Usually this UPL will be aborted/committed by the lower cluster layer.
791 *
792 * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
793 * a deadlock with another thread already inflating the file.
794 *
795 * b) In the case of content protection, EPERM is a valid error and we should respect it.
796 *
797 * In those cases, we must take care of our UPL at this layer itself.
798 */
799 if (must_commit) {
800 if (error == EAGAIN) {
801 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
802 }
803 if (error == EPERM) {
804 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
805 }
806 }
807 set_thread_pagein_error(current_thread(), error);
808 result = PAGER_ERROR;
809 error = PAGER_ERROR;
810 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
811 }
812 }
813 }
814 out:
815 vnode_put_from_pager(vp);
816
817 if (errorp) {
818 *errorp = result;
819 }
820
821 return error;
822 }
823