1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * File: vnode_pager.c
36 *
37 * "Swap" pager that pages to/from vnodes. Also
38 * handles demand paging from files.
39 *
40 */
41
42 #include <mach/boolean.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/user.h>
46 #include <sys/proc.h>
47 #include <sys/kauth.h>
48 #include <sys/buf.h>
49 #include <sys/uio.h>
50 #include <sys/vnode_internal.h>
51 #include <sys/namei.h>
52 #include <sys/mount_internal.h> /* needs internal due to fhandle_t */
53 #include <sys/ubc_internal.h>
54 #include <sys/lock.h>
55 #include <sys/disk.h> /* For DKIOC calls */
56
57 #include <mach/mach_types.h>
58 #include <mach/memory_object_types.h>
59 #include <mach/vm_map.h>
60 #include <mach/mach_vm.h>
61 #include <mach/upl.h>
62 #include <mach/sdt.h>
63
64 #include <vm/vm_map.h>
65 #include <vm/vm_kern.h>
66 #include <kern/zalloc.h>
67 #include <libkern/libkern.h>
68
69 #include <vm/vnode_pager.h>
70 #include <vm/vm_pageout.h>
71
72 #include <kern/assert.h>
73 #include <sys/kdebug.h>
74 #include <nfs/nfs.h>
75
76 #include <vm/vm_protos.h>
77
78 #include <sys/kdebug_triage.h>
79 #include <vfs/vfs_disk_conditioner.h>
80
81 void
vnode_pager_throttle(void)82 vnode_pager_throttle(void)
83 {
84 if (current_uthread()->uu_lowpri_window) {
85 throttle_lowpri_io(1);
86 }
87 }
88
89 boolean_t
vnode_pager_isSSD(vnode_t vp)90 vnode_pager_isSSD(vnode_t vp)
91 {
92 return disk_conditioner_mount_is_ssd(vp->v_mount);
93 }
94
95 #if CONFIG_IOSCHED
96 void
vnode_pager_issue_reprioritize_io(struct vnode * devvp,uint64_t blkno,uint32_t len,int priority)97 vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
98 {
99 u_int32_t blocksize = 0;
100 dk_extent_t extent;
101 dk_set_tier_t set_tier;
102 int error = 0;
103
104 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
105 if (error) {
106 return;
107 }
108
109 memset(&extent, 0, sizeof(dk_extent_t));
110 memset(&set_tier, 0, sizeof(dk_set_tier_t));
111
112 extent.offset = blkno * (u_int64_t) blocksize;
113 extent.length = len;
114
115 set_tier.extents = &extent;
116 set_tier.extentsCount = 1;
117 set_tier.tier = (uint8_t)priority;
118
119 error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
120 return;
121 }
122 #endif
123
124 void
vnode_pager_was_dirtied(struct vnode * vp,vm_object_offset_t s_offset,vm_object_offset_t e_offset)125 vnode_pager_was_dirtied(
126 struct vnode *vp,
127 vm_object_offset_t s_offset,
128 vm_object_offset_t e_offset)
129 {
130 cluster_update_state(vp, s_offset, e_offset, TRUE);
131 }
132
133 uint32_t
vnode_pager_isinuse(struct vnode * vp)134 vnode_pager_isinuse(struct vnode *vp)
135 {
136 if (vp->v_usecount > vp->v_kusecount) {
137 return 1;
138 }
139 return 0;
140 }
141
142 uint32_t
vnode_pager_return_throttle_io_limit(struct vnode * vp,uint32_t * limit)143 vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)
144 {
145 return cluster_throttle_io_limit(vp, limit);
146 }
147
148 vm_object_offset_t
vnode_pager_get_filesize(struct vnode * vp)149 vnode_pager_get_filesize(struct vnode *vp)
150 {
151 return (vm_object_offset_t) ubc_getsize(vp);
152 }
153
154 extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
155
156 kern_return_t
vnode_pager_get_name(struct vnode * vp,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)157 vnode_pager_get_name(
158 struct vnode *vp,
159 char *pathname,
160 vm_size_t pathname_len,
161 char *filename,
162 vm_size_t filename_len,
163 boolean_t *truncated_path_p)
164 {
165 *truncated_path_p = FALSE;
166 if (pathname != NULL) {
167 /* get the path name */
168 safe_getpath(vp, NULL,
169 pathname, (int) pathname_len,
170 truncated_path_p);
171 }
172 if ((pathname == NULL || *truncated_path_p) &&
173 filename != NULL) {
174 /* get the file name */
175 const char *name;
176
177 name = vnode_getname_printable(vp);
178 strlcpy(filename, name, (size_t) filename_len);
179 vnode_putname_printable(name);
180 }
181 return KERN_SUCCESS;
182 }
183
184 kern_return_t
vnode_pager_get_mtime(struct vnode * vp,struct timespec * current_mtime,struct timespec * cs_mtime)185 vnode_pager_get_mtime(
186 struct vnode *vp,
187 struct timespec *current_mtime,
188 struct timespec *cs_mtime)
189 {
190 vnode_mtime(vp, current_mtime, vfs_context_current());
191 if (cs_mtime != NULL) {
192 ubc_get_cs_mtime(vp, cs_mtime);
193 }
194 return KERN_SUCCESS;
195 }
196
197 kern_return_t
vnode_pager_get_cs_blobs(struct vnode * vp,void ** blobs)198 vnode_pager_get_cs_blobs(
199 struct vnode *vp,
200 void **blobs)
201 {
202 *blobs = ubc_get_cs_blobs(vp);
203 return KERN_SUCCESS;
204 }
205
206 /*
207 * vnode_trim:
208 * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
209 * Trims the region at offset bytes into the file, for length bytes.
210 *
211 * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this
212 * function is called; no iocounts or usecounts are taken on the vnode.
213 * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them
214 * are successfully discarded.
215 */
216 u_int32_t
vnode_trim(struct vnode * vp,off_t offset,size_t length)217 vnode_trim(
218 struct vnode *vp,
219 off_t offset,
220 size_t length)
221 {
222 daddr64_t io_blockno; /* Block number corresponding to the start of the extent */
223 size_t io_bytecount; /* Number of bytes in current extent for the specified range */
224 size_t trimmed = 0;
225 off_t current_offset = offset;
226 size_t remaining_length = length;
227 int error = 0;
228 u_int32_t blocksize = 0;
229 struct vnode *devvp;
230 dk_extent_t extent;
231 dk_unmap_t unmap;
232
233
234 /* Get the underlying device vnode */
235 devvp = vp->v_mount->mnt_devvp;
236
237 /* Figure out the underlying device block size */
238 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
239 if (error) {
240 goto trim_exit;
241 }
242
243 /*
244 * We may not get the entire range from offset -> offset+length in a single
245 * extent from the blockmap call. Keep looping/going until we are sure we've hit
246 * the whole range or if we encounter an error.
247 */
248 while (trimmed < length) {
249 /*
250 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
251 * specified offset. It returns blocks in contiguous chunks, so if the logical range is
252 * broken into multiple extents, it must be called multiple times, increasing the offset
253 * in each call to ensure that the entire range is covered.
254 */
255 error = VNOP_BLOCKMAP(vp, current_offset, remaining_length,
256 &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL);
257
258 if (error) {
259 goto trim_exit;
260 }
261 /*
262 * We have a contiguous run. Prepare & issue the ioctl for the device.
263 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
264 */
265 memset(&extent, 0, sizeof(dk_extent_t));
266 memset(&unmap, 0, sizeof(dk_unmap_t));
267 extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
268 extent.length = io_bytecount;
269 unmap.extents = &extent;
270 unmap.extentsCount = 1;
271 error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel());
272
273 if (error) {
274 goto trim_exit;
275 }
276 remaining_length = remaining_length - io_bytecount;
277 trimmed = trimmed + io_bytecount;
278 current_offset = current_offset + io_bytecount;
279 }
280 trim_exit:
281
282 return error;
283 }
284
285 pager_return_t
vnode_pageout(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)286 vnode_pageout(struct vnode *vp,
287 upl_t upl,
288 upl_offset_t upl_offset,
289 vm_object_offset_t f_offset,
290 upl_size_t size,
291 int flags,
292 int *errorp)
293 {
294 int result = PAGER_SUCCESS;
295 int error = 0;
296 int error_ret = 0;
297 daddr64_t blkno;
298 int isize;
299 int pg_index;
300 int base_index;
301 upl_offset_t offset;
302 upl_page_info_t *pl;
303 vfs_context_t ctx = vfs_context_current(); /* pager context */
304
305 isize = (int)size;
306
307 /*
308 * This call is non-blocking and does not ever fail but it can
309 * only be made when there is other explicit synchronization
310 * with reclaiming of the vnode which, in this path, is provided
311 * by the paging in progress counter.
312 *
313 * In addition, this may also be entered via explicit ubc_msync
314 * calls or vm_swapfile_io where the existing iocount provides
315 * the necessary synchronization. Ideally we would not take an
316 * additional iocount here in the cases where an explcit iocount
317 * has already been taken but this call doesn't cause a deadlock
318 * as other forms of vnode_get* might if this thread has already
319 * taken an iocount.
320 */
321 error = vnode_getalways_from_pager(vp);
322 if (error != 0) {
323 /* This can't happen */
324 panic("vnode_getalways returned %d for vp %p", error, vp);
325 }
326
327 if (isize <= 0) {
328 result = PAGER_ERROR;
329 error_ret = EINVAL;
330 goto out;
331 }
332
333 if (UBCINFOEXISTS(vp) == 0) {
334 result = PAGER_ERROR;
335 error_ret = EINVAL;
336
337 if (upl && !(flags & UPL_NOCOMMIT)) {
338 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
339 }
340 goto out;
341 }
342 if (!(flags & UPL_VNODE_PAGER)) {
343 /*
344 * This is a pageout from the default pager,
345 * just go ahead and call vnop_pageout since
346 * it has already sorted out the dirty ranges
347 */
348 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
349 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
350 size, 1, 0, 0, 0);
351
352 if ((error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
353 (size_t)size, flags, ctx))) {
354 result = PAGER_ERROR;
355 }
356
357 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
358 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
359 size, 1, 0, 0, 0);
360
361 goto out;
362 }
363 if (upl == NULL) {
364 int request_flags;
365
366 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
367 /*
368 * filesystem has requested the new form of VNOP_PAGEOUT for file
369 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT...
370 * it is the fileystem's responsibility to grab the range we're denoting
371 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
372 * take any locks it needs, before effectively locking the pages into a UPL...
373 */
374 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
375 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
376 size, (int)f_offset, 0, 0, 0);
377
378 if ((error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
379 size, flags, ctx))) {
380 result = PAGER_ERROR;
381 }
382 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
383 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
384 size, 0, 0, 0, 0);
385
386 goto out;
387 }
388 if (flags & UPL_MSYNC) {
389 request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
390 } else {
391 request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
392 }
393
394 if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) {
395 result = PAGER_ERROR;
396 error_ret = EINVAL;
397 goto out;
398 }
399 upl_offset = 0;
400 } else {
401 pl = ubc_upl_pageinfo(upl);
402 }
403
404 /*
405 * Ignore any non-present pages at the end of the
406 * UPL so that we aren't looking at a upl that
407 * may already have been freed by the preceeding
408 * aborts/completions.
409 */
410 base_index = upl_offset / PAGE_SIZE;
411
412 for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
413 if (upl_page_present(pl, --pg_index)) {
414 break;
415 }
416 if (pg_index == base_index) {
417 /*
418 * no pages were returned, so release
419 * our hold on the upl and leave
420 */
421 if (!(flags & UPL_NOCOMMIT)) {
422 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
423 }
424
425 goto out;
426 }
427 }
428 isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
429
430 /*
431 * we come here for pageouts to 'real' files and
432 * for msyncs... the upl may not contain any
433 * dirty pages.. it's our responsibility to sort
434 * through it and find the 'runs' of dirty pages
435 * to call VNOP_PAGEOUT on...
436 */
437
438 if (ubc_getsize(vp) == 0) {
439 /*
440 * if the file has been effectively deleted, then
441 * we need to go through the UPL and invalidate any
442 * buffer headers we might have that reference any
443 * of it's pages
444 */
445 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
446 if (vp->v_tag == VT_NFS) {
447 /* check with nfs if page is OK to drop */
448 error = nfs_buf_page_inval(vp, (off_t)f_offset);
449 } else {
450 blkno = ubc_offtoblk(vp, (off_t)f_offset);
451 error = buf_invalblkno(vp, blkno, 0);
452 }
453 if (error) {
454 if (!(flags & UPL_NOCOMMIT)) {
455 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
456 }
457 if (error_ret == 0) {
458 error_ret = error;
459 }
460 result = PAGER_ERROR;
461 } else if (!(flags & UPL_NOCOMMIT)) {
462 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
463 }
464 f_offset += PAGE_SIZE;
465 }
466 goto out;
467 }
468
469 offset = upl_offset;
470 pg_index = base_index;
471
472 while (isize) {
473 int xsize;
474 int num_of_pages;
475
476 if (!upl_page_present(pl, pg_index)) {
477 /*
478 * we asked for RET_ONLY_DIRTY, so it's possible
479 * to get back empty slots in the UPL
480 * just skip over them
481 */
482 f_offset += PAGE_SIZE;
483 offset += PAGE_SIZE;
484 isize -= PAGE_SIZE;
485 pg_index++;
486
487 continue;
488 }
489 if (!upl_dirty_page(pl, pg_index)) {
490 /*
491 * if the page is not dirty and reached here it is
492 * marked precious or it is due to invalidation in
493 * memory_object_lock request as part of truncation
494 * We also get here from vm_object_terminate()
495 * So all you need to do in these
496 * cases is to invalidate incore buffer if it is there
497 * Note we must not sleep here if the buffer is busy - that is
498 * a lock inversion which causes deadlock.
499 */
500 if (vp->v_tag == VT_NFS) {
501 /* check with nfs if page is OK to drop */
502 error = nfs_buf_page_inval(vp, (off_t)f_offset);
503 } else {
504 blkno = ubc_offtoblk(vp, (off_t)f_offset);
505 error = buf_invalblkno(vp, blkno, 0);
506 }
507 if (error) {
508 if (!(flags & UPL_NOCOMMIT)) {
509 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
510 }
511 if (error_ret == 0) {
512 error_ret = error;
513 }
514 result = PAGER_ERROR;
515 } else if (!(flags & UPL_NOCOMMIT)) {
516 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
517 }
518 f_offset += PAGE_SIZE;
519 offset += PAGE_SIZE;
520 isize -= PAGE_SIZE;
521 pg_index++;
522
523 continue;
524 }
525 num_of_pages = 1;
526 xsize = isize - PAGE_SIZE;
527
528 while (xsize) {
529 if (!upl_dirty_page(pl, pg_index + num_of_pages)) {
530 break;
531 }
532 num_of_pages++;
533 xsize -= PAGE_SIZE;
534 }
535 xsize = num_of_pages * PAGE_SIZE;
536
537 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
538 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
539 xsize, (int)f_offset, 0, 0, 0);
540
541 if ((error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
542 xsize, flags, ctx))) {
543 if (error_ret == 0) {
544 error_ret = error;
545 }
546 result = PAGER_ERROR;
547 }
548 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
549 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
550 xsize, 0, 0, 0, 0);
551
552 f_offset += xsize;
553 offset += xsize;
554 isize -= xsize;
555 pg_index += num_of_pages;
556 }
557 out:
558 vnode_put_from_pager(vp);
559
560 if (errorp) {
561 *errorp = error_ret;
562 }
563
564 return result;
565 }
566
567
568 pager_return_t
vnode_pagein(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)569 vnode_pagein(
570 struct vnode *vp,
571 upl_t upl,
572 upl_offset_t upl_offset,
573 vm_object_offset_t f_offset,
574 upl_size_t size,
575 int flags,
576 int *errorp)
577 {
578 upl_page_info_t *pl;
579 int result = PAGER_SUCCESS;
580 int error = 0;
581 int pages_in_upl;
582 int start_pg;
583 int last_pg;
584 int first_pg;
585 int xsize;
586 int must_commit = 1;
587 int ignore_valid_page_check = 0;
588
589 if (flags & UPL_NOCOMMIT) {
590 must_commit = 0;
591 }
592
593 if (flags & UPL_IGNORE_VALID_PAGE_CHECK) {
594 ignore_valid_page_check = 1;
595 }
596
597 /*
598 * This call is non-blocking and does not ever fail but it can
599 * only be made when there is other explicit synchronization
600 * with reclaiming of the vnode which, in this path, is provided
601 * by the paging in progress counter.
602 *
603 * In addition, this may also be entered via vm_swapfile_io
604 * where the existing iocount provides the necessary synchronization.
605 * Ideally we would not take an additional iocount here in the cases
606 * where an explcit iocount has already been taken but this call
607 * doesn't cause a deadlock as other forms of vnode_get* might if
608 * this thread has already taken an iocount.
609 */
610 error = vnode_getalways_from_pager(vp);
611 if (error != 0) {
612 /* This can't happen */
613 panic("vnode_getalways returned %d for vp %p", error, vp);
614 }
615
616 if (UBCINFOEXISTS(vp) == 0) {
617 result = PAGER_ERROR;
618 error = PAGER_ERROR;
619
620 if (upl && must_commit) {
621 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
622 }
623
624 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UBCINFO), 0 /* arg */);
625 goto out;
626 }
627 if (upl == (upl_t)NULL) {
628 flags &= ~UPL_NOCOMMIT;
629
630 if (size > MAX_UPL_SIZE_BYTES) {
631 result = PAGER_ERROR;
632 error = PAGER_ERROR;
633 goto out;
634 }
635 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) {
636 /*
637 * filesystem has requested the new form of VNOP_PAGEIN for file
638 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN...
639 * it is the fileystem's responsibility to grab the range we're denoting
640 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
641 * take any locks it needs, before effectively locking the pages into a UPL...
642 * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
643 * is used to identify the "must have" page in the extent... the filesystem is free
644 * to clip the extent to better fit the underlying FS blocksize if it desires as
645 * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
646 * identifies that page
647 */
648 if ((error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
649 size, flags, vfs_context_current()))) {
650 set_thread_pagein_error(current_thread(), error);
651 result = PAGER_ERROR;
652 error = PAGER_ERROR;
653 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
654 }
655 goto out;
656 }
657 ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE);
658
659 if (upl == (upl_t)NULL) {
660 result = PAGER_ABSENT;
661 error = PAGER_ABSENT;
662 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UPL), 0 /* arg */);
663 goto out;
664 }
665 ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
666
667 upl_offset = 0;
668 first_pg = 0;
669
670 /*
671 * if we get here, we've created the upl and
672 * are responsible for commiting/aborting it
673 * regardless of what the caller has passed in
674 */
675 must_commit = 1;
676 } else {
677 pl = ubc_upl_pageinfo(upl);
678 first_pg = upl_offset / PAGE_SIZE;
679 }
680 pages_in_upl = size / PAGE_SIZE;
681 DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
682
683 /*
684 * before we start marching forward, we must make sure we end on
685 * a present page, otherwise we will be working with a freed
686 * upl
687 */
688 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
689 if (upl_page_present(pl, last_pg)) {
690 break;
691 }
692 if (last_pg == first_pg) {
693 /*
694 * empty UPL, no pages are present
695 */
696 if (must_commit) {
697 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
698 }
699 goto out;
700 }
701 }
702 pages_in_upl = last_pg + 1;
703 last_pg = first_pg;
704
705 while (last_pg < pages_in_upl) {
706 /*
707 * skip over missing pages...
708 */
709 for (; last_pg < pages_in_upl; last_pg++) {
710 if (upl_page_present(pl, last_pg)) {
711 break;
712 }
713 }
714
715 if (ignore_valid_page_check == 1) {
716 start_pg = last_pg;
717 } else {
718 /*
719 * skip over 'valid' pages... we don't want to issue I/O for these
720 */
721 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
722 if (!upl_valid_page(pl, last_pg)) {
723 break;
724 }
725 }
726 }
727
728 if (last_pg > start_pg) {
729 /*
730 * we've found a range of valid pages
731 * if we've got COMMIT responsibility
732 * commit this range of pages back to the
733 * cache unchanged
734 */
735 xsize = (last_pg - start_pg) * PAGE_SIZE;
736
737 if (must_commit) {
738 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
739 }
740 }
741 if (last_pg == pages_in_upl) {
742 /*
743 * we're done... all pages that were present
744 * have either had I/O issued on them or
745 * were aborted unchanged...
746 */
747 break;
748 }
749
750 if (!upl_page_present(pl, last_pg)) {
751 /*
752 * we found a range of valid pages
753 * terminated by a missing page...
754 * bump index to the next page and continue on
755 */
756 last_pg++;
757 continue;
758 }
759 /*
760 * scan from the found invalid page looking for a valid
761 * or non-present page before the end of the upl is reached, if we
762 * find one, then it will be the last page of the request to
763 * 'cluster_io'
764 */
765 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
766 if ((!ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) {
767 break;
768 }
769 }
770 if (last_pg > start_pg) {
771 int xoff;
772 xsize = (last_pg - start_pg) * PAGE_SIZE;
773 xoff = start_pg * PAGE_SIZE;
774
775 if ((error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
776 (off_t)f_offset + xoff,
777 xsize, flags, vfs_context_current()))) {
778 /*
779 * Usually this UPL will be aborted/committed by the lower cluster layer.
780 *
781 * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
782 * a deadlock with another thread already inflating the file.
783 *
784 * b) In the case of content protection, EPERM is a valid error and we should respect it.
785 *
786 * In those cases, we must take care of our UPL at this layer itself.
787 */
788 if (must_commit) {
789 if (error == EAGAIN) {
790 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
791 }
792 if (error == EPERM) {
793 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
794 }
795 }
796 set_thread_pagein_error(current_thread(), error);
797 result = PAGER_ERROR;
798 error = PAGER_ERROR;
799 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
800 }
801 }
802 }
803 out:
804 vnode_put_from_pager(vp);
805
806 if (errorp) {
807 *errorp = result;
808 }
809
810 return error;
811 }
812
813 void *
upl_get_internal_page_list(upl_t upl)814 upl_get_internal_page_list(upl_t upl)
815 {
816 return UPL_GET_INTERNAL_PAGE_LIST(upl);
817 }
818