1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * File: vnode_pager.c
36 *
37 * "Swap" pager that pages to/from vnodes. Also
38 * handles demand paging from files.
39 *
40 */
41
42 #include <mach/boolean.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/user.h>
46 #include <sys/proc.h>
47 #include <sys/kauth.h>
48 #include <sys/buf.h>
49 #include <sys/uio.h>
50 #include <sys/vnode_internal.h>
51 #include <sys/namei.h>
52 #include <sys/mount_internal.h> /* needs internal due to fhandle_t */
53 #include <sys/ubc_internal.h>
54 #include <sys/lock.h>
55 #include <sys/disk.h> /* For DKIOC calls */
56
57 #include <mach/mach_types.h>
58 #include <mach/memory_object_types.h>
59 #include <mach/vm_map.h>
60 #include <mach/mach_vm.h>
61 #include <mach/upl.h>
62 #include <mach/sdt.h>
63
64 #include <vm/vm_map.h>
65 #include <vm/vm_kern.h>
66 #include <kern/zalloc.h>
67 #include <libkern/libkern.h>
68
69 #include <vm/vnode_pager.h>
70 #include <vm/vm_pageout.h>
71
72 #include <kern/assert.h>
73 #include <sys/kdebug.h>
74 #include <nfs/nfs_conf.h>
75 #include <nfs/rpcv2.h>
76 #include <nfs/nfsproto.h>
77 #include <nfs/nfs.h>
78
79 #include <vm/vm_protos.h>
80
81 #include <sys/kdebug_triage.h>
82 #include <vfs/vfs_disk_conditioner.h>
83
84 void
vnode_pager_throttle(void)85 vnode_pager_throttle(void)
86 {
87 if (current_uthread()->uu_lowpri_window) {
88 throttle_lowpri_io(1);
89 }
90 }
91
92 boolean_t
vnode_pager_isSSD(vnode_t vp)93 vnode_pager_isSSD(vnode_t vp)
94 {
95 return disk_conditioner_mount_is_ssd(vp->v_mount);
96 }
97
98 #if CONFIG_IOSCHED
99 void
vnode_pager_issue_reprioritize_io(struct vnode * devvp,uint64_t blkno,uint32_t len,int priority)100 vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
101 {
102 u_int32_t blocksize = 0;
103 dk_extent_t extent;
104 dk_set_tier_t set_tier;
105 int error = 0;
106
107 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
108 if (error) {
109 return;
110 }
111
112 memset(&extent, 0, sizeof(dk_extent_t));
113 memset(&set_tier, 0, sizeof(dk_set_tier_t));
114
115 extent.offset = blkno * (u_int64_t) blocksize;
116 extent.length = len;
117
118 set_tier.extents = &extent;
119 set_tier.extentsCount = 1;
120 set_tier.tier = (uint8_t)priority;
121
122 error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
123 return;
124 }
125 #endif
126
127 void
vnode_pager_was_dirtied(struct vnode * vp,vm_object_offset_t s_offset,vm_object_offset_t e_offset)128 vnode_pager_was_dirtied(
129 struct vnode *vp,
130 vm_object_offset_t s_offset,
131 vm_object_offset_t e_offset)
132 {
133 cluster_update_state(vp, s_offset, e_offset, TRUE);
134 }
135
136 uint32_t
vnode_pager_isinuse(struct vnode * vp)137 vnode_pager_isinuse(struct vnode *vp)
138 {
139 if (vp->v_usecount > vp->v_kusecount) {
140 return 1;
141 }
142 return 0;
143 }
144
145 uint32_t
vnode_pager_return_throttle_io_limit(struct vnode * vp,uint32_t * limit)146 vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)
147 {
148 return cluster_throttle_io_limit(vp, limit);
149 }
150
151 vm_object_offset_t
vnode_pager_get_filesize(struct vnode * vp)152 vnode_pager_get_filesize(struct vnode *vp)
153 {
154 return (vm_object_offset_t) ubc_getsize(vp);
155 }
156
157 extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
158
159 kern_return_t
vnode_pager_get_name(struct vnode * vp,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)160 vnode_pager_get_name(
161 struct vnode *vp,
162 char *pathname,
163 vm_size_t pathname_len,
164 char *filename,
165 vm_size_t filename_len,
166 boolean_t *truncated_path_p)
167 {
168 *truncated_path_p = FALSE;
169 if (pathname != NULL) {
170 /* get the path name */
171 safe_getpath(vp, NULL,
172 pathname, (int) pathname_len,
173 truncated_path_p);
174 }
175 if ((pathname == NULL || *truncated_path_p) &&
176 filename != NULL) {
177 /* get the file name */
178 const char *name;
179
180 name = vnode_getname_printable(vp);
181 strlcpy(filename, name, (size_t) filename_len);
182 vnode_putname_printable(name);
183 }
184 return KERN_SUCCESS;
185 }
186
187 kern_return_t
vnode_pager_get_mtime(struct vnode * vp,struct timespec * current_mtime,struct timespec * cs_mtime)188 vnode_pager_get_mtime(
189 struct vnode *vp,
190 struct timespec *current_mtime,
191 struct timespec *cs_mtime)
192 {
193 vnode_mtime(vp, current_mtime, vfs_context_current());
194 if (cs_mtime != NULL) {
195 ubc_get_cs_mtime(vp, cs_mtime);
196 }
197 return KERN_SUCCESS;
198 }
199
200 kern_return_t
vnode_pager_get_cs_blobs(struct vnode * vp,void ** blobs)201 vnode_pager_get_cs_blobs(
202 struct vnode *vp,
203 void **blobs)
204 {
205 *blobs = ubc_get_cs_blobs(vp);
206 return KERN_SUCCESS;
207 }
208
209 /*
210 * vnode_trim:
211 * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
212 * Trims the region at offset bytes into the file, for length bytes.
213 *
214 * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this
215 * function is called; no iocounts or usecounts are taken on the vnode.
216 * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them
217 * are successfully discarded.
218 */
219 u_int32_t
vnode_trim(struct vnode * vp,off_t offset,size_t length)220 vnode_trim(
221 struct vnode *vp,
222 off_t offset,
223 size_t length)
224 {
225 daddr64_t io_blockno; /* Block number corresponding to the start of the extent */
226 size_t io_bytecount; /* Number of bytes in current extent for the specified range */
227 size_t trimmed = 0;
228 off_t current_offset = offset;
229 size_t remaining_length = length;
230 int error = 0;
231 u_int32_t blocksize = 0;
232 struct vnode *devvp;
233 dk_extent_t extent;
234 dk_unmap_t unmap;
235
236
237 /* Get the underlying device vnode */
238 devvp = vp->v_mount->mnt_devvp;
239
240 /* Figure out the underlying device block size */
241 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
242 if (error) {
243 goto trim_exit;
244 }
245
246 /*
247 * We may not get the entire range from offset -> offset+length in a single
248 * extent from the blockmap call. Keep looping/going until we are sure we've hit
249 * the whole range or if we encounter an error.
250 */
251 while (trimmed < length) {
252 /*
253 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
254 * specified offset. It returns blocks in contiguous chunks, so if the logical range is
255 * broken into multiple extents, it must be called multiple times, increasing the offset
256 * in each call to ensure that the entire range is covered.
257 */
258 error = VNOP_BLOCKMAP(vp, current_offset, remaining_length,
259 &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL);
260
261 if (error) {
262 goto trim_exit;
263 }
264 /*
265 * We have a contiguous run. Prepare & issue the ioctl for the device.
266 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
267 */
268 memset(&extent, 0, sizeof(dk_extent_t));
269 memset(&unmap, 0, sizeof(dk_unmap_t));
270 extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
271 extent.length = io_bytecount;
272 unmap.extents = &extent;
273 unmap.extentsCount = 1;
274 error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel());
275
276 if (error) {
277 goto trim_exit;
278 }
279 remaining_length = remaining_length - io_bytecount;
280 trimmed = trimmed + io_bytecount;
281 current_offset = current_offset + io_bytecount;
282 }
283 trim_exit:
284
285 return error;
286 }
287
288 pager_return_t
vnode_pageout(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)289 vnode_pageout(struct vnode *vp,
290 upl_t upl,
291 upl_offset_t upl_offset,
292 vm_object_offset_t f_offset,
293 upl_size_t size,
294 int flags,
295 int *errorp)
296 {
297 int result = PAGER_SUCCESS;
298 int error = 0;
299 int error_ret = 0;
300 daddr64_t blkno;
301 int isize;
302 int pg_index;
303 int base_index;
304 upl_offset_t offset;
305 upl_page_info_t *pl;
306 vfs_context_t ctx = vfs_context_current(); /* pager context */
307
308 isize = (int)size;
309
310 /*
311 * This call is non-blocking and does not ever fail but it can
312 * only be made when there is other explicit synchronization
313 * with reclaiming of the vnode which, in this path, is provided
314 * by the paging in progress counter.
315 *
316 * In addition, this may also be entered via explicit ubc_msync
317 * calls or vm_swapfile_io where the existing iocount provides
318 * the necessary synchronization. Ideally we would not take an
319 * additional iocount here in the cases where an explcit iocount
320 * has already been taken but this call doesn't cause a deadlock
321 * as other forms of vnode_get* might if this thread has already
322 * taken an iocount.
323 */
324 error = vnode_getalways_from_pager(vp);
325 if (error != 0) {
326 /* This can't happen */
327 panic("vnode_getalways returned %d for vp %p", error, vp);
328 }
329
330 if (isize <= 0) {
331 result = PAGER_ERROR;
332 error_ret = EINVAL;
333 goto out;
334 }
335
336 if (UBCINFOEXISTS(vp) == 0) {
337 result = PAGER_ERROR;
338 error_ret = EINVAL;
339
340 if (upl && !(flags & UPL_NOCOMMIT)) {
341 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
342 }
343 goto out;
344 }
345 if (!(flags & UPL_VNODE_PAGER)) {
346 /*
347 * This is a pageout from the default pager,
348 * just go ahead and call vnop_pageout since
349 * it has already sorted out the dirty ranges
350 */
351 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
352 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
353 size, 1, 0, 0, 0);
354
355 if ((error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
356 (size_t)size, flags, ctx))) {
357 result = PAGER_ERROR;
358 }
359
360 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
361 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
362 size, 1, 0, 0, 0);
363
364 goto out;
365 }
366 if (upl == NULL) {
367 int request_flags;
368
369 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
370 /*
371 * filesystem has requested the new form of VNOP_PAGEOUT for file
372 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT...
373 * it is the fileystem's responsibility to grab the range we're denoting
374 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
375 * take any locks it needs, before effectively locking the pages into a UPL...
376 */
377 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
378 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
379 size, (int)f_offset, 0, 0, 0);
380
381 if ((error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
382 size, flags, ctx))) {
383 result = PAGER_ERROR;
384 }
385 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
386 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
387 size, 0, 0, 0, 0);
388
389 goto out;
390 }
391 if (flags & UPL_MSYNC) {
392 request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
393 } else {
394 request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
395 }
396
397 if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) {
398 result = PAGER_ERROR;
399 error_ret = EINVAL;
400 goto out;
401 }
402 upl_offset = 0;
403 } else {
404 pl = ubc_upl_pageinfo(upl);
405 }
406
407 /*
408 * Ignore any non-present pages at the end of the
409 * UPL so that we aren't looking at a upl that
410 * may already have been freed by the preceeding
411 * aborts/completions.
412 */
413 base_index = upl_offset / PAGE_SIZE;
414
415 for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
416 if (upl_page_present(pl, --pg_index)) {
417 break;
418 }
419 if (pg_index == base_index) {
420 /*
421 * no pages were returned, so release
422 * our hold on the upl and leave
423 */
424 if (!(flags & UPL_NOCOMMIT)) {
425 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
426 }
427
428 goto out;
429 }
430 }
431 isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
432
433 /*
434 * we come here for pageouts to 'real' files and
435 * for msyncs... the upl may not contain any
436 * dirty pages.. it's our responsibility to sort
437 * through it and find the 'runs' of dirty pages
438 * to call VNOP_PAGEOUT on...
439 */
440
441 if (ubc_getsize(vp) == 0) {
442 /*
443 * if the file has been effectively deleted, then
444 * we need to go through the UPL and invalidate any
445 * buffer headers we might have that reference any
446 * of it's pages
447 */
448 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
449 #if CONFIG_NFS_CLIENT
450 if (vp->v_tag == VT_NFS) {
451 /* check with nfs if page is OK to drop */
452 error = nfs_buf_page_inval(vp, (off_t)f_offset);
453 } else
454 #endif /* CONFIG_NFS_CLIENT */
455 {
456 blkno = ubc_offtoblk(vp, (off_t)f_offset);
457 error = buf_invalblkno(vp, blkno, 0);
458 }
459 if (error) {
460 if (!(flags & UPL_NOCOMMIT)) {
461 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
462 }
463 if (error_ret == 0) {
464 error_ret = error;
465 }
466 result = PAGER_ERROR;
467 } else if (!(flags & UPL_NOCOMMIT)) {
468 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
469 }
470 f_offset += PAGE_SIZE;
471 }
472 goto out;
473 }
474
475 offset = upl_offset;
476 pg_index = base_index;
477
478 while (isize) {
479 int xsize;
480 int num_of_pages;
481
482 if (!upl_page_present(pl, pg_index)) {
483 /*
484 * we asked for RET_ONLY_DIRTY, so it's possible
485 * to get back empty slots in the UPL
486 * just skip over them
487 */
488 f_offset += PAGE_SIZE;
489 offset += PAGE_SIZE;
490 isize -= PAGE_SIZE;
491 pg_index++;
492
493 continue;
494 }
495 if (!upl_dirty_page(pl, pg_index)) {
496 /*
497 * if the page is not dirty and reached here it is
498 * marked precious or it is due to invalidation in
499 * memory_object_lock request as part of truncation
500 * We also get here from vm_object_terminate()
501 * So all you need to do in these
502 * cases is to invalidate incore buffer if it is there
503 * Note we must not sleep here if the buffer is busy - that is
504 * a lock inversion which causes deadlock.
505 */
506 #if CONFIG_NFS_CLIENT
507 if (vp->v_tag == VT_NFS) {
508 /* check with nfs if page is OK to drop */
509 error = nfs_buf_page_inval(vp, (off_t)f_offset);
510 } else
511 #endif /* CONFIG_NFS_CLIENT */
512 {
513 blkno = ubc_offtoblk(vp, (off_t)f_offset);
514 error = buf_invalblkno(vp, blkno, 0);
515 }
516 if (error) {
517 if (!(flags & UPL_NOCOMMIT)) {
518 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
519 }
520 if (error_ret == 0) {
521 error_ret = error;
522 }
523 result = PAGER_ERROR;
524 } else if (!(flags & UPL_NOCOMMIT)) {
525 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
526 }
527 f_offset += PAGE_SIZE;
528 offset += PAGE_SIZE;
529 isize -= PAGE_SIZE;
530 pg_index++;
531
532 continue;
533 }
534 num_of_pages = 1;
535 xsize = isize - PAGE_SIZE;
536
537 while (xsize) {
538 if (!upl_dirty_page(pl, pg_index + num_of_pages)) {
539 break;
540 }
541 num_of_pages++;
542 xsize -= PAGE_SIZE;
543 }
544 xsize = num_of_pages * PAGE_SIZE;
545
546 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
547 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
548 xsize, (int)f_offset, 0, 0, 0);
549
550 if ((error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
551 xsize, flags, ctx))) {
552 if (error_ret == 0) {
553 error_ret = error;
554 }
555 result = PAGER_ERROR;
556 }
557 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
558 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
559 xsize, 0, 0, 0, 0);
560
561 f_offset += xsize;
562 offset += xsize;
563 isize -= xsize;
564 pg_index += num_of_pages;
565 }
566 out:
567 vnode_put_from_pager(vp);
568
569 if (errorp) {
570 *errorp = error_ret;
571 }
572
573 return result;
574 }
575
576
577 pager_return_t
vnode_pagein(struct vnode * vp,upl_t upl,upl_offset_t upl_offset,vm_object_offset_t f_offset,upl_size_t size,int flags,int * errorp)578 vnode_pagein(
579 struct vnode *vp,
580 upl_t upl,
581 upl_offset_t upl_offset,
582 vm_object_offset_t f_offset,
583 upl_size_t size,
584 int flags,
585 int *errorp)
586 {
587 upl_page_info_t *pl;
588 int result = PAGER_SUCCESS;
589 int error = 0;
590 int pages_in_upl;
591 int start_pg;
592 int last_pg;
593 int first_pg;
594 int xsize;
595 int must_commit = 1;
596 int ignore_valid_page_check = 0;
597
598 if (flags & UPL_NOCOMMIT) {
599 must_commit = 0;
600 }
601
602 if (flags & UPL_IGNORE_VALID_PAGE_CHECK) {
603 ignore_valid_page_check = 1;
604 }
605
606 /*
607 * This call is non-blocking and does not ever fail but it can
608 * only be made when there is other explicit synchronization
609 * with reclaiming of the vnode which, in this path, is provided
610 * by the paging in progress counter.
611 *
612 * In addition, this may also be entered via vm_swapfile_io
613 * where the existing iocount provides the necessary synchronization.
614 * Ideally we would not take an additional iocount here in the cases
615 * where an explcit iocount has already been taken but this call
616 * doesn't cause a deadlock as other forms of vnode_get* might if
617 * this thread has already taken an iocount.
618 */
619 error = vnode_getalways_from_pager(vp);
620 if (error != 0) {
621 /* This can't happen */
622 panic("vnode_getalways returned %d for vp %p", error, vp);
623 }
624
625 if (UBCINFOEXISTS(vp) == 0) {
626 result = PAGER_ERROR;
627 error = PAGER_ERROR;
628
629 if (upl && must_commit) {
630 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
631 }
632
633 kernel_triage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UBCINFO), 0 /* arg */);
634 goto out;
635 }
636 if (upl == (upl_t)NULL) {
637 flags &= ~UPL_NOCOMMIT;
638
639 if (size > MAX_UPL_SIZE_BYTES) {
640 result = PAGER_ERROR;
641 error = PAGER_ERROR;
642 goto out;
643 }
644 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) {
645 /*
646 * filesystem has requested the new form of VNOP_PAGEIN for file
647 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN...
648 * it is the fileystem's responsibility to grab the range we're denoting
649 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
650 * take any locks it needs, before effectively locking the pages into a UPL...
651 * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
652 * is used to identify the "must have" page in the extent... the filesystem is free
653 * to clip the extent to better fit the underlying FS blocksize if it desires as
654 * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
655 * identifies that page
656 */
657 if ((error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
658 size, flags, vfs_context_current()))) {
659 set_thread_pagein_error(current_thread(), error);
660 result = PAGER_ERROR;
661 error = PAGER_ERROR;
662 kernel_triage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
663 }
664 goto out;
665 }
666 ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE);
667
668 if (upl == (upl_t)NULL) {
669 result = PAGER_ABSENT;
670 error = PAGER_ABSENT;
671 kernel_triage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UPL), 0 /* arg */);
672 goto out;
673 }
674 ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
675
676 upl_offset = 0;
677 first_pg = 0;
678
679 /*
680 * if we get here, we've created the upl and
681 * are responsible for commiting/aborting it
682 * regardless of what the caller has passed in
683 */
684 must_commit = 1;
685 } else {
686 pl = ubc_upl_pageinfo(upl);
687 first_pg = upl_offset / PAGE_SIZE;
688 }
689 pages_in_upl = size / PAGE_SIZE;
690 DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
691
692 /*
693 * before we start marching forward, we must make sure we end on
694 * a present page, otherwise we will be working with a freed
695 * upl
696 */
697 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
698 if (upl_page_present(pl, last_pg)) {
699 break;
700 }
701 if (last_pg == first_pg) {
702 /*
703 * empty UPL, no pages are present
704 */
705 if (must_commit) {
706 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
707 }
708 goto out;
709 }
710 }
711 pages_in_upl = last_pg + 1;
712 last_pg = first_pg;
713
714 while (last_pg < pages_in_upl) {
715 /*
716 * skip over missing pages...
717 */
718 for (; last_pg < pages_in_upl; last_pg++) {
719 if (upl_page_present(pl, last_pg)) {
720 break;
721 }
722 }
723
724 if (ignore_valid_page_check == 1) {
725 start_pg = last_pg;
726 } else {
727 /*
728 * skip over 'valid' pages... we don't want to issue I/O for these
729 */
730 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
731 if (!upl_valid_page(pl, last_pg)) {
732 break;
733 }
734 }
735 }
736
737 if (last_pg > start_pg) {
738 /*
739 * we've found a range of valid pages
740 * if we've got COMMIT responsibility
741 * commit this range of pages back to the
742 * cache unchanged
743 */
744 xsize = (last_pg - start_pg) * PAGE_SIZE;
745
746 if (must_commit) {
747 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
748 }
749 }
750 if (last_pg == pages_in_upl) {
751 /*
752 * we're done... all pages that were present
753 * have either had I/O issued on them or
754 * were aborted unchanged...
755 */
756 break;
757 }
758
759 if (!upl_page_present(pl, last_pg)) {
760 /*
761 * we found a range of valid pages
762 * terminated by a missing page...
763 * bump index to the next page and continue on
764 */
765 last_pg++;
766 continue;
767 }
768 /*
769 * scan from the found invalid page looking for a valid
770 * or non-present page before the end of the upl is reached, if we
771 * find one, then it will be the last page of the request to
772 * 'cluster_io'
773 */
774 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
775 if ((!ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) {
776 break;
777 }
778 }
779 if (last_pg > start_pg) {
780 int xoff;
781 xsize = (last_pg - start_pg) * PAGE_SIZE;
782 xoff = start_pg * PAGE_SIZE;
783
784 if ((error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
785 (off_t)f_offset + xoff,
786 xsize, flags, vfs_context_current()))) {
787 /*
788 * Usually this UPL will be aborted/committed by the lower cluster layer.
789 *
790 * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
791 * a deadlock with another thread already inflating the file.
792 *
793 * b) In the case of content protection, EPERM is a valid error and we should respect it.
794 *
795 * In those cases, we must take care of our UPL at this layer itself.
796 */
797 if (must_commit) {
798 if (error == EAGAIN) {
799 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
800 }
801 if (error == EPERM) {
802 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
803 }
804 }
805 set_thread_pagein_error(current_thread(), error);
806 result = PAGER_ERROR;
807 error = PAGER_ERROR;
808 kernel_triage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL), 0 /* arg */);
809 }
810 }
811 }
812 out:
813 vnode_put_from_pager(vp);
814
815 if (errorp) {
816 *errorp = result;
817 }
818
819 return error;
820 }
821
822 void *
upl_get_internal_page_list(upl_t upl)823 upl_get_internal_page_list(upl_t upl)
824 {
825 return UPL_GET_INTERNAL_PAGE_LIST(upl);
826 }
827