1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1989, 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * This code is derived from software contributed to Berkeley by
34 * Rick Macklem at The University of Guelph.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgement:
46 * This product includes software developed by the University of
47 * California, Berkeley and its contributors.
48 * 4. Neither the name of the University nor the names of its contributors
49 * may be used to endorse or promote products derived from this software
50 * without specific prior written permission.
51 *
52 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
54 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
55 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
56 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
57 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
58 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
59 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
60 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
61 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * SUCH DAMAGE.
63 *
64 * @(#)nfs_bio.c 8.9 (Berkeley) 3/30/95
65 * FreeBSD-Id: nfs_bio.c,v 1.44 1997/09/10 19:52:25 phk Exp $
66 */
67
68 #include <nfs/nfs_conf.h>
69 #if CONFIG_NFS_CLIENT
70
71 #include <sys/param.h>
72 #include <sys/systm.h>
73 #include <sys/resourcevar.h>
74 #include <sys/signalvar.h>
75 #include <sys/proc_internal.h>
76 #include <sys/kauth.h>
77 #include <sys/malloc.h>
78 #include <sys/vnode.h>
79 #include <sys/dirent.h>
80 #include <sys/mount_internal.h>
81 #include <sys/kernel.h>
82 #include <sys/ubc_internal.h>
83 #include <sys/uio_internal.h>
84 #include <sys/kpi_mbuf.h>
85
86 #include <sys/vm.h>
87 #include <sys/vmparam.h>
88
89 #include <sys/time.h>
90 #include <kern/clock.h>
91 #include <libkern/OSAtomic.h>
92 #include <kern/kalloc.h>
93 #include <kern/thread_call.h>
94
95 #include <nfs/rpcv2.h>
96 #include <nfs/nfsproto.h>
97 #include <nfs/nfs.h>
98 #include <nfs/nfs_gss.h>
99 #include <nfs/nfsmount.h>
100 #include <nfs/nfsnode.h>
101 #include <sys/buf_internal.h>
102 #include <libkern/OSAtomic.h>
103 #include <os/refcnt.h>
104
105 #define NFS_BIO_DBG(...) NFSCLNT_DBG(NFSCLNT_FAC_BIO, 7, ## __VA_ARGS__)
106
107 kern_return_t thread_terminate(thread_t); /* XXX */
108
109 #define NFSBUFHASH(np, lbn) \
110 (&nfsbufhashtbl[((long)(np) / sizeof(*(np)) + (int)(lbn)) & nfsbufhash])
111 LIST_HEAD(nfsbufhashhead, nfsbuf) * nfsbufhashtbl;
112 struct nfsbuffreehead nfsbuffree, nfsbuffreemeta, nfsbufdelwri;
113 u_long nfsbufhash;
114 int nfsbufcnt, nfsbufmin, nfsbufmax, nfsbufmetacnt, nfsbufmetamax;
115 int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer;
116 int nfs_nbdwrite;
117 int nfs_buf_timer_on = 0;
118 thread_t nfsbufdelwrithd = NULL;
119
120 static ZONE_DEFINE_TYPE(nfsbuf_zone, "NFS bio", struct nfsbuf, ZC_NONE);
121
122 static LCK_GRP_DECLARE(nfs_buf_lck_grp, "nfs buf");
123 LCK_MTX_DECLARE(nfs_buf_mutex, &nfs_buf_lck_grp);
124
125 #define NFSBUF_FREE_PERIOD 30 /* seconds */
126 #define NFSBUF_LRU_STALE 120
127 #define NFSBUF_META_STALE 240
128
129 /* number of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list */
130 #define LRU_TO_FREEUP 6
131 /* number of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list */
132 #define META_TO_FREEUP 3
133 /* total number of nfsbufs nfs_buf_freeup() should attempt to free */
134 #define TOTAL_TO_FREEUP (LRU_TO_FREEUP+META_TO_FREEUP)
135 /* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list when called from timer */
136 #define LRU_FREEUP_FRAC_ON_TIMER 8
137 /* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list when called from timer */
138 #define META_FREEUP_FRAC_ON_TIMER 16
139 /* fraction of total nfsbufs that nfsbuffreecnt should exceed before bothering to call nfs_buf_freeup() */
140 #define LRU_FREEUP_MIN_FRAC 4
141 /* fraction of total nfsbufs that nfsbuffreemetacnt should exceed before bothering to call nfs_buf_freeup() */
142 #define META_FREEUP_MIN_FRAC 2
143
144 #define NFS_ROUND_BLOCK(p, blksize) ((((uint64_t)(p) + blksize - 1) & ~((uint64_t)blksize - 1)) / blksize)
145
146 #define NFS_BUF_FREEUP() \
147 do { \
148 /* only call nfs_buf_freeup() if it has work to do: */ \
149 if (((nfsbuffreecnt > nfsbufcnt/LRU_FREEUP_MIN_FRAC) || \
150 (nfsbuffreemetacnt > nfsbufcnt/META_FREEUP_MIN_FRAC)) && \
151 ((nfsbufcnt - TOTAL_TO_FREEUP) > nfsbufmin)) \
152 nfs_buf_freeup(0); \
153 } while (0)
154
155 void
nfs_buf_pgs_get_page_mask(nfsbufpgs * nfsbp,off_t page)156 nfs_buf_pgs_get_page_mask(nfsbufpgs *nfsbp, off_t page)
157 {
158 off_t page_pos = page / NBPGS_ELEMENT_PAGES;
159 off_t max_page = NBPGS_STRUCT_SIZE * 8;
160 NBPGS_ERASE(nfsbp);
161
162 if (page >= max_page) {
163 nfs_buf_pgs_bit_not(nfsbp);
164 return;
165 }
166
167 NBPGS_SET(nfsbp, page);
168 nfsbp->pages[page_pos]--;
169 for (off_t i = page_pos - 1; i >= 0; i--) {
170 nfsbp->pages[i] = ~0;
171 }
172 }
173
174 void
nfs_buf_pgs_bit_not(nfsbufpgs * nfsbp)175 nfs_buf_pgs_bit_not(nfsbufpgs *nfsbp)
176 {
177 for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
178 nfsbp->pages[i] = ~nfsbp->pages[i];
179 }
180 }
181
182 void
nfs_buf_pgs_bit_and(nfsbufpgs * nfsbp_src1,nfsbufpgs * nfsbp_src2,nfsbufpgs * nfsbp_dst)183 nfs_buf_pgs_bit_and(nfsbufpgs *nfsbp_src1, nfsbufpgs *nfsbp_src2, nfsbufpgs *nfsbp_dst)
184 {
185 for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
186 nfsbp_dst->pages[i] = nfsbp_src1->pages[i] & nfsbp_src2->pages[i];
187 }
188 }
189
190 void
nfs_buf_pgs_set_pages_between(nfsbufpgs * nfsbp,off_t firstpg,off_t lastpg)191 nfs_buf_pgs_set_pages_between(nfsbufpgs *nfsbp, off_t firstpg, off_t lastpg)
192 {
193 nfsbufpgs pagemaskfirst, pagemasklast;
194
195 nfs_buf_pgs_get_page_mask(&pagemasklast, lastpg);
196 nfs_buf_pgs_get_page_mask(&pagemaskfirst, firstpg);
197 nfs_buf_pgs_bit_not(&pagemaskfirst);
198 nfs_buf_pgs_bit_and(&pagemaskfirst, &pagemasklast, nfsbp);
199 }
200
201 int
nfs_buf_pgs_is_set(nfsbufpgs * nfsbp)202 nfs_buf_pgs_is_set(nfsbufpgs *nfsbp)
203 {
204 for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
205 if (nfsbp->pages[i] != 0) {
206 return 1;
207 }
208 }
209 return 0;
210 }
211
212 /*
213 * Initialize nfsbuf lists
214 */
215 void
nfs_nbinit(void)216 nfs_nbinit(void)
217 {
218 nfsbufcnt = nfsbufmetacnt =
219 nfsbuffreecnt = nfsbuffreemetacnt = nfsbufdelwricnt = 0;
220 nfsbufmin = 128;
221 /* size nfsbufmax to cover at most half sane_size (w/default buf size) */
222 nfsbufmax = (int)(sane_size >> PAGE_SHIFT) / (2 * (NFS_RWSIZE >> PAGE_SHIFT));
223 nfsbufmetamax = nfsbufmax / 4;
224 nfsneedbuffer = 0;
225 nfs_nbdwrite = 0;
226
227 nfsbufhashtbl = hashinit(nfsbufmax / 4, M_NFSBIO, &nfsbufhash);
228 TAILQ_INIT(&nfsbuffree);
229 TAILQ_INIT(&nfsbuffreemeta);
230 TAILQ_INIT(&nfsbufdelwri);
231 }
232
233 /*
234 * Check periodically for stale/unused nfs bufs
235 */
236 void
nfs_buf_timer(__unused void * param0,__unused void * param1)237 nfs_buf_timer(__unused void *param0, __unused void *param1)
238 {
239 nfs_buf_freeup(1);
240
241 lck_mtx_lock(&nfs_buf_mutex);
242 if (nfsbufcnt <= nfsbufmin) {
243 nfs_buf_timer_on = 0;
244 lck_mtx_unlock(&nfs_buf_mutex);
245 return;
246 }
247 lck_mtx_unlock(&nfs_buf_mutex);
248
249 nfs_interval_timer_start(nfs_buf_timer_call,
250 NFSBUF_FREE_PERIOD * 1000);
251 }
252
253 /*
254 * try to free up some excess, unused nfsbufs
255 */
256 void
nfs_buf_freeup(int timer)257 nfs_buf_freeup(int timer)
258 {
259 struct nfsbuf *fbp;
260 struct timeval now;
261 int count;
262 struct nfsbuffreehead nfsbuffreeup;
263
264 TAILQ_INIT(&nfsbuffreeup);
265
266 lck_mtx_lock(&nfs_buf_mutex);
267
268 microuptime(&now);
269
270 FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, 0);
271
272 count = timer ? nfsbuffreecnt / LRU_FREEUP_FRAC_ON_TIMER : LRU_TO_FREEUP;
273 while ((nfsbufcnt > nfsbufmin) && (count-- > 0)) {
274 fbp = TAILQ_FIRST(&nfsbuffree);
275 if (!fbp) {
276 break;
277 }
278 if (os_ref_get_count(&fbp->nb_refs) > 1) {
279 break;
280 }
281 if (NBUFSTAMPVALID(fbp) &&
282 (fbp->nb_timestamp + (2 * NFSBUF_LRU_STALE)) > now.tv_sec) {
283 break;
284 }
285 nfs_buf_remfree(fbp);
286 /* disassociate buffer from any nfsnode */
287 if (fbp->nb_np) {
288 if (fbp->nb_vnbufs.le_next != NFSNOLIST) {
289 LIST_REMOVE(fbp, nb_vnbufs);
290 fbp->nb_vnbufs.le_next = NFSNOLIST;
291 }
292 fbp->nb_np = NULL;
293 }
294 LIST_REMOVE(fbp, nb_hash);
295 TAILQ_INSERT_TAIL(&nfsbuffreeup, fbp, nb_free);
296 nfsbufcnt--;
297 }
298
299 count = timer ? nfsbuffreemetacnt / META_FREEUP_FRAC_ON_TIMER : META_TO_FREEUP;
300 while ((nfsbufcnt > nfsbufmin) && (count-- > 0)) {
301 fbp = TAILQ_FIRST(&nfsbuffreemeta);
302 if (!fbp) {
303 break;
304 }
305 if (os_ref_get_count(&fbp->nb_refs) > 1) {
306 break;
307 }
308 if (NBUFSTAMPVALID(fbp) &&
309 (fbp->nb_timestamp + (2 * NFSBUF_META_STALE)) > now.tv_sec) {
310 break;
311 }
312 nfs_buf_remfree(fbp);
313 /* disassociate buffer from any nfsnode */
314 if (fbp->nb_np) {
315 if (fbp->nb_vnbufs.le_next != NFSNOLIST) {
316 LIST_REMOVE(fbp, nb_vnbufs);
317 fbp->nb_vnbufs.le_next = NFSNOLIST;
318 }
319 fbp->nb_np = NULL;
320 }
321 LIST_REMOVE(fbp, nb_hash);
322 TAILQ_INSERT_TAIL(&nfsbuffreeup, fbp, nb_free);
323 nfsbufcnt--;
324 nfsbufmetacnt--;
325 }
326
327 FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, 0);
328 NFSBUFCNTCHK();
329
330 lck_mtx_unlock(&nfs_buf_mutex);
331
332 while ((fbp = TAILQ_FIRST(&nfsbuffreeup))) {
333 TAILQ_REMOVE(&nfsbuffreeup, fbp, nb_free);
334 /* nuke any creds */
335 if (IS_VALID_CRED(fbp->nb_rcred)) {
336 kauth_cred_unref(&fbp->nb_rcred);
337 }
338 if (IS_VALID_CRED(fbp->nb_wcred)) {
339 kauth_cred_unref(&fbp->nb_wcred);
340 }
341 /* if buf was NB_META, dump buffer */
342 if (ISSET(fbp->nb_flags, NB_META) && fbp->nb_data) {
343 kfree_data(fbp->nb_data, fbp->nb_bufsize);
344 }
345 NFS_ZFREE(nfsbuf_zone, fbp);
346 }
347 }
348
349 /*
350 * remove a buffer from the freelist
351 * (must be called with nfs_buf_mutex held)
352 */
353 void
nfs_buf_remfree(struct nfsbuf * bp)354 nfs_buf_remfree(struct nfsbuf *bp)
355 {
356 if (bp->nb_free.tqe_next == NFSNOLIST) {
357 panic("nfsbuf not on free list");
358 }
359 if (ISSET(bp->nb_flags, NB_DELWRI)) {
360 nfsbufdelwricnt--;
361 TAILQ_REMOVE(&nfsbufdelwri, bp, nb_free);
362 } else if (ISSET(bp->nb_flags, NB_META)) {
363 nfsbuffreemetacnt--;
364 TAILQ_REMOVE(&nfsbuffreemeta, bp, nb_free);
365 } else {
366 nfsbuffreecnt--;
367 TAILQ_REMOVE(&nfsbuffree, bp, nb_free);
368 }
369 bp->nb_free.tqe_next = NFSNOLIST;
370 NFSBUFCNTCHK();
371 }
372
373 /*
374 * check for existence of nfsbuf in cache
375 */
376 boolean_t
nfs_buf_is_incore(nfsnode_t np,daddr64_t blkno)377 nfs_buf_is_incore(nfsnode_t np, daddr64_t blkno)
378 {
379 boolean_t rv;
380 lck_mtx_lock(&nfs_buf_mutex);
381 if (nfs_buf_incore(np, blkno)) {
382 rv = TRUE;
383 } else {
384 rv = FALSE;
385 }
386 lck_mtx_unlock(&nfs_buf_mutex);
387 return rv;
388 }
389
390 /*
391 * return incore buffer (must be called with nfs_buf_mutex held)
392 */
393 struct nfsbuf *
nfs_buf_incore(nfsnode_t np,daddr64_t blkno)394 nfs_buf_incore(nfsnode_t np, daddr64_t blkno)
395 {
396 /* Search hash chain */
397 struct nfsbuf * bp = NFSBUFHASH(np, blkno)->lh_first;
398 for (; bp != NULL; bp = bp->nb_hash.le_next) {
399 if ((bp->nb_lblkno == blkno) && (bp->nb_np == np)) {
400 if (!ISSET(bp->nb_flags, NB_INVAL)) {
401 FSDBG(547, bp, blkno, bp->nb_flags, bp->nb_np);
402 return bp;
403 }
404 }
405 }
406 return NULL;
407 }
408
409 /*
410 * Check if it's OK to drop a page.
411 *
412 * Called by vnode_pager() on pageout request of non-dirty page.
413 * We need to make sure that it's not part of a delayed write.
414 * If it is, we can't let the VM drop it because we may need it
415 * later when/if we need to write the data (again).
416 */
417 int
nfs_buf_page_inval_internal(vnode_t vp,off_t offset)418 nfs_buf_page_inval_internal(vnode_t vp, off_t offset)
419 {
420 struct nfsmount *nmp = VTONMP(vp);
421 struct nfsbuf *bp;
422 int error = 0;
423
424 if (nfs_mount_gone(nmp)) {
425 return ENXIO;
426 }
427
428 lck_mtx_lock(&nfs_buf_mutex);
429 bp = nfs_buf_incore(VTONFS(vp), (daddr64_t)(offset / nmp->nm_biosize));
430 if (!bp) {
431 goto out;
432 }
433 FSDBG(325, bp, bp->nb_flags, bp->nb_dirtyoff, bp->nb_dirtyend);
434 if (ISSET(bp->nb_lflags, NBL_BUSY)) {
435 error = EBUSY;
436 goto out;
437 }
438 /*
439 * If there's a dirty range in the buffer, check to
440 * see if this page intersects with the dirty range.
441 * If it does, we can't let the pager drop the page.
442 */
443 if (bp->nb_dirtyend > 0) {
444 off_t start = offset - NBOFF(bp);
445 if ((bp->nb_dirtyend > start) &&
446 (bp->nb_dirtyoff < (start + PAGE_SIZE))) {
447 /*
448 * Before returning the bad news, move the
449 * buffer to the start of the delwri list and
450 * give the list a push to try to flush the
451 * buffer out.
452 */
453 error = EBUSY;
454 nfs_buf_remfree(bp);
455 TAILQ_INSERT_HEAD(&nfsbufdelwri, bp, nb_free);
456 nfsbufdelwricnt++;
457 nfs_buf_delwri_push(1);
458 }
459 }
460 out:
461 lck_mtx_unlock(&nfs_buf_mutex);
462 return error;
463 }
464
465 /*
466 * set up the UPL for a buffer
467 * (must NOT be called with nfs_buf_mutex held)
468 */
469 int
nfs_buf_upl_setup(struct nfsbuf * bp)470 nfs_buf_upl_setup(struct nfsbuf *bp)
471 {
472 kern_return_t kret;
473 upl_t upl;
474 int upl_flags;
475
476 if (ISSET(bp->nb_flags, NB_PAGELIST)) {
477 return 0;
478 }
479
480 upl_flags = UPL_PRECIOUS;
481 if (!ISSET(bp->nb_flags, NB_READ)) {
482 /*
483 * We're doing a "write", so we intend to modify
484 * the pages we're gathering.
485 */
486 upl_flags |= UPL_WILL_MODIFY;
487 }
488 kret = ubc_create_upl_kernel(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize,
489 &upl, NULL, upl_flags, VM_KERN_MEMORY_FILE);
490 if (kret == KERN_INVALID_ARGUMENT) {
491 /* vm object probably doesn't exist any more */
492 bp->nb_pagelist = NULL;
493 return EINVAL;
494 }
495 if (kret != KERN_SUCCESS) {
496 printf("nfs_buf_upl_setup(): failed to get pagelist %d\n", kret);
497 bp->nb_pagelist = NULL;
498 return EIO;
499 }
500
501 FSDBG(538, bp, NBOFF(bp), bp->nb_bufsize, bp->nb_np);
502
503 bp->nb_pagelist = upl;
504 SET(bp->nb_flags, NB_PAGELIST);
505 return 0;
506 }
507
508 /*
509 * update buffer's valid/dirty info from UBC
510 * (must NOT be called with nfs_buf_mutex held)
511 */
512 void
nfs_buf_upl_check(struct nfsbuf * bp)513 nfs_buf_upl_check(struct nfsbuf *bp)
514 {
515 upl_page_info_t *pl;
516 off_t filesize, fileoffset;
517 int i, npages;
518
519 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
520 return;
521 }
522
523 npages = round_page_32(bp->nb_bufsize) / PAGE_SIZE;
524 filesize = ubc_getsize(NFSTOV(bp->nb_np));
525 fileoffset = NBOFF(bp);
526 if (fileoffset < filesize) {
527 SET(bp->nb_flags, NB_CACHE);
528 } else {
529 CLR(bp->nb_flags, NB_CACHE);
530 }
531
532 pl = ubc_upl_pageinfo(bp->nb_pagelist);
533 NBPGS_ERASE(&bp->nb_valid);
534 NBPGS_ERASE(&bp->nb_dirty);
535
536 for (i = 0; i < npages; i++, fileoffset += PAGE_SIZE_64) {
537 /* anything beyond the end of the file is not valid or dirty */
538 if (fileoffset >= filesize) {
539 break;
540 }
541 if (!upl_valid_page(pl, i)) {
542 CLR(bp->nb_flags, NB_CACHE);
543 continue;
544 }
545 NBPGVALID_SET(bp, i);
546 if (upl_dirty_page(pl, i)) {
547 NBPGDIRTY_SET(bp, i);
548 }
549 }
550 fileoffset = NBOFF(bp);
551 if (ISSET(bp->nb_flags, NB_CACHE)) {
552 bp->nb_validoff = 0;
553 bp->nb_validend = bp->nb_bufsize;
554 if (fileoffset + bp->nb_validend > filesize) {
555 bp->nb_validend = filesize - fileoffset;
556 }
557 } else {
558 bp->nb_validoff = bp->nb_validend = -1;
559 }
560 FSDBG(539, bp, fileoffset, bp->nb_valid, bp->nb_dirty);
561 FSDBG(539, bp->nb_validoff, bp->nb_validend, bp->nb_dirtyoff, bp->nb_dirtyend);
562 }
563
564 /*
565 * make sure that a buffer is mapped
566 * (must NOT be called with nfs_buf_mutex held)
567 */
568 int
nfs_buf_map(struct nfsbuf * bp)569 nfs_buf_map(struct nfsbuf *bp)
570 {
571 kern_return_t kret;
572
573 if (bp->nb_data) {
574 return 0;
575 }
576 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
577 return EINVAL;
578 }
579
580 kret = ubc_upl_map(bp->nb_pagelist, (vm_offset_t *)&(bp->nb_data));
581 if (kret != KERN_SUCCESS) {
582 panic("nfs_buf_map: ubc_upl_map() failed with (%d)", kret);
583 }
584 if (bp->nb_data == 0) {
585 panic("ubc_upl_map mapped 0");
586 }
587 FSDBG(540, bp, bp->nb_flags, NBOFF(bp), bp->nb_data);
588 return 0;
589 }
590
591 /*
592 * normalize an nfsbuf's valid range
593 *
594 * the read/write code guarantees that we'll always have a valid
595 * region that is an integral number of pages. If either end
596 * of the valid range isn't page-aligned, it gets corrected
597 * here as we extend the valid range through all of the
598 * contiguous valid pages.
599 */
600 void
nfs_buf_normalize_valid_range(nfsnode_t np,struct nfsbuf * bp)601 nfs_buf_normalize_valid_range(nfsnode_t np, struct nfsbuf *bp)
602 {
603 off_t pg, npg;
604 /* pull validoff back to start of contiguous valid page range */
605 pg = bp->nb_validoff / PAGE_SIZE;
606 while (pg >= 0 && NBPGVALID(bp, pg)) {
607 pg--;
608 }
609 bp->nb_validoff = (pg + 1) * PAGE_SIZE;
610 /* push validend forward to end of contiguous valid page range */
611 npg = bp->nb_bufsize / PAGE_SIZE;
612 pg = bp->nb_validend / PAGE_SIZE;
613 while (pg < npg && NBPGVALID(bp, pg)) {
614 pg++;
615 }
616 bp->nb_validend = pg * PAGE_SIZE;
617 /* clip to EOF */
618 if (NBOFF(bp) + bp->nb_validend > (off_t)np->n_size) {
619 bp->nb_validend = np->n_size % bp->nb_bufsize;
620 }
621 }
622
623 /*
624 * process some entries on the delayed write queue
625 * (must be called with nfs_buf_mutex held)
626 */
627 void
nfs_buf_delwri_service(void)628 nfs_buf_delwri_service(void)
629 {
630 struct nfsbuf *bp;
631 nfsnode_t np;
632 int error, i = 0;
633
634 while (i < 8 && (bp = TAILQ_FIRST(&nfsbufdelwri)) != NULL) {
635 np = bp->nb_np;
636 nfs_buf_remfree(bp);
637 nfs_buf_refget(bp);
638 while ((error = nfs_buf_acquire(bp, 0, 0, 0)) == EAGAIN) {
639 ;
640 }
641 nfs_buf_refrele(bp);
642 if (error) {
643 break;
644 }
645 if (!bp->nb_np) {
646 /* buffer is no longer valid */
647 nfs_buf_drop(bp);
648 continue;
649 }
650 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
651 nfs_buf_check_write_verifier(np, bp);
652 }
653 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
654 /* put buffer at end of delwri list */
655 TAILQ_INSERT_TAIL(&nfsbufdelwri, bp, nb_free);
656 nfsbufdelwricnt++;
657 nfs_buf_drop(bp);
658 lck_mtx_unlock(&nfs_buf_mutex);
659 nfs_flushcommits(np, 1);
660 } else {
661 SET(bp->nb_flags, NB_ASYNC);
662 lck_mtx_unlock(&nfs_buf_mutex);
663 nfs_buf_write(bp);
664 }
665 i++;
666 lck_mtx_lock(&nfs_buf_mutex);
667 }
668 }
669
670 /*
671 * thread to service the delayed write queue when asked
672 */
673 void
nfs_buf_delwri_thread(__unused void * arg,__unused wait_result_t wr)674 nfs_buf_delwri_thread(__unused void *arg, __unused wait_result_t wr)
675 {
676 struct timespec ts = { .tv_sec = 30, .tv_nsec = 0 };
677 int error = 0;
678
679 lck_mtx_lock(&nfs_buf_mutex);
680 while (!error) {
681 nfs_buf_delwri_service();
682 error = msleep(&nfsbufdelwrithd, &nfs_buf_mutex, 0, "nfsbufdelwri", &ts);
683 }
684 nfsbufdelwrithd = NULL;
685 lck_mtx_unlock(&nfs_buf_mutex);
686 thread_terminate(nfsbufdelwrithd);
687 }
688
689 /*
690 * try to push out some delayed/uncommitted writes
691 * ("locked" indicates whether nfs_buf_mutex is already held)
692 */
693 void
nfs_buf_delwri_push(int locked)694 nfs_buf_delwri_push(int locked)
695 {
696 if (TAILQ_EMPTY(&nfsbufdelwri)) {
697 return;
698 }
699 if (!locked) {
700 lck_mtx_lock(&nfs_buf_mutex);
701 }
702 /* wake up the delayed write service thread */
703 if (nfsbufdelwrithd) {
704 wakeup(&nfsbufdelwrithd);
705 } else if (kernel_thread_start(nfs_buf_delwri_thread, NULL, &nfsbufdelwrithd) == KERN_SUCCESS) {
706 thread_deallocate(nfsbufdelwrithd);
707 }
708 /* otherwise, try to do some of the work ourselves */
709 if (!nfsbufdelwrithd) {
710 nfs_buf_delwri_service();
711 }
712 if (!locked) {
713 lck_mtx_unlock(&nfs_buf_mutex);
714 }
715 }
716
717 /*
718 * Get an nfs buffer.
719 *
720 * Returns errno on error, 0 otherwise.
721 * Any buffer is returned in *bpp.
722 *
723 * If NBLK_ONLYVALID is set, only return buffer if found in cache.
724 * If NBLK_NOWAIT is set, don't wait for the buffer if it's marked BUSY.
725 *
726 * Check for existence of buffer in cache.
727 * Or attempt to reuse a buffer from one of the free lists.
728 * Or allocate a new buffer if we haven't already hit max allocation.
729 * Or wait for a free buffer.
730 *
731 * If available buffer found, prepare it, and return it.
732 *
733 * If the calling process is interrupted by a signal for
734 * an interruptible mount point, return EINTR.
735 */
736 int
nfs_buf_get(nfsnode_t np,daddr64_t blkno,uint32_t size,thread_t thd,int flags,struct nfsbuf ** bpp)737 nfs_buf_get(
738 nfsnode_t np,
739 daddr64_t blkno,
740 uint32_t size,
741 thread_t thd,
742 int flags,
743 struct nfsbuf **bpp)
744 {
745 vnode_t vp = NFSTOV(np);
746 struct nfsmount *nmp = VTONMP(vp);
747 struct nfsbuf *bp;
748 uint32_t bufsize;
749 int slpflag = PCATCH;
750 int operation = (flags & NBLK_OPMASK);
751 int error = 0;
752 struct timespec ts;
753
754 FSDBG_TOP(541, np, blkno, size, flags);
755 *bpp = NULL;
756
757 bufsize = size;
758 if (bufsize > NFS_MAXBSIZE) {
759 panic("nfs_buf_get: buffer larger than NFS_MAXBSIZE requested");
760 }
761
762 if (nfs_mount_gone(nmp)) {
763 FSDBG_BOT(541, np, blkno, 0, ENXIO);
764 return ENXIO;
765 }
766
767 if (!UBCINFOEXISTS(vp)) {
768 operation = NBLK_META;
769 } else if (bufsize < (uint32_t)nmp->nm_biosize) {
770 /* reg files should always have biosize blocks */
771 bufsize = nmp->nm_biosize;
772 }
773
774 /* if NBLK_WRITE, check for too many delayed/uncommitted writes */
775 if ((operation == NBLK_WRITE) && (nfs_nbdwrite > NFS_A_LOT_OF_DELAYED_WRITES)) {
776 FSDBG_TOP(542, np, blkno, nfs_nbdwrite, NFS_A_LOT_OF_DELAYED_WRITES);
777
778 /* poke the delwri list */
779 nfs_buf_delwri_push(0);
780
781 /* sleep to let other threads run... */
782 tsleep(&nfs_nbdwrite, PCATCH, "nfs_nbdwrite", 1);
783 FSDBG_BOT(542, np, blkno, nfs_nbdwrite, NFS_A_LOT_OF_DELAYED_WRITES);
784 }
785
786 loop:
787 lck_mtx_lock(&nfs_buf_mutex);
788
789 /* wait for any buffer invalidation/flushing to complete */
790 while (np->n_bflag & NBINVALINPROG) {
791 np->n_bflag |= NBINVALWANT;
792 ts.tv_sec = 2;
793 ts.tv_nsec = 0;
794 msleep(&np->n_bflag, &nfs_buf_mutex, slpflag, "nfs_buf_get_invalwait", &ts);
795 if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0))) {
796 lck_mtx_unlock(&nfs_buf_mutex);
797 FSDBG_BOT(541, np, blkno, 0, error);
798 return error;
799 }
800 if (np->n_bflag & NBINVALINPROG) {
801 slpflag = 0;
802 }
803 }
804
805 /* check for existence of nfsbuf in cache */
806 if ((bp = nfs_buf_incore(np, blkno))) {
807 /* if busy, set wanted and wait */
808 if (ISSET(bp->nb_lflags, NBL_BUSY)) {
809 if (flags & NBLK_NOWAIT) {
810 lck_mtx_unlock(&nfs_buf_mutex);
811 FSDBG_BOT(541, np, blkno, bp, 0xbcbcbcbc);
812 return 0;
813 }
814 FSDBG_TOP(543, np, blkno, bp, bp->nb_flags);
815 SET(bp->nb_lflags, NBL_WANTED);
816
817 ts.tv_sec = 2;
818 ts.tv_nsec = 0;
819 msleep(bp, &nfs_buf_mutex, slpflag | (PRIBIO + 1) | PDROP,
820 "nfsbufget", (slpflag == PCATCH) ? NULL : &ts);
821 slpflag = 0;
822 FSDBG_BOT(543, np, blkno, bp, bp->nb_flags);
823 if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0))) {
824 FSDBG_BOT(541, np, blkno, 0, error);
825 return error;
826 }
827 goto loop;
828 }
829 if (bp->nb_bufsize != bufsize) {
830 panic("nfsbuf size mismatch");
831 }
832 SET(bp->nb_lflags, NBL_BUSY);
833 SET(bp->nb_flags, NB_CACHE);
834 nfs_buf_remfree(bp);
835 /* additional paranoia: */
836 if (ISSET(bp->nb_flags, NB_PAGELIST)) {
837 panic("pagelist buffer was not busy");
838 }
839 goto buffer_setup;
840 }
841
842 if (flags & NBLK_ONLYVALID) {
843 lck_mtx_unlock(&nfs_buf_mutex);
844 FSDBG_BOT(541, np, blkno, 0, 0x0000cace);
845 return 0;
846 }
847
848 /*
849 * where to get a free buffer:
850 * - if meta and maxmeta reached, must reuse meta
851 * - alloc new if we haven't reached min bufs
852 * - if free lists are NOT empty
853 * - if free list is stale, use it
854 * - else if freemeta list is stale, use it
855 * - else if max bufs allocated, use least-time-to-stale
856 * - alloc new if we haven't reached max allowed
857 * - start clearing out delwri list and try again
858 */
859
860 if ((operation == NBLK_META) && (nfsbufmetacnt >= nfsbufmetamax)) {
861 /* if we've hit max meta buffers, must reuse a meta buffer */
862 bp = TAILQ_FIRST(&nfsbuffreemeta);
863 } else if ((nfsbufcnt > nfsbufmin) &&
864 (!TAILQ_EMPTY(&nfsbuffree) || !TAILQ_EMPTY(&nfsbuffreemeta))) {
865 /* try to pull an nfsbuf off a free list */
866 struct nfsbuf *lrubp, *metabp;
867 struct timeval now;
868 microuptime(&now);
869
870 /* if the next LRU or META buffer is invalid or stale, use it */
871 lrubp = TAILQ_FIRST(&nfsbuffree);
872 if (lrubp && (!NBUFSTAMPVALID(lrubp) ||
873 ((lrubp->nb_timestamp + NFSBUF_LRU_STALE) < now.tv_sec))) {
874 bp = lrubp;
875 }
876 metabp = TAILQ_FIRST(&nfsbuffreemeta);
877 if (!bp && metabp && (!NBUFSTAMPVALID(metabp) ||
878 ((metabp->nb_timestamp + NFSBUF_META_STALE) < now.tv_sec))) {
879 bp = metabp;
880 }
881
882 if (!bp && (nfsbufcnt >= nfsbufmax)) {
883 /* we've already allocated all bufs, so */
884 /* choose the buffer that'll go stale first */
885 if (!metabp) {
886 bp = lrubp;
887 } else if (!lrubp) {
888 bp = metabp;
889 } else {
890 time_t lru_stale_time, meta_stale_time;
891 lru_stale_time = lrubp->nb_timestamp + NFSBUF_LRU_STALE;
892 meta_stale_time = metabp->nb_timestamp + NFSBUF_META_STALE;
893 if (lru_stale_time <= meta_stale_time) {
894 bp = lrubp;
895 } else {
896 bp = metabp;
897 }
898 }
899 }
900 }
901
902 if (bp) {
903 /* we have a buffer to reuse */
904 FSDBG(544, np, blkno, bp, bp->nb_flags);
905 nfs_buf_remfree(bp);
906 if (ISSET(bp->nb_flags, NB_DELWRI)) {
907 panic("nfs_buf_get: delwri");
908 }
909 SET(bp->nb_lflags, NBL_BUSY);
910 /* disassociate buffer from previous nfsnode */
911 if (bp->nb_np) {
912 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
913 LIST_REMOVE(bp, nb_vnbufs);
914 bp->nb_vnbufs.le_next = NFSNOLIST;
915 }
916 bp->nb_np = NULL;
917 }
918 LIST_REMOVE(bp, nb_hash);
919 /* nuke any creds we're holding */
920 if (IS_VALID_CRED(bp->nb_rcred)) {
921 kauth_cred_unref(&bp->nb_rcred);
922 }
923 if (IS_VALID_CRED(bp->nb_wcred)) {
924 kauth_cred_unref(&bp->nb_wcred);
925 }
926 /* if buf will no longer be NB_META, dump old buffer */
927 if (operation == NBLK_META) {
928 if (!ISSET(bp->nb_flags, NB_META)) {
929 nfsbufmetacnt++;
930 }
931 } else if (ISSET(bp->nb_flags, NB_META)) {
932 if (bp->nb_data) {
933 kfree_data(bp->nb_data, bp->nb_bufsize);
934 bp->nb_data = NULL;
935 }
936 nfsbufmetacnt--;
937 }
938 /* re-init buf fields */
939 bp->nb_error = 0;
940 bp->nb_validoff = bp->nb_validend = -1;
941 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
942 NBPGS_ERASE(&bp->nb_valid);
943 NBPGS_ERASE(&bp->nb_dirty);
944 bp->nb_verf = 0;
945 } else {
946 /* no buffer to reuse */
947 if ((nfsbufcnt < nfsbufmax) &&
948 ((operation != NBLK_META) || (nfsbufmetacnt < nfsbufmetamax))) {
949 /* just alloc a new one */
950 bp = zalloc_flags(nfsbuf_zone, Z_WAITOK | Z_ZERO);
951 nfsbufcnt++;
952
953 /*
954 * If any excess bufs, make sure the timer
955 * is running to free them up later.
956 */
957 if (nfsbufcnt > nfsbufmin && !nfs_buf_timer_on) {
958 nfs_buf_timer_on = 1;
959 nfs_interval_timer_start(nfs_buf_timer_call,
960 NFSBUF_FREE_PERIOD * 1000);
961 }
962
963 if (operation == NBLK_META) {
964 nfsbufmetacnt++;
965 }
966 NFSBUFCNTCHK();
967 /* init nfsbuf */
968 os_ref_init(&bp->nb_refs, NULL);
969
970 bp->nb_free.tqe_next = NFSNOLIST;
971 bp->nb_validoff = bp->nb_validend = -1;
972 FSDBG(545, np, blkno, bp, 0);
973 } else {
974 /* too many bufs... wait for buffers to free up */
975 FSDBG_TOP(546, np, blkno, nfsbufcnt, nfsbufmax);
976
977 /* poke the delwri list */
978 nfs_buf_delwri_push(1);
979
980 nfsneedbuffer = 1;
981 msleep(&nfsneedbuffer, &nfs_buf_mutex, PCATCH | PDROP, "nfsbufget", NULL);
982 FSDBG_BOT(546, np, blkno, nfsbufcnt, nfsbufmax);
983 if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0))) {
984 FSDBG_BOT(541, np, blkno, 0, error);
985 return error;
986 }
987 goto loop;
988 }
989 }
990
991 /* set up nfsbuf */
992 SET(bp->nb_lflags, NBL_BUSY);
993 bp->nb_flags = 0;
994 bp->nb_lblkno = blkno;
995 /* insert buf in hash */
996 LIST_INSERT_HEAD(NFSBUFHASH(np, blkno), bp, nb_hash);
997 /* associate buffer with new nfsnode */
998 bp->nb_np = np;
999 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
1000
1001 buffer_setup:
1002
1003 /* unlock hash */
1004 lck_mtx_unlock(&nfs_buf_mutex);
1005
1006 switch (operation) {
1007 case NBLK_META:
1008 SET(bp->nb_flags, NB_META);
1009 if ((bp->nb_bufsize != bufsize) && bp->nb_data) {
1010 kfree_data(bp->nb_data, bp->nb_bufsize);
1011 bp->nb_data = NULL;
1012 bp->nb_validoff = bp->nb_validend = -1;
1013 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
1014 NBPGS_ERASE(&bp->nb_valid);
1015 NBPGS_ERASE(&bp->nb_dirty);
1016 CLR(bp->nb_flags, NB_CACHE);
1017 }
1018 if (!bp->nb_data) {
1019 bp->nb_data = kalloc_data(bufsize, Z_WAITOK);
1020 }
1021 if (!bp->nb_data) {
1022 /* Ack! couldn't allocate the data buffer! */
1023 /* clean up buffer and return error */
1024 lck_mtx_lock(&nfs_buf_mutex);
1025 LIST_REMOVE(bp, nb_vnbufs);
1026 bp->nb_vnbufs.le_next = NFSNOLIST;
1027 bp->nb_np = NULL;
1028 /* invalidate usage timestamp to allow immediate freeing */
1029 NBUFSTAMPINVALIDATE(bp);
1030 if (bp->nb_free.tqe_next != NFSNOLIST) {
1031 panic("nfsbuf on freelist");
1032 }
1033 TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free);
1034 nfsbuffreecnt++;
1035 lck_mtx_unlock(&nfs_buf_mutex);
1036 FSDBG_BOT(541, np, blkno, 0xb00, ENOMEM);
1037 return ENOMEM;
1038 }
1039 bp->nb_bufsize = bufsize;
1040 break;
1041
1042 case NBLK_READ:
1043 case NBLK_WRITE:
1044 /*
1045 * Set or clear NB_READ now to let the UPL subsystem know
1046 * if we intend to modify the pages or not.
1047 */
1048 if (operation == NBLK_READ) {
1049 SET(bp->nb_flags, NB_READ);
1050 } else {
1051 CLR(bp->nb_flags, NB_READ);
1052 }
1053 if (bufsize < PAGE_SIZE) {
1054 bufsize = PAGE_SIZE;
1055 }
1056 bp->nb_bufsize = bufsize;
1057 bp->nb_validoff = bp->nb_validend = -1;
1058
1059 if (UBCINFOEXISTS(vp)) {
1060 /* set up upl */
1061 if (nfs_buf_upl_setup(bp)) {
1062 /* unable to create upl */
1063 /* vm object must no longer exist */
1064 /* clean up buffer and return error */
1065 lck_mtx_lock(&nfs_buf_mutex);
1066 LIST_REMOVE(bp, nb_vnbufs);
1067 bp->nb_vnbufs.le_next = NFSNOLIST;
1068 bp->nb_np = NULL;
1069 /* invalidate usage timestamp to allow immediate freeing */
1070 NBUFSTAMPINVALIDATE(bp);
1071 if (bp->nb_free.tqe_next != NFSNOLIST) {
1072 panic("nfsbuf on freelist");
1073 }
1074 TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free);
1075 nfsbuffreecnt++;
1076 lck_mtx_unlock(&nfs_buf_mutex);
1077 FSDBG_BOT(541, np, blkno, 0x2bc, EIO);
1078 return EIO;
1079 }
1080 nfs_buf_upl_check(bp);
1081 }
1082 break;
1083
1084 default:
1085 panic("nfs_buf_get: %d unknown operation", operation);
1086 }
1087
1088 *bpp = bp;
1089
1090 FSDBG_BOT(541, np, blkno, bp, bp->nb_flags);
1091
1092 return 0;
1093 }
1094
1095 void
nfs_buf_release(struct nfsbuf * bp,int freeup)1096 nfs_buf_release(struct nfsbuf *bp, int freeup)
1097 {
1098 nfsnode_t np = bp->nb_np;
1099 vnode_t vp;
1100 struct timeval now;
1101 int wakeup_needbuffer, wakeup_buffer, wakeup_nbdwrite;
1102
1103 FSDBG_TOP(548, bp, NBOFF(bp), bp->nb_flags, bp->nb_data);
1104 FSDBG(548, bp->nb_validoff, bp->nb_validend, bp->nb_dirtyoff, bp->nb_dirtyend);
1105 FSDBG(548, bp->nb_valid, 0, bp->nb_dirty, 0);
1106
1107 vp = np ? NFSTOV(np) : NULL;
1108 if (vp && UBCINFOEXISTS(vp) && bp->nb_bufsize) {
1109 int upl_flags, rv;
1110 upl_t upl;
1111 uint32_t i;
1112
1113 if (!ISSET(bp->nb_flags, NB_PAGELIST) && !ISSET(bp->nb_flags, NB_INVAL)) {
1114 rv = nfs_buf_upl_setup(bp);
1115 if (rv) {
1116 printf("nfs_buf_release: upl create failed %d\n", rv);
1117 } else {
1118 nfs_buf_upl_check(bp);
1119 }
1120 }
1121 upl = bp->nb_pagelist;
1122 if (!upl) {
1123 goto pagelist_cleanup_done;
1124 }
1125 if (bp->nb_data) {
1126 if (ubc_upl_unmap(upl) != KERN_SUCCESS) {
1127 panic("ubc_upl_unmap failed");
1128 }
1129 bp->nb_data = NULL;
1130 }
1131 /*
1132 * Abort the pages on error or: if this is an invalid or
1133 * non-needcommit nocache buffer AND no pages are dirty.
1134 */
1135 if (ISSET(bp->nb_flags, NB_ERROR) || (!nfs_buf_pgs_is_set(&bp->nb_dirty) && (ISSET(bp->nb_flags, NB_INVAL) ||
1136 (ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI)))))) {
1137 if (ISSET(bp->nb_flags, (NB_READ | NB_INVAL | NB_NOCACHE))) {
1138 upl_flags = UPL_ABORT_DUMP_PAGES;
1139 } else {
1140 upl_flags = 0;
1141 }
1142 ubc_upl_abort(upl, upl_flags);
1143 goto pagelist_cleanup_done;
1144 }
1145 for (i = 0; i <= (bp->nb_bufsize - 1) / PAGE_SIZE; i++) {
1146 if (!NBPGVALID(bp, i)) {
1147 ubc_upl_abort_range(upl,
1148 i * PAGE_SIZE, PAGE_SIZE,
1149 UPL_ABORT_DUMP_PAGES |
1150 UPL_ABORT_FREE_ON_EMPTY);
1151 } else {
1152 if (NBPGDIRTY(bp, i)) {
1153 upl_flags = UPL_COMMIT_SET_DIRTY;
1154 } else {
1155 upl_flags = UPL_COMMIT_CLEAR_DIRTY;
1156 }
1157
1158 if (!ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI))) {
1159 upl_flags |= UPL_COMMIT_CLEAR_PRECIOUS;
1160 }
1161
1162 ubc_upl_commit_range(upl,
1163 i * PAGE_SIZE, PAGE_SIZE,
1164 upl_flags |
1165 UPL_COMMIT_INACTIVATE |
1166 UPL_COMMIT_FREE_ON_EMPTY);
1167 }
1168 }
1169 pagelist_cleanup_done:
1170 /* invalidate any pages past EOF */
1171 if (NBOFF(bp) + bp->nb_bufsize > (off_t)(np->n_size)) {
1172 off_t start, end;
1173 start = trunc_page_64(np->n_size) + PAGE_SIZE_64;
1174 end = trunc_page_64(NBOFF(bp) + bp->nb_bufsize);
1175 if (start < NBOFF(bp)) {
1176 start = NBOFF(bp);
1177 }
1178 if (end > start) {
1179 if ((rv = ubc_msync(vp, start, end, NULL, UBC_INVALIDATE))) {
1180 printf("nfs_buf_release(): ubc_msync failed!, error %d\n", rv);
1181 }
1182 }
1183 }
1184 CLR(bp->nb_flags, NB_PAGELIST);
1185 bp->nb_pagelist = NULL;
1186 }
1187
1188 lck_mtx_lock(&nfs_buf_mutex);
1189
1190 wakeup_needbuffer = wakeup_buffer = wakeup_nbdwrite = 0;
1191
1192 /* Wake up any processes waiting for any buffer to become free. */
1193 if (nfsneedbuffer) {
1194 nfsneedbuffer = 0;
1195 wakeup_needbuffer = 1;
1196 }
1197 /* Wake up any processes waiting for _this_ buffer to become free. */
1198 if (ISSET(bp->nb_lflags, NBL_WANTED)) {
1199 CLR(bp->nb_lflags, NBL_WANTED);
1200 wakeup_buffer = 1;
1201 }
1202
1203 /* If it's non-needcommit nocache, or an error, mark it invalid. */
1204 if (ISSET(bp->nb_flags, NB_ERROR) ||
1205 (ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI)))) {
1206 SET(bp->nb_flags, NB_INVAL);
1207 }
1208
1209 if ((bp->nb_bufsize <= 0) || ISSET(bp->nb_flags, NB_INVAL)) {
1210 /* If it's invalid or empty, dissociate it from its nfsnode */
1211 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
1212 LIST_REMOVE(bp, nb_vnbufs);
1213 bp->nb_vnbufs.le_next = NFSNOLIST;
1214 }
1215 bp->nb_np = NULL;
1216 /* if this was a delayed write, wakeup anyone */
1217 /* waiting for delayed writes to complete */
1218 if (ISSET(bp->nb_flags, NB_DELWRI)) {
1219 CLR(bp->nb_flags, NB_DELWRI);
1220 nfs_nbdwrite--;
1221 NFSBUFCNTCHK();
1222 wakeup_nbdwrite = 1;
1223 }
1224 /* invalidate usage timestamp to allow immediate freeing */
1225 NBUFSTAMPINVALIDATE(bp);
1226 /* put buffer at head of free list */
1227 if (bp->nb_free.tqe_next != NFSNOLIST) {
1228 panic("nfsbuf on freelist");
1229 }
1230 SET(bp->nb_flags, NB_INVAL);
1231 if (ISSET(bp->nb_flags, NB_META)) {
1232 TAILQ_INSERT_HEAD(&nfsbuffreemeta, bp, nb_free);
1233 nfsbuffreemetacnt++;
1234 } else {
1235 TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free);
1236 nfsbuffreecnt++;
1237 }
1238 } else if (ISSET(bp->nb_flags, NB_DELWRI)) {
1239 /* put buffer at end of delwri list */
1240 if (bp->nb_free.tqe_next != NFSNOLIST) {
1241 panic("nfsbuf on freelist");
1242 }
1243 TAILQ_INSERT_TAIL(&nfsbufdelwri, bp, nb_free);
1244 nfsbufdelwricnt++;
1245 freeup = 0;
1246 } else {
1247 /* update usage timestamp */
1248 microuptime(&now);
1249 bp->nb_timestamp = now.tv_sec;
1250 /* put buffer at end of free list */
1251 if (bp->nb_free.tqe_next != NFSNOLIST) {
1252 panic("nfsbuf on freelist");
1253 }
1254 if (ISSET(bp->nb_flags, NB_META)) {
1255 TAILQ_INSERT_TAIL(&nfsbuffreemeta, bp, nb_free);
1256 nfsbuffreemetacnt++;
1257 } else {
1258 TAILQ_INSERT_TAIL(&nfsbuffree, bp, nb_free);
1259 nfsbuffreecnt++;
1260 }
1261 }
1262
1263 NFSBUFCNTCHK();
1264
1265 /* Unlock the buffer. */
1266 CLR(bp->nb_flags, (NB_ASYNC | NB_STABLE));
1267 CLR(bp->nb_lflags, NBL_BUSY);
1268
1269 FSDBG_BOT(548, bp, NBOFF(bp), bp->nb_flags, bp->nb_data);
1270
1271 lck_mtx_unlock(&nfs_buf_mutex);
1272
1273 if (wakeup_needbuffer) {
1274 wakeup(&nfsneedbuffer);
1275 }
1276 if (wakeup_buffer) {
1277 wakeup(bp);
1278 }
1279 if (wakeup_nbdwrite) {
1280 wakeup(&nfs_nbdwrite);
1281 }
1282 if (freeup) {
1283 NFS_BUF_FREEUP();
1284 }
1285 }
1286
1287 /*
1288 * Wait for operations on the buffer to complete.
1289 * When they do, extract and return the I/O's error value.
1290 */
1291 int
nfs_buf_iowait(struct nfsbuf * bp)1292 nfs_buf_iowait(struct nfsbuf *bp)
1293 {
1294 FSDBG_TOP(549, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
1295
1296 lck_mtx_lock(&nfs_buf_mutex);
1297
1298 while (!ISSET(bp->nb_flags, NB_DONE)) {
1299 msleep(bp, &nfs_buf_mutex, PRIBIO + 1, "nfs_buf_iowait", NULL);
1300 }
1301
1302 lck_mtx_unlock(&nfs_buf_mutex);
1303
1304 FSDBG_BOT(549, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
1305
1306 /* check for interruption of I/O, then errors. */
1307 if (ISSET(bp->nb_flags, NB_EINTR)) {
1308 CLR(bp->nb_flags, NB_EINTR);
1309 return EINTR;
1310 } else if (ISSET(bp->nb_flags, NB_ERROR)) {
1311 return bp->nb_error ? bp->nb_error : EIO;
1312 }
1313 return 0;
1314 }
1315
1316 /*
1317 * Mark I/O complete on a buffer.
1318 */
1319 void
nfs_buf_iodone(struct nfsbuf * bp)1320 nfs_buf_iodone(struct nfsbuf *bp)
1321 {
1322 FSDBG_TOP(550, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
1323
1324 if (ISSET(bp->nb_flags, NB_DONE)) {
1325 panic("nfs_buf_iodone already");
1326 }
1327
1328 if (!ISSET(bp->nb_flags, NB_READ)) {
1329 CLR(bp->nb_flags, NB_WRITEINPROG);
1330 /*
1331 * vnode_writedone() takes care of waking up
1332 * any throttled write operations
1333 */
1334 vnode_writedone(NFSTOV(bp->nb_np));
1335 nfs_node_lock_force(bp->nb_np);
1336 bp->nb_np->n_numoutput--;
1337 nfs_node_unlock(bp->nb_np);
1338 }
1339 if (ISSET(bp->nb_flags, NB_ASYNC)) { /* if async, release it */
1340 SET(bp->nb_flags, NB_DONE); /* note that it's done */
1341 nfs_buf_release(bp, 1);
1342 } else { /* or just wakeup the buffer */
1343 lck_mtx_lock(&nfs_buf_mutex);
1344 SET(bp->nb_flags, NB_DONE); /* note that it's done */
1345 CLR(bp->nb_lflags, NBL_WANTED);
1346 lck_mtx_unlock(&nfs_buf_mutex);
1347 wakeup(bp);
1348 }
1349
1350 FSDBG_BOT(550, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
1351 }
1352
1353 void
nfs_buf_write_delayed(struct nfsbuf * bp)1354 nfs_buf_write_delayed(struct nfsbuf *bp)
1355 {
1356 nfsnode_t np = bp->nb_np;
1357
1358 FSDBG_TOP(551, bp, NBOFF(bp), bp->nb_flags, 0);
1359 FSDBG(551, bp, bp->nb_dirtyoff, bp->nb_dirtyend, bp->nb_dirty);
1360
1361 /*
1362 * If the block hasn't been seen before:
1363 * (1) Mark it as having been seen,
1364 * (2) Make sure it's on its node's correct block list,
1365 */
1366 if (!ISSET(bp->nb_flags, NB_DELWRI)) {
1367 SET(bp->nb_flags, NB_DELWRI);
1368 /* move to dirty list */
1369 lck_mtx_lock(&nfs_buf_mutex);
1370 nfs_nbdwrite++;
1371 NFSBUFCNTCHK();
1372 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
1373 LIST_REMOVE(bp, nb_vnbufs);
1374 }
1375 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
1376 lck_mtx_unlock(&nfs_buf_mutex);
1377 }
1378
1379 /*
1380 * If the vnode has "too many" write operations in progress
1381 * wait for them to finish the IO
1382 */
1383 vnode_waitforwrites(NFSTOV(np), VNODE_ASYNC_THROTTLE, 0, 0, "nfs_buf_write_delayed");
1384
1385 /* the file is in a modified state, so make sure the flag's set */
1386 nfs_node_lock_force(np);
1387 np->n_flag |= NMODIFIED;
1388 nfs_node_unlock(np);
1389
1390 /*
1391 * If we have too many delayed write buffers,
1392 * just fall back to doing the async write.
1393 */
1394 if (nfs_nbdwrite < 0) {
1395 panic("nfs_buf_write_delayed: Negative nfs_nbdwrite");
1396 }
1397 if (nfs_nbdwrite > NFS_A_LOT_OF_DELAYED_WRITES) {
1398 /* issue async write */
1399 SET(bp->nb_flags, NB_ASYNC);
1400 nfs_buf_write(bp);
1401 FSDBG_BOT(551, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
1402 return;
1403 }
1404
1405 /* Otherwise, the "write" is done, so mark and release the buffer. */
1406 SET(bp->nb_flags, NB_DONE);
1407 nfs_buf_release(bp, 1);
1408 FSDBG_BOT(551, bp, NBOFF(bp), bp->nb_flags, 0);
1409 return;
1410 }
1411
1412 /*
1413 * Check that a "needcommit" buffer can still be committed.
1414 * If the write verifier has changed, we need to clear the
1415 * the needcommit flag.
1416 */
1417 void
nfs_buf_check_write_verifier(nfsnode_t np,struct nfsbuf * bp)1418 nfs_buf_check_write_verifier(nfsnode_t np, struct nfsbuf *bp)
1419 {
1420 struct nfsmount *nmp;
1421
1422 if (!ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
1423 return;
1424 }
1425
1426 nmp = NFSTONMP(np);
1427 if (nfs_mount_gone(nmp)) {
1428 return;
1429 }
1430 if (!ISSET(bp->nb_flags, NB_STALEWVERF) && (bp->nb_verf == nmp->nm_verf)) {
1431 return;
1432 }
1433
1434 /* write verifier changed, clear commit/wverf flags */
1435 CLR(bp->nb_flags, (NB_NEEDCOMMIT | NB_STALEWVERF));
1436 bp->nb_verf = 0;
1437 nfs_node_lock_force(np);
1438 np->n_needcommitcnt--;
1439 CHECK_NEEDCOMMITCNT(np);
1440 nfs_node_unlock(np);
1441 }
1442
1443 /*
1444 * add a reference to a buffer so it doesn't disappear while being used
1445 * (must be called with nfs_buf_mutex held)
1446 */
1447 void
nfs_buf_refget(struct nfsbuf * bp)1448 nfs_buf_refget(struct nfsbuf *bp)
1449 {
1450 os_ref_retain_locked(&bp->nb_refs);
1451 }
1452 /*
1453 * release a reference on a buffer
1454 * (must be called with nfs_buf_mutex held)
1455 */
1456 void
nfs_buf_refrele(struct nfsbuf * bp)1457 nfs_buf_refrele(struct nfsbuf *bp)
1458 {
1459 (void) os_ref_release_locked(&bp->nb_refs);
1460 }
1461
1462 /*
1463 * mark a particular buffer as BUSY
1464 * (must be called with nfs_buf_mutex held)
1465 */
1466 errno_t
nfs_buf_acquire(struct nfsbuf * bp,int flags,int slpflag,int slptimeo)1467 nfs_buf_acquire(struct nfsbuf *bp, int flags, int slpflag, int slptimeo)
1468 {
1469 errno_t error;
1470 struct timespec ts;
1471
1472 if (ISSET(bp->nb_lflags, NBL_BUSY)) {
1473 /*
1474 * since the lck_mtx_lock may block, the buffer
1475 * may become BUSY, so we need to recheck for
1476 * a NOWAIT request
1477 */
1478 if (flags & NBAC_NOWAIT) {
1479 return EBUSY;
1480 }
1481 SET(bp->nb_lflags, NBL_WANTED);
1482
1483 ts.tv_sec = (slptimeo / 100);
1484 /* the hz value is 100; which leads to 10ms */
1485 ts.tv_nsec = (slptimeo % 100) * 10 * NSEC_PER_USEC * 1000;
1486
1487 error = msleep(bp, &nfs_buf_mutex, slpflag | (PRIBIO + 1),
1488 "nfs_buf_acquire", &ts);
1489 if (error) {
1490 return error;
1491 }
1492 return EAGAIN;
1493 }
1494 if (flags & NBAC_REMOVE) {
1495 nfs_buf_remfree(bp);
1496 }
1497 SET(bp->nb_lflags, NBL_BUSY);
1498
1499 return 0;
1500 }
1501
1502 /*
1503 * simply drop the BUSY status of a buffer
1504 * (must be called with nfs_buf_mutex held)
1505 */
1506 void
nfs_buf_drop(struct nfsbuf * bp)1507 nfs_buf_drop(struct nfsbuf *bp)
1508 {
1509 int need_wakeup = 0;
1510
1511 if (!ISSET(bp->nb_lflags, NBL_BUSY)) {
1512 panic("nfs_buf_drop: buffer not busy!");
1513 }
1514 if (ISSET(bp->nb_lflags, NBL_WANTED)) {
1515 /* delay the actual wakeup until after we clear NBL_BUSY */
1516 need_wakeup = 1;
1517 }
1518 /* Unlock the buffer. */
1519 CLR(bp->nb_lflags, (NBL_BUSY | NBL_WANTED));
1520
1521 if (need_wakeup) {
1522 wakeup(bp);
1523 }
1524 }
1525
1526 /*
1527 * prepare for iterating over an nfsnode's buffer list
1528 * this lock protects the queue manipulation
1529 * (must be called with nfs_buf_mutex held)
1530 */
1531 int
nfs_buf_iterprepare(nfsnode_t np,struct nfsbuflists * iterheadp,int flags)1532 nfs_buf_iterprepare(nfsnode_t np, struct nfsbuflists *iterheadp, int flags)
1533 {
1534 struct nfsbuflists *listheadp;
1535
1536 if (flags & NBI_DIRTY) {
1537 listheadp = &np->n_dirtyblkhd;
1538 } else {
1539 listheadp = &np->n_cleanblkhd;
1540 }
1541
1542 if ((flags & NBI_NOWAIT) && (np->n_bufiterflags & NBI_ITER)) {
1543 LIST_INIT(iterheadp);
1544 return EWOULDBLOCK;
1545 }
1546
1547 while (np->n_bufiterflags & NBI_ITER) {
1548 np->n_bufiterflags |= NBI_ITERWANT;
1549 msleep(&np->n_bufiterflags, &nfs_buf_mutex, 0, "nfs_buf_iterprepare", NULL);
1550 }
1551 if (LIST_EMPTY(listheadp)) {
1552 LIST_INIT(iterheadp);
1553 return EINVAL;
1554 }
1555 np->n_bufiterflags |= NBI_ITER;
1556
1557 iterheadp->lh_first = listheadp->lh_first;
1558 listheadp->lh_first->nb_vnbufs.le_prev = &iterheadp->lh_first;
1559 LIST_INIT(listheadp);
1560
1561 return 0;
1562 }
1563
1564 /*
1565 * clean up after iterating over an nfsnode's buffer list
1566 * this lock protects the queue manipulation
1567 * (must be called with nfs_buf_mutex held)
1568 */
1569 void
nfs_buf_itercomplete(nfsnode_t np,struct nfsbuflists * iterheadp,int flags)1570 nfs_buf_itercomplete(nfsnode_t np, struct nfsbuflists *iterheadp, int flags)
1571 {
1572 struct nfsbuflists * listheadp;
1573 struct nfsbuf *bp;
1574
1575 if (flags & NBI_DIRTY) {
1576 listheadp = &np->n_dirtyblkhd;
1577 } else {
1578 listheadp = &np->n_cleanblkhd;
1579 }
1580
1581 while (!LIST_EMPTY(iterheadp)) {
1582 bp = LIST_FIRST(iterheadp);
1583 LIST_REMOVE(bp, nb_vnbufs);
1584 LIST_INSERT_HEAD(listheadp, bp, nb_vnbufs);
1585 }
1586
1587 np->n_bufiterflags &= ~NBI_ITER;
1588 if (np->n_bufiterflags & NBI_ITERWANT) {
1589 np->n_bufiterflags &= ~NBI_ITERWANT;
1590 wakeup(&np->n_bufiterflags);
1591 }
1592 }
1593
1594
1595 /*
1596 * Read an NFS buffer for a file.
1597 */
1598 int
nfs_buf_read(struct nfsbuf * bp)1599 nfs_buf_read(struct nfsbuf *bp)
1600 {
1601 int error = 0;
1602 nfsnode_t np;
1603 thread_t thd;
1604 kauth_cred_t cred;
1605
1606 np = bp->nb_np;
1607 cred = bp->nb_rcred;
1608 if (IS_VALID_CRED(cred)) {
1609 kauth_cred_ref(cred);
1610 }
1611 thd = ISSET(bp->nb_flags, NB_ASYNC) ? NULL : current_thread();
1612
1613 /* sanity checks */
1614 if (!ISSET(bp->nb_flags, NB_READ)) {
1615 panic("nfs_buf_read: !NB_READ");
1616 }
1617 if (ISSET(bp->nb_flags, NB_DONE)) {
1618 CLR(bp->nb_flags, NB_DONE);
1619 }
1620
1621 NFS_BUF_MAP(bp);
1622
1623 OSAddAtomic64(1, &nfsclntstats.read_bios);
1624
1625 error = nfs_buf_read_rpc(bp, thd, cred);
1626 /*
1627 * For async I/O, the callbacks will finish up the
1628 * read. Otherwise, the read has already been finished.
1629 */
1630
1631 if (IS_VALID_CRED(cred)) {
1632 kauth_cred_unref(&cred);
1633 }
1634 return error;
1635 }
1636
1637 /*
1638 * finish the reading of a buffer
1639 */
1640 void
nfs_buf_read_finish(struct nfsbuf * bp)1641 nfs_buf_read_finish(struct nfsbuf *bp)
1642 {
1643 nfsnode_t np = bp->nb_np;
1644 struct nfsmount *nmp;
1645
1646 if (!ISSET(bp->nb_flags, NB_ERROR)) {
1647 /* update valid range */
1648 bp->nb_validoff = 0;
1649 bp->nb_validend = bp->nb_endio;
1650 if (bp->nb_endio < bp->nb_bufsize) {
1651 /*
1652 * The read may be short because we have unflushed writes
1653 * that are extending the file size and the reads hit the
1654 * (old) EOF on the server. So, just make sure nb_validend
1655 * correctly tracks EOF.
1656 * Note that the missing data should have already been zeroed
1657 * in nfs_buf_read_rpc_finish().
1658 */
1659 off_t boff = NBOFF(bp);
1660 if ((off_t)np->n_size >= (boff + bp->nb_bufsize)) {
1661 bp->nb_validend = bp->nb_bufsize;
1662 } else if ((off_t)np->n_size >= boff) {
1663 bp->nb_validend = np->n_size - boff;
1664 } else {
1665 bp->nb_validend = 0;
1666 }
1667 }
1668 if ((nmp = NFSTONMP(np)) && (nmp->nm_vers == NFS_VER2) &&
1669 ((NBOFF(bp) + bp->nb_validend) > 0x100000000LL)) {
1670 bp->nb_validend = 0x100000000LL - NBOFF(bp);
1671 }
1672 nfs_buf_pgs_get_page_mask(&bp->nb_valid, round_page_64(bp->nb_validend) / PAGE_SIZE);
1673 if (bp->nb_validend & PAGE_MASK) {
1674 /* zero-fill remainder of last page */
1675 bzero(bp->nb_data + bp->nb_validend, PAGE_SIZE - (bp->nb_validend & PAGE_MASK));
1676 }
1677 }
1678 nfs_buf_iodone(bp);
1679 }
1680
1681 /*
1682 * initiate the NFS READ RPC(s) for a buffer
1683 */
1684 int
nfs_buf_read_rpc(struct nfsbuf * bp,thread_t thd,kauth_cred_t cred)1685 nfs_buf_read_rpc(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
1686 {
1687 struct nfsmount *nmp;
1688 nfsnode_t np = bp->nb_np;
1689 int error = 0, nfsvers, async;
1690 int offset;
1691 uint64_t length, nrpcs;
1692 uint32_t nmrsize;
1693 size_t len;
1694 off_t boff;
1695 struct nfsreq *req;
1696 struct nfsreq_cbinfo cb;
1697
1698 nmp = NFSTONMP(np);
1699 if (nfs_mount_gone(nmp)) {
1700 bp->nb_error = error = ENXIO;
1701 SET(bp->nb_flags, NB_ERROR);
1702 nfs_buf_iodone(bp);
1703 return error;
1704 }
1705 nfsvers = nmp->nm_vers;
1706 nmrsize = nmp->nm_rsize;
1707
1708 boff = NBOFF(bp);
1709 offset = 0;
1710 length = bp->nb_bufsize;
1711
1712 if (nfsvers == NFS_VER2) {
1713 if (boff > 0xffffffffLL) {
1714 bp->nb_error = error = EFBIG;
1715 SET(bp->nb_flags, NB_ERROR);
1716 nfs_buf_iodone(bp);
1717 return error;
1718 }
1719 if ((boff + length - 1) > 0xffffffffLL) {
1720 length = 0x100000000LL - boff;
1721 }
1722 }
1723
1724 /* Note: Can only do async I/O if nfsiods are configured. */
1725 async = (bp->nb_flags & NB_ASYNC);
1726 cb.rcb_func = async ? nfs_buf_read_rpc_finish : NULL;
1727 cb.rcb_bp = bp;
1728
1729 bp->nb_offio = bp->nb_endio = 0;
1730 bp->nb_rpcs = nrpcs = (length + nmrsize - 1) / nmrsize;
1731 if (async && (nrpcs > 1)) {
1732 SET(bp->nb_flags, NB_MULTASYNCRPC);
1733 } else {
1734 CLR(bp->nb_flags, NB_MULTASYNCRPC);
1735 }
1736
1737 while (length > 0) {
1738 if (ISSET(bp->nb_flags, NB_ERROR)) {
1739 error = bp->nb_error;
1740 break;
1741 }
1742 len = (length > nmrsize) ? nmrsize : (uint32_t)length;
1743 cb.rcb_args.offset = offset;
1744 cb.rcb_args.length = len;
1745 #if CONFIG_NFS4
1746 if (nmp->nm_vers >= NFS_VER4) {
1747 cb.rcb_args.stategenid = nmp->nm_stategenid;
1748 }
1749 #endif
1750 req = NULL;
1751 error = nmp->nm_funcs->nf_read_rpc_async(np, boff + offset, len, thd, cred, &cb, &req);
1752 if (error) {
1753 break;
1754 }
1755 offset += len;
1756 length -= len;
1757 if (async) {
1758 continue;
1759 }
1760 nfs_buf_read_rpc_finish(req);
1761 if (ISSET(bp->nb_flags, NB_ERROR)) {
1762 error = bp->nb_error;
1763 break;
1764 }
1765 }
1766
1767 if (length > 0) {
1768 /*
1769 * Something bad happened while trying to send the RPC(s).
1770 * Wait for any outstanding requests to complete.
1771 */
1772 bp->nb_error = error;
1773 SET(bp->nb_flags, NB_ERROR);
1774 if (ISSET(bp->nb_flags, NB_MULTASYNCRPC)) {
1775 nrpcs = (length + nmrsize - 1) / nmrsize;
1776 lck_mtx_lock(&nfs_buf_mutex);
1777 bp->nb_rpcs -= nrpcs;
1778 if (bp->nb_rpcs == 0) {
1779 /* No RPCs left, so the buffer's done */
1780 lck_mtx_unlock(&nfs_buf_mutex);
1781 nfs_buf_iodone(bp);
1782 } else {
1783 /* wait for the last RPC to mark it done */
1784 while (bp->nb_rpcs > 0) {
1785 msleep(&bp->nb_rpcs, &nfs_buf_mutex, 0,
1786 "nfs_buf_read_rpc_cancel", NULL);
1787 }
1788 lck_mtx_unlock(&nfs_buf_mutex);
1789 }
1790 } else {
1791 nfs_buf_iodone(bp);
1792 }
1793 }
1794
1795 return error;
1796 }
1797
1798 /*
1799 * finish up an NFS READ RPC on a buffer
1800 */
1801 void
nfs_buf_read_rpc_finish(struct nfsreq * req)1802 nfs_buf_read_rpc_finish(struct nfsreq *req)
1803 {
1804 struct nfsmount *nmp;
1805 size_t rlen, length;
1806 struct nfsreq_cbinfo cb;
1807 struct nfsbuf *bp;
1808 int error = 0, nfsvers, eof = 0, multasyncrpc, finished;
1809 off_t offset;
1810 void *wakeme = NULL;
1811 struct nfsreq *rreq = NULL;
1812 nfsnode_t np;
1813 thread_t thd;
1814 kauth_cred_t cred;
1815 uio_t auio;
1816
1817 finish:
1818 np = req->r_np;
1819 thd = req->r_thread;
1820 cred = req->r_cred;
1821 if (IS_VALID_CRED(cred)) {
1822 kauth_cred_ref(cred);
1823 }
1824 cb = req->r_callback;
1825 bp = cb.rcb_bp;
1826 if (cb.rcb_func) { /* take an extra reference on the nfsreq in case we want to resend it later due to grace error */
1827 nfs_request_ref(req, 0);
1828 }
1829
1830 nmp = NFSTONMP(np);
1831 if (nfs_mount_gone(nmp)) {
1832 SET(bp->nb_flags, NB_ERROR);
1833 bp->nb_error = error = ENXIO;
1834 }
1835 if (error || ISSET(bp->nb_flags, NB_ERROR)) {
1836 /* just drop it */
1837 nfs_request_async_cancel(req);
1838 goto out;
1839 }
1840
1841 nfsvers = nmp->nm_vers;
1842 offset = cb.rcb_args.offset;
1843 rlen = length = cb.rcb_args.length;
1844
1845 auio = uio_create(1, NBOFF(bp) + offset, UIO_SYSSPACE, UIO_READ);
1846 uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
1847
1848 /* finish the RPC */
1849 error = nmp->nm_funcs->nf_read_rpc_async_finish(np, req, auio, &rlen, &eof);
1850
1851 /* Free allocated uio buffer */
1852 uio_free(auio);
1853
1854 if ((error == EINPROGRESS) && cb.rcb_func) {
1855 /* async request restarted */
1856 if (cb.rcb_func) {
1857 nfs_request_rele(req);
1858 }
1859 if (IS_VALID_CRED(cred)) {
1860 kauth_cred_unref(&cred);
1861 }
1862 return;
1863 }
1864 #if CONFIG_NFS4
1865 if ((nmp->nm_vers >= NFS_VER4) && nfs_mount_state_error_should_restart(error) && !ISSET(bp->nb_flags, NB_ERROR)) {
1866 lck_mtx_lock(&nmp->nm_lock);
1867 if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args.stategenid == nmp->nm_stategenid)) {
1868 NP(np, "nfs_buf_read_rpc_finish: error %d @ 0x%llx, 0x%x 0x%x, initiating recovery",
1869 error, NBOFF(bp) + offset, cb.rcb_args.stategenid, nmp->nm_stategenid);
1870 nfs_need_recover(nmp, error);
1871 }
1872 lck_mtx_unlock(&nmp->nm_lock);
1873 if (np->n_flag & NREVOKE) {
1874 error = EIO;
1875 } else {
1876 if (error == NFSERR_GRACE) {
1877 if (cb.rcb_func) {
1878 /*
1879 * For an async I/O request, handle a grace delay just like
1880 * jukebox errors. Set the resend time and queue it up.
1881 */
1882 struct timeval now;
1883 if (req->r_nmrep.nmc_mhead) {
1884 mbuf_freem(req->r_nmrep.nmc_mhead);
1885 req->r_nmrep.nmc_mhead = NULL;
1886 }
1887 req->r_error = 0;
1888 microuptime(&now);
1889 lck_mtx_lock(&req->r_mtx);
1890 req->r_resendtime = now.tv_sec + 2;
1891 req->r_xid = 0; // get a new XID
1892 req->r_flags |= R_RESTART;
1893 req->r_start = 0;
1894 nfs_asyncio_resend(req);
1895 lck_mtx_unlock(&req->r_mtx);
1896 if (IS_VALID_CRED(cred)) {
1897 kauth_cred_unref(&cred);
1898 }
1899 /* Note: nfsreq reference taken will be dropped later when finished */
1900 return;
1901 }
1902 /* otherwise, just pause a couple seconds and retry */
1903 tsleep(&nmp->nm_state, (PZERO - 1), "nfsgrace", 2 * hz);
1904 }
1905 if (!(error = nfs_mount_state_wait_for_recovery(nmp))) {
1906 rlen = 0;
1907 goto readagain;
1908 }
1909 }
1910 }
1911 #endif
1912 if (error) {
1913 SET(bp->nb_flags, NB_ERROR);
1914 bp->nb_error = error;
1915 goto out;
1916 }
1917
1918 if ((rlen > 0) && (bp->nb_endio < (offset + (int)rlen))) {
1919 bp->nb_endio = offset + rlen;
1920 }
1921
1922 if ((nfsvers == NFS_VER2) || eof || (rlen == 0)) {
1923 /* zero out the remaining data (up to EOF) */
1924 off_t rpcrem, eofrem, rem;
1925 rpcrem = (length - rlen);
1926 eofrem = np->n_size - (NBOFF(bp) + offset + rlen);
1927 rem = (rpcrem < eofrem) ? rpcrem : eofrem;
1928 if (rem > 0) {
1929 NFS_BZERO(bp->nb_data + offset + rlen, rem);
1930 }
1931 } else if ((rlen < length) && !ISSET(bp->nb_flags, NB_ERROR)) {
1932 /*
1933 * short read
1934 *
1935 * We haven't hit EOF and we didn't get all the data
1936 * requested, so we need to issue another read for the rest.
1937 * (Don't bother if the buffer already hit an error.)
1938 */
1939 #if CONFIG_NFS4
1940 readagain:
1941 #endif
1942 offset += rlen;
1943 length -= rlen;
1944 cb.rcb_args.offset = offset;
1945 cb.rcb_args.length = length;
1946 #if CONFIG_NFS4
1947 if (nmp->nm_vers >= NFS_VER4) {
1948 cb.rcb_args.stategenid = nmp->nm_stategenid;
1949 }
1950 #endif
1951 error = nmp->nm_funcs->nf_read_rpc_async(np, NBOFF(bp) + offset, length, thd, cred, &cb, &rreq);
1952 if (!error) {
1953 if (IS_VALID_CRED(cred)) {
1954 kauth_cred_unref(&cred);
1955 }
1956 if (!cb.rcb_func) {
1957 /* if !async we'll need to wait for this RPC to finish */
1958 req = rreq;
1959 rreq = NULL;
1960 goto finish;
1961 }
1962 nfs_request_rele(req);
1963 /*
1964 * We're done here.
1965 * Outstanding RPC count is unchanged.
1966 * Callback will be called when RPC is done.
1967 */
1968 return;
1969 }
1970 SET(bp->nb_flags, NB_ERROR);
1971 bp->nb_error = error;
1972 }
1973
1974 out:
1975 if (cb.rcb_func) {
1976 nfs_request_rele(req);
1977 }
1978 if (IS_VALID_CRED(cred)) {
1979 kauth_cred_unref(&cred);
1980 }
1981
1982 /*
1983 * Decrement outstanding RPC count on buffer
1984 * and call nfs_buf_read_finish on last RPC.
1985 *
1986 * (Note: when there are multiple async RPCs issued for a
1987 * buffer we need nfs_buffer_mutex to avoid problems when
1988 * aborting a partially-initiated set of RPCs)
1989 */
1990
1991 multasyncrpc = ISSET(bp->nb_flags, NB_MULTASYNCRPC);
1992 if (multasyncrpc) {
1993 lck_mtx_lock(&nfs_buf_mutex);
1994 }
1995
1996 bp->nb_rpcs--;
1997 finished = (bp->nb_rpcs == 0);
1998
1999 if (multasyncrpc) {
2000 lck_mtx_unlock(&nfs_buf_mutex);
2001 }
2002
2003 if (finished) {
2004 if (multasyncrpc) {
2005 wakeme = &bp->nb_rpcs;
2006 }
2007 nfs_buf_read_finish(bp);
2008 if (wakeme) {
2009 wakeup(wakeme);
2010 }
2011 }
2012 }
2013
2014 /*
2015 * Do buffer readahead.
2016 * Initiate async I/O to read buffers not in cache.
2017 */
2018 int
nfs_buf_readahead(nfsnode_t np,int ioflag,daddr64_t * rabnp,daddr64_t lastrabn,thread_t thd,kauth_cred_t cred)2019 nfs_buf_readahead(nfsnode_t np, int ioflag, daddr64_t *rabnp, daddr64_t lastrabn, thread_t thd, kauth_cred_t cred)
2020 {
2021 struct nfsmount *nmp = NFSTONMP(np);
2022 struct nfsbuf *bp;
2023 int error = 0;
2024 uint32_t nra;
2025
2026 if (nfs_mount_gone(nmp)) {
2027 return ENXIO;
2028 }
2029 if (nmp->nm_readahead <= 0) {
2030 return 0;
2031 }
2032 if (*rabnp > lastrabn) {
2033 return 0;
2034 }
2035
2036 for (nra = 0; (nra < nmp->nm_readahead) && (*rabnp <= lastrabn); nra++, *rabnp = *rabnp + 1) {
2037 /* check if block exists and is valid. */
2038 if ((*rabnp * nmp->nm_biosize) >= (off_t)np->n_size) {
2039 /* stop reading ahead if we're beyond EOF */
2040 *rabnp = lastrabn;
2041 break;
2042 }
2043 error = nfs_buf_get(np, *rabnp, nmp->nm_biosize, thd, NBLK_READ | NBLK_NOWAIT, &bp);
2044 if (error) {
2045 break;
2046 }
2047 nfs_node_lock_force(np);
2048 np->n_lastrahead = *rabnp;
2049 nfs_node_unlock(np);
2050 if (!bp) {
2051 continue;
2052 }
2053 if ((ioflag & IO_NOCACHE) && ISSET(bp->nb_flags, NB_CACHE) &&
2054 !nfs_buf_pgs_is_set(&bp->nb_dirty) && !ISSET(bp->nb_flags, (NB_DELWRI | NB_NCRDAHEAD))) {
2055 CLR(bp->nb_flags, NB_CACHE);
2056 NBPGS_ERASE(&bp->nb_valid);
2057 bp->nb_validoff = bp->nb_validend = -1;
2058 }
2059 if ((bp->nb_dirtyend <= 0) && !nfs_buf_pgs_is_set(&bp->nb_dirty) &&
2060 !ISSET(bp->nb_flags, (NB_CACHE | NB_DELWRI))) {
2061 SET(bp->nb_flags, (NB_READ | NB_ASYNC));
2062 if (ioflag & IO_NOCACHE) {
2063 SET(bp->nb_flags, NB_NCRDAHEAD);
2064 }
2065 if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
2066 kauth_cred_ref(cred);
2067 bp->nb_rcred = cred;
2068 }
2069 if ((error = nfs_buf_read(bp))) {
2070 break;
2071 }
2072 continue;
2073 }
2074 nfs_buf_release(bp, 1);
2075 }
2076 return error;
2077 }
2078
2079 /*
2080 * NFS buffer I/O for reading files.
2081 */
2082 int
nfs_bioread(nfsnode_t np,uio_t uio,int ioflag,vfs_context_t ctx)2083 nfs_bioread(nfsnode_t np, uio_t uio, int ioflag, vfs_context_t ctx)
2084 {
2085 vnode_t vp = NFSTOV(np);
2086 struct nfsbuf *bp = NULL;
2087 struct nfsmount *nmp = VTONMP(vp);
2088 daddr64_t lbn, rabn = 0, lastrabn, maxrabn = -1;
2089 off_t diff, on = 0, n = 0;
2090 int error = 0, n32;
2091 int nfsvers, biosize, modified, readaheads = 0;
2092 thread_t thd;
2093 kauth_cred_t cred;
2094 int64_t io_resid;
2095
2096 FSDBG_TOP(514, np, uio_offset(uio), uio_resid(uio), ioflag);
2097
2098 nfsvers = nmp->nm_vers;
2099 biosize = nmp->nm_biosize;
2100 thd = vfs_context_thread(ctx);
2101 cred = vfs_context_ucred(ctx);
2102
2103 if (vnode_vtype(vp) != VREG) {
2104 printf("nfs_bioread: type %x unexpected\n", vnode_vtype(vp));
2105 FSDBG_BOT(514, np, 0xd1e0016, 0, EINVAL);
2106 return EINVAL;
2107 }
2108
2109 /*
2110 * For NFS, cache consistency can only be maintained approximately.
2111 * Although RFC1094 does not specify the criteria, the following is
2112 * believed to be compatible with the reference port.
2113 *
2114 * If the file has changed since the last read RPC or you have
2115 * written to the file, you may have lost data cache consistency
2116 * with the server. So, check for a change, and flush all of the
2117 * file's data out of the cache.
2118 * NB: This implies that cache data can be read when up to
2119 * NFS_MAXATTRTIMO seconds out of date. If you find that you
2120 * need current attributes, nfs_getattr() can be forced to fetch
2121 * new attributes (via NATTRINVALIDATE() or NGA_UNCACHED).
2122 */
2123
2124 if (ISSET(np->n_flag, NUPDATESIZE)) {
2125 nfs_data_update_size(np, 0);
2126 }
2127
2128 if ((error = nfs_node_lock(np))) {
2129 FSDBG_BOT(514, np, 0xd1e0222, 0, error);
2130 return error;
2131 }
2132
2133 if (np->n_flag & NNEEDINVALIDATE) {
2134 np->n_flag &= ~NNEEDINVALIDATE;
2135 nfs_node_unlock(np);
2136 error = nfs_vinvalbuf1(vp, V_SAVE | V_IGNORE_WRITEERR, ctx, 1);
2137 if (!error) {
2138 error = nfs_node_lock(np);
2139 }
2140 if (error) {
2141 FSDBG_BOT(514, np, 0xd1e0322, 0, error);
2142 return error;
2143 }
2144 }
2145
2146 modified = (np->n_flag & NMODIFIED);
2147 nfs_node_unlock(np);
2148 /* nfs_getattr() will check changed and purge caches */
2149 error = nfs_getattr(np, NULL, ctx, modified ? NGA_UNCACHED : NGA_CACHED);
2150 if (error) {
2151 FSDBG_BOT(514, np, 0xd1e0004, 0, error);
2152 return error;
2153 }
2154
2155 if (uio_resid(uio) == 0) {
2156 FSDBG_BOT(514, np, 0xd1e0001, 0, 0);
2157 return 0;
2158 }
2159 if (uio_offset(uio) < 0) {
2160 FSDBG_BOT(514, np, 0xd1e0002, 0, EINVAL);
2161 return EINVAL;
2162 }
2163
2164 /*
2165 * set up readahead - which may be limited by:
2166 * + current request length (for IO_NOCACHE)
2167 * + readahead setting
2168 * + file size
2169 */
2170 if (nmp->nm_readahead > 0) {
2171 off_t end = uio_offset(uio) + uio_resid(uio);
2172 if (end > (off_t)np->n_size) {
2173 end = np->n_size;
2174 }
2175 rabn = uio_offset(uio) / biosize;
2176 maxrabn = (end - 1) / biosize;
2177 nfs_node_lock_force(np);
2178 if (!(ioflag & IO_NOCACHE) &&
2179 (!rabn || (rabn == np->n_lastread) || (rabn == (np->n_lastread + 1)))) {
2180 maxrabn += nmp->nm_readahead;
2181 if ((maxrabn * biosize) >= (off_t)np->n_size) {
2182 maxrabn = ((off_t)np->n_size - 1) / biosize;
2183 }
2184 }
2185 if (maxrabn < np->n_lastrahead) {
2186 np->n_lastrahead = -1;
2187 }
2188 if (rabn < np->n_lastrahead) {
2189 rabn = np->n_lastrahead + 1;
2190 }
2191 nfs_node_unlock(np);
2192 } else {
2193 rabn = maxrabn = 0;
2194 }
2195
2196 do {
2197 nfs_data_lock(np, NFS_DATA_LOCK_SHARED);
2198 lbn = uio_offset(uio) / biosize;
2199
2200 /*
2201 * Copy directly from any cached pages without grabbing the bufs.
2202 * (If we are NOCACHE and we've issued readahead requests, we need
2203 * to grab the NB_NCRDAHEAD bufs to drop them.)
2204 */
2205 if ((!(ioflag & IO_NOCACHE) || !readaheads) && uio_isuserspace(uio)) {
2206 io_resid = uio_resid(uio);
2207 diff = np->n_size - uio_offset(uio);
2208 if (diff < io_resid) {
2209 io_resid = diff;
2210 }
2211 if (io_resid > 0) {
2212 int count = (io_resid > INT_MAX) ? INT_MAX : (int)io_resid;
2213 error = cluster_copy_ubc_data(vp, uio, &count, 0);
2214 if (error) {
2215 nfs_data_unlock(np);
2216 FSDBG_BOT(514, np, uio_offset(uio), 0xcacefeed, error);
2217 return error;
2218 }
2219 }
2220 /* count any biocache reads that we just copied directly */
2221 if (lbn != (uio_offset(uio) / biosize)) {
2222 OSAddAtomic64(NFS_ROUND_BLOCK(uio_offset(uio), biosize) - lbn, &nfsclntstats.biocache_reads);
2223 FSDBG(514, np, 0xcacefeed, uio_offset(uio), error);
2224 }
2225 }
2226
2227 lbn = uio_offset(uio) / biosize;
2228 on = uio_offset(uio) % biosize;
2229 nfs_node_lock_force(np);
2230 np->n_lastread = (uio_offset(uio) - 1) / biosize;
2231 nfs_node_unlock(np);
2232
2233 if ((uio_resid(uio) <= 0) || (uio_offset(uio) >= (off_t)np->n_size)) {
2234 nfs_data_unlock(np);
2235 FSDBG_BOT(514, np, uio_offset(uio), uio_resid(uio), 0xaaaaaaaa);
2236 return 0;
2237 }
2238
2239 /* adjust readahead block number, if necessary */
2240 if (rabn < lbn) {
2241 rabn = lbn;
2242 }
2243 lastrabn = MIN(maxrabn, lbn + nmp->nm_readahead);
2244 if (rabn <= lastrabn) { /* start readaheads */
2245 error = nfs_buf_readahead(np, ioflag, &rabn, lastrabn, thd, cred);
2246 if (error) {
2247 nfs_data_unlock(np);
2248 FSDBG_BOT(514, np, 0xd1e000b, 1, error);
2249 return error;
2250 }
2251 readaheads = 1;
2252 OSAddAtomic64(rabn - lbn, &nfsclntstats.biocache_reads);
2253 } else {
2254 OSAddAtomic64(1, &nfsclntstats.biocache_reads);
2255 }
2256
2257 /*
2258 * If the block is in the cache and has the required data
2259 * in a valid region, just copy it out.
2260 * Otherwise, get the block and write back/read in,
2261 * as required.
2262 */
2263 again:
2264 io_resid = uio_resid(uio);
2265 n = (io_resid > (biosize - on)) ? (biosize - on) : io_resid;
2266 diff = np->n_size - uio_offset(uio);
2267 if (diff < n) {
2268 n = diff;
2269 }
2270
2271 error = nfs_buf_get(np, lbn, biosize, thd, NBLK_READ, &bp);
2272 if (error) {
2273 nfs_data_unlock(np);
2274 FSDBG_BOT(514, np, 0xd1e000c, 0, error);
2275 return error;
2276 }
2277
2278 if ((ioflag & IO_NOCACHE) && ISSET(bp->nb_flags, NB_CACHE)) {
2279 /*
2280 * IO_NOCACHE found a cached buffer.
2281 * Flush the buffer if it's dirty.
2282 * Invalidate the data if it wasn't just read
2283 * in as part of a "nocache readahead".
2284 */
2285 if (nfs_buf_pgs_is_set(&bp->nb_dirty) || (bp->nb_dirtyend > 0)) {
2286 /* so write the buffer out and try again */
2287 SET(bp->nb_flags, NB_NOCACHE);
2288 goto flushbuffer;
2289 }
2290 if (ISSET(bp->nb_flags, NB_NCRDAHEAD)) {
2291 CLR(bp->nb_flags, NB_NCRDAHEAD);
2292 SET(bp->nb_flags, NB_NOCACHE);
2293 }
2294 }
2295
2296 /* if any pages are valid... */
2297 if (nfs_buf_pgs_is_set(&bp->nb_valid)) {
2298 /* ...check for any invalid pages in the read range */
2299 off_t pg, firstpg, lastpg, dirtypg;
2300 dirtypg = firstpg = lastpg = -1;
2301 pg = on / PAGE_SIZE;
2302 while (pg <= (on + n - 1) / PAGE_SIZE) {
2303 if (!NBPGVALID(bp, pg)) {
2304 if (firstpg < 0) {
2305 firstpg = pg;
2306 }
2307 lastpg = pg;
2308 } else if (firstpg >= 0 && dirtypg < 0 && NBPGDIRTY(bp, pg)) {
2309 dirtypg = pg;
2310 }
2311 pg++;
2312 }
2313
2314 /* if there are no invalid pages, we're all set */
2315 if (firstpg < 0) {
2316 if (bp->nb_validoff < 0) {
2317 /* valid range isn't set up, so */
2318 /* set it to what we know is valid */
2319 bp->nb_validoff = trunc_page_64(on);
2320 bp->nb_validend = round_page_64(on + n);
2321 nfs_buf_normalize_valid_range(np, bp);
2322 }
2323 goto buffer_ready;
2324 }
2325
2326 /* there are invalid pages in the read range */
2327 if (((dirtypg > firstpg) && (dirtypg < lastpg)) ||
2328 (((firstpg * PAGE_SIZE) < bp->nb_dirtyend) && (((lastpg + 1) * PAGE_SIZE) > bp->nb_dirtyoff))) {
2329 /* there are also dirty page(s) (or range) in the read range, */
2330 /* so write the buffer out and try again */
2331 flushbuffer:
2332 CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
2333 SET(bp->nb_flags, NB_ASYNC);
2334 if (!IS_VALID_CRED(bp->nb_wcred)) {
2335 kauth_cred_ref(cred);
2336 bp->nb_wcred = cred;
2337 }
2338 error = nfs_buf_write(bp);
2339 if (error) {
2340 nfs_data_unlock(np);
2341 FSDBG_BOT(514, np, 0xd1e000d, 0, error);
2342 return error;
2343 }
2344 goto again;
2345 }
2346 if (!nfs_buf_pgs_is_set(&bp->nb_dirty) && bp->nb_dirtyend <= 0 &&
2347 (lastpg - firstpg + 1) > (biosize / PAGE_SIZE) / 2) {
2348 /* we need to read in more than half the buffer and the */
2349 /* buffer's not dirty, so just fetch the whole buffer */
2350 NBPGS_ERASE(&bp->nb_valid);
2351 } else {
2352 /* read the page range in */
2353 uio_t auio;
2354
2355 NFS_BUF_MAP(bp);
2356 auio = uio_create(1, (NBOFF(bp) + firstpg * PAGE_SIZE_64), UIO_SYSSPACE, UIO_READ);
2357 if (!auio) {
2358 error = ENOMEM;
2359 } else {
2360 NFS_UIO_ADDIOV(auio, CAST_USER_ADDR_T(bp->nb_data + (firstpg * PAGE_SIZE)),
2361 ((lastpg - firstpg + 1) * PAGE_SIZE));
2362 error = nfs_read_rpc(np, auio, ctx);
2363 }
2364 if (error) {
2365 if (ioflag & IO_NOCACHE) {
2366 SET(bp->nb_flags, NB_NOCACHE);
2367 }
2368 /* Free allocated uio buffer */
2369 uio_free(auio);
2370 nfs_buf_release(bp, 1);
2371 nfs_data_unlock(np);
2372 FSDBG_BOT(514, np, 0xd1e000e, 0, error);
2373 return error;
2374 }
2375 /* Make sure that the valid range is set to cover this read. */
2376 bp->nb_validoff = trunc_page_64(on);
2377 bp->nb_validend = round_page_64(on + n);
2378 nfs_buf_normalize_valid_range(np, bp);
2379 if (uio_resid(auio) > 0) {
2380 /* if short read, must have hit EOF, */
2381 /* so zero the rest of the range */
2382 bzero(CAST_DOWN(caddr_t, uio_curriovbase(auio)), uio_resid(auio));
2383 }
2384 /* mark the pages (successfully read) as valid */
2385 for (pg = firstpg; pg <= lastpg; pg++) {
2386 NBPGVALID_SET(bp, pg);
2387 }
2388
2389 /* Free allocated uio buffer */
2390 uio_free(auio);
2391 }
2392 }
2393 /* if no pages are valid, read the whole block */
2394 if (!nfs_buf_pgs_is_set(&bp->nb_valid)) {
2395 if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
2396 kauth_cred_ref(cred);
2397 bp->nb_rcred = cred;
2398 }
2399 SET(bp->nb_flags, NB_READ);
2400 CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
2401 error = nfs_buf_read(bp);
2402 if (ioflag & IO_NOCACHE) {
2403 SET(bp->nb_flags, NB_NOCACHE);
2404 }
2405 if (error) {
2406 nfs_data_unlock(np);
2407 nfs_buf_release(bp, 1);
2408 FSDBG_BOT(514, np, 0xd1e000f, 0, error);
2409 return error;
2410 }
2411 }
2412 buffer_ready:
2413 /* validate read range against valid range and clip */
2414 if (bp->nb_validend > 0) {
2415 diff = (on >= bp->nb_validend) ? 0 : (bp->nb_validend - on);
2416 if (diff < n) {
2417 n = diff;
2418 }
2419 }
2420 if (n > 0) {
2421 NFS_BUF_MAP(bp);
2422 n32 = n > INT_MAX ? INT_MAX : (int)n;
2423 error = uiomove(bp->nb_data + on, n32, uio);
2424 if (!error && n > n32) {
2425 error = uiomove(bp->nb_data + on + n32, (int)(n - n32), uio);
2426 }
2427 }
2428
2429 nfs_buf_release(bp, 1);
2430 nfs_data_unlock(np);
2431 nfs_node_lock_force(np);
2432 np->n_lastread = (uio_offset(uio) - 1) / biosize;
2433 nfs_node_unlock(np);
2434 } while (error == 0 && uio_resid(uio) > 0 && n > 0);
2435 FSDBG_BOT(514, np, uio_offset(uio), uio_resid(uio), error);
2436 return error;
2437 }
2438
2439 /*
2440 * limit the number of outstanding async I/O writes
2441 */
2442 int
nfs_async_write_start(struct nfsmount * nmp)2443 nfs_async_write_start(struct nfsmount *nmp)
2444 {
2445 int error = 0, slpflag = NMFLAG(nmp, INTR) ? PCATCH : 0;
2446 struct timespec ts = { .tv_sec = 1, .tv_nsec = 0 };
2447
2448 if (nfs_max_async_writes <= 0) {
2449 return 0;
2450 }
2451 lck_mtx_lock(&nmp->nm_lock);
2452 while ((nfs_max_async_writes > 0) && (nmp->nm_asyncwrites >= nfs_max_async_writes)) {
2453 if ((error = nfs_sigintr(nmp, NULL, current_thread(), 1))) {
2454 break;
2455 }
2456 msleep(&nmp->nm_asyncwrites, &nmp->nm_lock, slpflag | (PZERO - 1), "nfsasyncwrites", &ts);
2457 slpflag = 0;
2458 }
2459 if (!error) {
2460 nmp->nm_asyncwrites++;
2461 }
2462 lck_mtx_unlock(&nmp->nm_lock);
2463 return error;
2464 }
2465 void
nfs_async_write_done(struct nfsmount * nmp)2466 nfs_async_write_done(struct nfsmount *nmp)
2467 {
2468 if (nmp->nm_asyncwrites <= 0) {
2469 return;
2470 }
2471 lck_mtx_lock(&nmp->nm_lock);
2472 if (nmp->nm_asyncwrites-- >= nfs_max_async_writes) {
2473 wakeup(&nmp->nm_asyncwrites);
2474 }
2475 lck_mtx_unlock(&nmp->nm_lock);
2476 }
2477
2478 /*
2479 * write (or commit) the given NFS buffer
2480 *
2481 * Commit the buffer if we can.
2482 * Write out any dirty range.
2483 * If any dirty pages remain, write them out.
2484 * Mark buffer done.
2485 *
2486 * For async requests, all the work beyond sending the initial
2487 * write RPC is handled in the RPC callback(s).
2488 */
2489 int
nfs_buf_write(struct nfsbuf * bp)2490 nfs_buf_write(struct nfsbuf *bp)
2491 {
2492 int error = 0, oldflags, async;
2493 nfsnode_t np;
2494 thread_t thd;
2495 kauth_cred_t cred;
2496 proc_t p = current_proc();
2497 int iomode;
2498 off_t doff, dend, firstpg, lastpg;
2499
2500 FSDBG_TOP(553, bp, NBOFF(bp), bp->nb_flags, 0);
2501
2502 if (!ISSET(bp->nb_lflags, NBL_BUSY)) {
2503 panic("nfs_buf_write: buffer is not busy???");
2504 }
2505
2506 np = bp->nb_np;
2507 async = ISSET(bp->nb_flags, NB_ASYNC);
2508 oldflags = bp->nb_flags;
2509
2510 CLR(bp->nb_flags, (NB_READ | NB_DONE | NB_ERROR | NB_DELWRI));
2511 if (ISSET(oldflags, NB_DELWRI)) {
2512 lck_mtx_lock(&nfs_buf_mutex);
2513 nfs_nbdwrite--;
2514 NFSBUFCNTCHK();
2515 lck_mtx_unlock(&nfs_buf_mutex);
2516 wakeup(&nfs_nbdwrite);
2517 }
2518
2519 /* move to clean list */
2520 if (ISSET(oldflags, (NB_ASYNC | NB_DELWRI))) {
2521 lck_mtx_lock(&nfs_buf_mutex);
2522 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
2523 LIST_REMOVE(bp, nb_vnbufs);
2524 }
2525 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
2526 lck_mtx_unlock(&nfs_buf_mutex);
2527 }
2528 nfs_node_lock_force(np);
2529 np->n_numoutput++;
2530 nfs_node_unlock(np);
2531 vnode_startwrite(NFSTOV(np));
2532
2533 if (p && p->p_stats) {
2534 OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock);
2535 }
2536
2537 cred = bp->nb_wcred;
2538 if (!IS_VALID_CRED(cred) && ISSET(bp->nb_flags, NB_READ)) {
2539 cred = bp->nb_rcred; /* shouldn't really happen, but... */
2540 }
2541 if (IS_VALID_CRED(cred)) {
2542 kauth_cred_ref(cred);
2543 }
2544 thd = async ? NULL : current_thread();
2545
2546 /* We need to make sure the pages are locked before doing I/O. */
2547 if (!ISSET(bp->nb_flags, NB_META)) {
2548 if (UBCINFOEXISTS(NFSTOV(np))) {
2549 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
2550 error = nfs_buf_upl_setup(bp);
2551 if (error) {
2552 printf("nfs_buf_write: upl create failed %d\n", error);
2553 SET(bp->nb_flags, NB_ERROR);
2554 bp->nb_error = error = EIO;
2555 nfs_buf_iodone(bp);
2556 goto out;
2557 }
2558 nfs_buf_upl_check(bp);
2559 }
2560 } else {
2561 /* We should never be in nfs_buf_write() with no UBCINFO. */
2562 printf("nfs_buf_write: ubcinfo already gone\n");
2563 SET(bp->nb_flags, NB_ERROR);
2564 bp->nb_error = error = EIO;
2565 nfs_buf_iodone(bp);
2566 goto out;
2567 }
2568 }
2569
2570 /* If NB_NEEDCOMMIT is set, a commit RPC may do the trick. */
2571 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
2572 nfs_buf_check_write_verifier(np, bp);
2573 }
2574 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
2575 struct nfsmount *nmp = NFSTONMP(np);
2576 if (nfs_mount_gone(nmp)) {
2577 SET(bp->nb_flags, NB_ERROR);
2578 bp->nb_error = error = EIO;
2579 nfs_buf_iodone(bp);
2580 goto out;
2581 }
2582 SET(bp->nb_flags, NB_WRITEINPROG);
2583 error = nmp->nm_funcs->nf_commit_rpc(np, NBOFF(bp) + bp->nb_dirtyoff,
2584 bp->nb_dirtyend - bp->nb_dirtyoff, bp->nb_wcred, bp->nb_verf);
2585 CLR(bp->nb_flags, NB_WRITEINPROG);
2586 if (error) {
2587 if (error != NFSERR_STALEWRITEVERF) {
2588 SET(bp->nb_flags, NB_ERROR);
2589 bp->nb_error = error;
2590 }
2591 nfs_buf_iodone(bp);
2592 goto out;
2593 }
2594 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
2595 CLR(bp->nb_flags, NB_NEEDCOMMIT);
2596 nfs_node_lock_force(np);
2597 np->n_needcommitcnt--;
2598 CHECK_NEEDCOMMITCNT(np);
2599 nfs_node_unlock(np);
2600 }
2601 if (!error && (bp->nb_dirtyend > 0)) {
2602 /* sanity check the dirty range */
2603 if (NBOFF(bp) + bp->nb_dirtyend > (off_t) np->n_size) {
2604 bp->nb_dirtyend = np->n_size - NBOFF(bp);
2605 if (bp->nb_dirtyoff >= bp->nb_dirtyend) {
2606 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
2607 }
2608 }
2609 }
2610 if (!error && (bp->nb_dirtyend > 0)) {
2611 /* there's a dirty range that needs to be written out */
2612 nfsbufpgs pagemask, pagemaskand;
2613 NFS_BUF_MAP(bp);
2614
2615 doff = bp->nb_dirtyoff;
2616 dend = bp->nb_dirtyend;
2617
2618 /* if doff page is dirty, move doff to start of page */
2619 if (NBPGDIRTY(bp, doff / PAGE_SIZE)) {
2620 doff -= doff & PAGE_MASK;
2621 }
2622 /* try to expand write range to include preceding dirty pages */
2623 if (!(doff & PAGE_MASK)) {
2624 while ((doff > 0) && NBPGDIRTY(bp, (doff - 1) / PAGE_SIZE)) {
2625 doff -= PAGE_SIZE;
2626 }
2627 }
2628 /* if dend page is dirty, move dend to start of next page */
2629 if ((dend & PAGE_MASK) && NBPGDIRTY(bp, dend / PAGE_SIZE)) {
2630 dend = round_page_64(dend);
2631 }
2632 /* try to expand write range to include trailing dirty pages */
2633 if (!(dend & PAGE_MASK)) {
2634 while ((dend < (int)bp->nb_bufsize) && NBPGDIRTY(bp, dend / PAGE_SIZE)) {
2635 dend += PAGE_SIZE;
2636 }
2637 }
2638 /* make sure to keep dend clipped to EOF */
2639 if ((NBOFF(bp) + dend) > (off_t) np->n_size) {
2640 dend = np->n_size - NBOFF(bp);
2641 }
2642 /* calculate range of complete pages being written */
2643 if (dend > doff) {
2644 firstpg = doff / PAGE_SIZE;
2645 lastpg = (dend - 1) / PAGE_SIZE;
2646 /* calculate mask for that page range */
2647 nfs_buf_pgs_set_pages_between(&pagemask, firstpg, lastpg + 1);
2648 } else {
2649 NBPGS_ERASE(&pagemask);
2650 }
2651
2652 /*
2653 * compare page mask to nb_dirty; if there are other dirty pages
2654 * then write FILESYNC; otherwise, write UNSTABLE if async and
2655 * not needcommit/stable; otherwise write FILESYNC
2656 */
2657 nfs_buf_pgs_bit_not(&pagemask);
2658 nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &pagemaskand);
2659 if (nfs_buf_pgs_is_set(&pagemaskand)) {
2660 iomode = NFS_WRITE_FILESYNC;
2661 } else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_STABLE)) == NB_ASYNC) {
2662 iomode = NFS_WRITE_UNSTABLE;
2663 } else {
2664 iomode = NFS_WRITE_FILESYNC;
2665 }
2666
2667 /* write the whole contiguous dirty range */
2668 bp->nb_offio = doff;
2669 bp->nb_endio = dend;
2670
2671 OSAddAtomic64(1, &nfsclntstats.write_bios);
2672
2673 SET(bp->nb_flags, NB_WRITEINPROG);
2674 error = nfs_buf_write_rpc(bp, iomode, thd, cred);
2675 /*
2676 * For async I/O, the callbacks will finish up the
2677 * write and push out any dirty pages. Otherwise,
2678 * the write has already been finished and any dirty
2679 * pages pushed out.
2680 */
2681 } else {
2682 if (!error && nfs_buf_pgs_is_set(&bp->nb_dirty)) { /* write out any dirty pages */
2683 error = nfs_buf_write_dirty_pages(bp, thd, cred);
2684 }
2685 nfs_buf_iodone(bp);
2686 }
2687 /* note: bp is still valid only for !async case */
2688 out:
2689 if (!async) {
2690 error = nfs_buf_iowait(bp);
2691 /* move to clean list */
2692 if (oldflags & NB_DELWRI) {
2693 lck_mtx_lock(&nfs_buf_mutex);
2694 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
2695 LIST_REMOVE(bp, nb_vnbufs);
2696 }
2697 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
2698 lck_mtx_unlock(&nfs_buf_mutex);
2699 }
2700 FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, error);
2701 nfs_buf_release(bp, 1);
2702 /* check if we need to invalidate (and we can) */
2703 if ((np->n_flag & NNEEDINVALIDATE) &&
2704 !(np->n_bflag & (NBINVALINPROG | NBFLUSHINPROG))) {
2705 int invalidate = 0;
2706 nfs_node_lock_force(np);
2707 if (np->n_flag & NNEEDINVALIDATE) {
2708 invalidate = 1;
2709 np->n_flag &= ~NNEEDINVALIDATE;
2710 }
2711 nfs_node_unlock(np);
2712 if (invalidate) {
2713 /*
2714 * There was a write error and we need to
2715 * invalidate attrs and flush buffers in
2716 * order to sync up with the server.
2717 * (if this write was extending the file,
2718 * we may no longer know the correct size)
2719 *
2720 * But we couldn't call vinvalbuf while holding
2721 * the buffer busy. So we call vinvalbuf() after
2722 * releasing the buffer.
2723 */
2724 nfs_vinvalbuf2(NFSTOV(np), V_SAVE | V_IGNORE_WRITEERR, thd, cred, 1);
2725 }
2726 }
2727 }
2728
2729 if (IS_VALID_CRED(cred)) {
2730 kauth_cred_unref(&cred);
2731 }
2732 return error;
2733 }
2734
2735 /*
2736 * finish the writing of a buffer
2737 */
2738 void
nfs_buf_write_finish(struct nfsbuf * bp,thread_t thd,kauth_cred_t cred)2739 nfs_buf_write_finish(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
2740 {
2741 nfsnode_t np = bp->nb_np;
2742 int error = (bp->nb_flags & NB_ERROR) ? bp->nb_error : 0;
2743 off_t firstpg, lastpg;
2744
2745 if ((error == EINTR) || (error == ERESTART)) {
2746 CLR(bp->nb_flags, NB_ERROR);
2747 SET(bp->nb_flags, NB_EINTR);
2748 }
2749
2750 if (!error) {
2751 nfsbufpgs pagemask;
2752 /* calculate range of complete pages being written */
2753 if (bp->nb_endio > bp->nb_offio) {
2754 firstpg = bp->nb_offio / PAGE_SIZE;
2755 lastpg = (bp->nb_endio - 1) / PAGE_SIZE;
2756 /* calculate mask for that page range written */
2757 nfs_buf_pgs_set_pages_between(&pagemask, firstpg, lastpg + 1);
2758 } else {
2759 NBPGS_ERASE(&pagemask);
2760 }
2761 /* clear dirty bits for pages we've written */
2762 nfs_buf_pgs_bit_not(&pagemask);
2763 nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &bp->nb_dirty);
2764 }
2765
2766 /* manage needcommit state */
2767 if (!error && (bp->nb_commitlevel == NFS_WRITE_UNSTABLE)) {
2768 if (!ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
2769 nfs_node_lock_force(np);
2770 np->n_needcommitcnt++;
2771 nfs_node_unlock(np);
2772 SET(bp->nb_flags, NB_NEEDCOMMIT);
2773 }
2774 /* make sure nb_dirtyoff/nb_dirtyend reflect actual range written */
2775 bp->nb_dirtyoff = bp->nb_offio;
2776 bp->nb_dirtyend = bp->nb_endio;
2777 } else if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
2778 nfs_node_lock_force(np);
2779 np->n_needcommitcnt--;
2780 CHECK_NEEDCOMMITCNT(np);
2781 nfs_node_unlock(np);
2782 CLR(bp->nb_flags, NB_NEEDCOMMIT);
2783 }
2784
2785 CLR(bp->nb_flags, NB_WRITEINPROG);
2786
2787 /*
2788 * For an unstable write, the buffer is still treated as dirty until
2789 * a commit (or stable (re)write) is performed. Buffers needing only
2790 * a commit are marked with the NB_DELWRI and NB_NEEDCOMMIT flags.
2791 *
2792 * If the write was interrupted we set NB_EINTR. Don't set NB_ERROR
2793 * because that would cause the buffer to be dropped. The buffer is
2794 * still valid and simply needs to be written again.
2795 */
2796 if ((error == EINTR) || (error == ERESTART) || (!error && (bp->nb_flags & NB_NEEDCOMMIT))) {
2797 CLR(bp->nb_flags, NB_INVAL);
2798 if (!ISSET(bp->nb_flags, NB_DELWRI)) {
2799 SET(bp->nb_flags, NB_DELWRI);
2800 lck_mtx_lock(&nfs_buf_mutex);
2801 nfs_nbdwrite++;
2802 NFSBUFCNTCHK();
2803 lck_mtx_unlock(&nfs_buf_mutex);
2804 }
2805 /*
2806 * Since for the NB_ASYNC case, we've reassigned the buffer to the
2807 * clean list, we have to reassign it back to the dirty one. Ugh.
2808 */
2809 if (ISSET(bp->nb_flags, NB_ASYNC)) {
2810 /* move to dirty list */
2811 lck_mtx_lock(&nfs_buf_mutex);
2812 if (bp->nb_vnbufs.le_next != NFSNOLIST) {
2813 LIST_REMOVE(bp, nb_vnbufs);
2814 }
2815 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
2816 lck_mtx_unlock(&nfs_buf_mutex);
2817 }
2818 } else {
2819 /* either there's an error or we don't need to commit */
2820 if (error) {
2821 /*
2822 * There was a write error and we need to invalidate
2823 * attrs and flush buffers in order to sync up with the
2824 * server. (if this write was extending the file, we
2825 * may no longer know the correct size)
2826 *
2827 * But we can't call vinvalbuf while holding this
2828 * buffer busy. Set a flag to do it after releasing
2829 * the buffer.
2830 */
2831 nfs_node_lock_force(np);
2832 np->n_error = error;
2833 np->n_flag |= (NWRITEERR | NNEEDINVALIDATE);
2834 NATTRINVALIDATE(np);
2835 nfs_node_unlock(np);
2836 }
2837 /* clear the dirty range */
2838 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
2839 }
2840
2841 if (!error && nfs_buf_pgs_is_set(&bp->nb_dirty)) {
2842 nfs_buf_write_dirty_pages(bp, thd, cred);
2843 }
2844 nfs_buf_iodone(bp);
2845 }
2846
2847 /*
2848 * write out any pages marked dirty in a buffer
2849 *
2850 * We do use unstable writes and follow up with a commit.
2851 * If we catch the write verifier changing we'll restart
2852 * do the writes filesync.
2853 */
2854 int
nfs_buf_write_dirty_pages(struct nfsbuf * bp,thread_t thd,kauth_cred_t cred)2855 nfs_buf_write_dirty_pages(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
2856 {
2857 nfsnode_t np = bp->nb_np;
2858 struct nfsmount *nmp = NFSTONMP(np);
2859 int error = 0, commit, iomode, iomode2, len, pg, count, npages, off;
2860 nfsbufpgs dirty;
2861 uint64_t wverf;
2862 uio_t auio;
2863
2864 if (!nfs_buf_pgs_is_set(&bp->nb_dirty)) {
2865 return 0;
2866 }
2867
2868 /* there are pages marked dirty that need to be written out */
2869 OSAddAtomic64(1, &nfsclntstats.write_bios);
2870 NFS_BUF_MAP(bp);
2871 SET(bp->nb_flags, NB_WRITEINPROG);
2872 npages = bp->nb_bufsize / PAGE_SIZE;
2873 iomode = NFS_WRITE_UNSTABLE;
2874
2875 auio = uio_create(1, 0, UIO_SYSSPACE, UIO_WRITE);
2876
2877 again:
2878 NBPGS_COPY(&dirty, &bp->nb_dirty);
2879 wverf = bp->nb_verf;
2880 commit = NFS_WRITE_FILESYNC;
2881 for (pg = 0; pg < npages; pg++) {
2882 if (!NBPGDIRTY(bp, pg)) {
2883 continue;
2884 }
2885 count = 1;
2886 while (((pg + count) < npages) && NBPGDIRTY(bp, pg + count)) {
2887 count++;
2888 }
2889 /* write count pages starting with page pg */
2890 off = pg * PAGE_SIZE;
2891 len = count * PAGE_SIZE;
2892 /* clip writes to EOF */
2893 if (NBOFF(bp) + off + len > (off_t) np->n_size) {
2894 len -= (NBOFF(bp) + off + len) - np->n_size;
2895 }
2896 if (len > 0) {
2897 iomode2 = iomode;
2898 uio_reset(auio, NBOFF(bp) + off, UIO_SYSSPACE, UIO_WRITE);
2899 uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + off), len);
2900 error = nfs_write_rpc2(np, auio, thd, cred, &iomode2, &bp->nb_verf);
2901 if (error) {
2902 break;
2903 }
2904 if (iomode2 < commit) { /* Retain the lowest commitment level returned. */
2905 commit = iomode2;
2906 }
2907 if ((commit != NFS_WRITE_FILESYNC) && (wverf != bp->nb_verf)) {
2908 /* verifier changed, redo all the writes filesync */
2909 iomode = NFS_WRITE_FILESYNC;
2910 goto again;
2911 }
2912 }
2913 /* clear dirty bits */
2914 while (count--) {
2915 NBPGS_UNSET(&dirty, pg);
2916 if (count) { /* leave pg on last page */
2917 pg++;
2918 }
2919 }
2920 }
2921 CLR(bp->nb_flags, NB_WRITEINPROG);
2922
2923 if (!error && (commit != NFS_WRITE_FILESYNC)) {
2924 error = nmp->nm_funcs->nf_commit_rpc(np, NBOFF(bp), bp->nb_bufsize, cred, wverf);
2925 if (error == NFSERR_STALEWRITEVERF) {
2926 /* verifier changed, so we need to restart all the writes */
2927 iomode = NFS_WRITE_FILESYNC;
2928 goto again;
2929 }
2930 }
2931 if (!error) {
2932 NBPGS_COPY(&bp->nb_dirty, &dirty);
2933 } else {
2934 SET(bp->nb_flags, NB_ERROR);
2935 bp->nb_error = error;
2936 }
2937
2938 /* Free allocated uio buffer */
2939 uio_free(auio);
2940
2941 return error;
2942 }
2943
2944 /*
2945 * initiate the NFS WRITE RPC(s) for a buffer
2946 */
2947 int
nfs_buf_write_rpc(struct nfsbuf * bp,int iomode,thread_t thd,kauth_cred_t cred)2948 nfs_buf_write_rpc(struct nfsbuf *bp, int iomode, thread_t thd, kauth_cred_t cred)
2949 {
2950 struct nfsmount *nmp;
2951 nfsnode_t np = bp->nb_np;
2952 int error = 0, nfsvers, async;
2953 int64_t nrpcs;
2954 size_t len;
2955 uint32_t nmwsize;
2956 struct nfsreq *req;
2957 struct nfsreq_cbinfo cb;
2958 uio_t auio;
2959 off_t offset, length;
2960
2961 nmp = NFSTONMP(np);
2962 if (nfs_mount_gone(nmp)) {
2963 bp->nb_error = error = ENXIO;
2964 SET(bp->nb_flags, NB_ERROR);
2965 nfs_buf_iodone(bp);
2966 return error;
2967 }
2968 nfsvers = nmp->nm_vers;
2969 nmwsize = nmp->nm_wsize;
2970
2971 offset = bp->nb_offio;
2972 length = bp->nb_endio - bp->nb_offio;
2973
2974 /* Note: Can only do async I/O if nfsiods are configured. */
2975 async = (bp->nb_flags & NB_ASYNC) && (NFSIOD_MAX > 0);
2976 bp->nb_commitlevel = NFS_WRITE_FILESYNC;
2977 cb.rcb_func = async ? nfs_buf_write_rpc_finish : NULL;
2978 cb.rcb_bp = bp;
2979
2980 if ((nfsvers == NFS_VER2) && ((NBOFF(bp) + bp->nb_endio) > 0xffffffffLL)) {
2981 bp->nb_error = error = EFBIG;
2982 SET(bp->nb_flags, NB_ERROR);
2983 nfs_buf_iodone(bp);
2984 return error;
2985 }
2986
2987 if (length == 0) {
2988 /* We should never get here */
2989 #if DEVELOPMENT
2990 printf("nfs_buf_write_rpc: Got request with zero length. np %p, bp %p, offset %lld\n", np, bp, offset);
2991 #else
2992 printf("nfs_buf_write_rpc: Got request with zero length.\n");
2993 #endif /* DEVELOPMENT */
2994 nfs_buf_iodone(bp);
2995 return 0;
2996 }
2997
2998 auio = uio_create(1, NBOFF(bp) + offset, UIO_SYSSPACE, UIO_WRITE);
2999 NFS_UIO_ADDIOV(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
3000
3001 bp->nb_rpcs = nrpcs = (length + nmwsize - 1) / nmwsize;
3002 if (async && (nrpcs > 1)) {
3003 SET(bp->nb_flags, NB_MULTASYNCRPC);
3004 } else {
3005 CLR(bp->nb_flags, NB_MULTASYNCRPC);
3006 }
3007
3008 while (length > 0) {
3009 if (ISSET(bp->nb_flags, NB_ERROR)) {
3010 error = bp->nb_error;
3011 break;
3012 }
3013 len = (length > nmwsize) ? nmwsize : (uint32_t)length;
3014 cb.rcb_args.offset = offset;
3015 cb.rcb_args.length = len;
3016 #if CONFIG_NFS4
3017 if (nmp->nm_vers >= NFS_VER4) {
3018 cb.rcb_args.stategenid = nmp->nm_stategenid;
3019 }
3020 #endif
3021 if (async && ((error = nfs_async_write_start(nmp)))) {
3022 break;
3023 }
3024 req = NULL;
3025 error = nmp->nm_funcs->nf_write_rpc_async(np, auio, len, thd, cred,
3026 iomode, &cb, &req);
3027 if (error) {
3028 if (async) {
3029 nfs_async_write_done(nmp);
3030 }
3031 break;
3032 }
3033 offset += len;
3034 length -= len;
3035 if (async) {
3036 continue;
3037 }
3038 nfs_buf_write_rpc_finish(req);
3039 }
3040
3041 if (length > 0) {
3042 /*
3043 * Something bad happened while trying to send the RPCs.
3044 * Wait for any outstanding requests to complete.
3045 */
3046 bp->nb_error = error;
3047 SET(bp->nb_flags, NB_ERROR);
3048 if (ISSET(bp->nb_flags, NB_MULTASYNCRPC)) {
3049 nrpcs = (length + nmwsize - 1) / nmwsize;
3050 lck_mtx_lock(&nfs_buf_mutex);
3051 bp->nb_rpcs -= nrpcs;
3052 if (bp->nb_rpcs == 0) {
3053 /* No RPCs left, so the buffer's done */
3054 lck_mtx_unlock(&nfs_buf_mutex);
3055 nfs_buf_write_finish(bp, thd, cred);
3056 } else {
3057 /* wait for the last RPC to mark it done */
3058 while (bp->nb_rpcs > 0) {
3059 msleep(&bp->nb_rpcs, &nfs_buf_mutex, 0,
3060 "nfs_buf_write_rpc_cancel", NULL);
3061 }
3062 lck_mtx_unlock(&nfs_buf_mutex);
3063 }
3064 } else {
3065 nfs_buf_write_finish(bp, thd, cred);
3066 }
3067 /* It may have just been an interrupt... that's OK */
3068 if (!ISSET(bp->nb_flags, NB_ERROR)) {
3069 error = 0;
3070 }
3071 }
3072
3073 /* Free allocated uio buffer */
3074 uio_free(auio);
3075
3076 return error;
3077 }
3078
3079 /*
3080 * finish up an NFS WRITE RPC on a buffer
3081 */
3082 void
nfs_buf_write_rpc_finish(struct nfsreq * req)3083 nfs_buf_write_rpc_finish(struct nfsreq *req)
3084 {
3085 int error = 0, nfsvers, multasyncrpc, finished;
3086 int committed = NFS_WRITE_FILESYNC;
3087 uint64_t wverf = 0;
3088 off_t offset;
3089 size_t rlen, length;
3090 void *wakeme = NULL;
3091 struct nfsreq_cbinfo cb;
3092 struct nfsreq *wreq = NULL;
3093 struct nfsbuf *bp;
3094 struct nfsmount *nmp;
3095 nfsnode_t np;
3096 thread_t thd;
3097 kauth_cred_t cred;
3098 uio_t auio;
3099
3100 finish:
3101 np = req->r_np;
3102 thd = req->r_thread;
3103 cred = req->r_cred;
3104 if (IS_VALID_CRED(cred)) {
3105 kauth_cred_ref(cred);
3106 }
3107 cb = req->r_callback;
3108 bp = cb.rcb_bp;
3109 if (cb.rcb_func) { /* take an extra reference on the nfsreq in case we want to resend it later due to grace error */
3110 nfs_request_ref(req, 0);
3111 }
3112
3113 nmp = NFSTONMP(np);
3114 if (nfs_mount_gone(nmp)) {
3115 SET(bp->nb_flags, NB_ERROR);
3116 bp->nb_error = error = ENXIO;
3117 }
3118 if (error || ISSET(bp->nb_flags, NB_ERROR)) {
3119 /* just drop it */
3120 nfs_request_async_cancel(req);
3121 goto out;
3122 }
3123 nfsvers = nmp->nm_vers;
3124
3125 offset = cb.rcb_args.offset;
3126 rlen = length = cb.rcb_args.length;
3127
3128 /* finish the RPC */
3129 error = nmp->nm_funcs->nf_write_rpc_async_finish(np, req, &committed, &rlen, &wverf);
3130 if ((error == EINPROGRESS) && cb.rcb_func) {
3131 /* async request restarted */
3132 if (cb.rcb_func) {
3133 nfs_request_rele(req);
3134 }
3135 if (IS_VALID_CRED(cred)) {
3136 kauth_cred_unref(&cred);
3137 }
3138 return;
3139 }
3140 #if CONFIG_NFS4
3141 if ((nmp->nm_vers >= NFS_VER4) && nfs_mount_state_error_should_restart(error) && !ISSET(bp->nb_flags, NB_ERROR)) {
3142 lck_mtx_lock(&nmp->nm_lock);
3143 if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args.stategenid == nmp->nm_stategenid)) {
3144 NP(np, "nfs_buf_write_rpc_finish: error %d @ 0x%llx, 0x%x 0x%x, initiating recovery",
3145 error, NBOFF(bp) + offset, cb.rcb_args.stategenid, nmp->nm_stategenid);
3146 nfs_need_recover(nmp, error);
3147 }
3148 lck_mtx_unlock(&nmp->nm_lock);
3149 if (np->n_flag & NREVOKE) {
3150 error = EIO;
3151 } else {
3152 if (error == NFSERR_GRACE) {
3153 if (cb.rcb_func) {
3154 /*
3155 * For an async I/O request, handle a grace delay just like
3156 * jukebox errors. Set the resend time and queue it up.
3157 */
3158 struct timeval now;
3159 if (req->r_nmrep.nmc_mhead) {
3160 mbuf_freem(req->r_nmrep.nmc_mhead);
3161 req->r_nmrep.nmc_mhead = NULL;
3162 }
3163 req->r_error = 0;
3164 microuptime(&now);
3165 lck_mtx_lock(&req->r_mtx);
3166 req->r_resendtime = now.tv_sec + 2;
3167 req->r_xid = 0; // get a new XID
3168 req->r_flags |= R_RESTART;
3169 req->r_start = 0;
3170 nfs_asyncio_resend(req);
3171 lck_mtx_unlock(&req->r_mtx);
3172 if (IS_VALID_CRED(cred)) {
3173 kauth_cred_unref(&cred);
3174 }
3175 /* Note: nfsreq reference taken will be dropped later when finished */
3176 return;
3177 }
3178 /* otherwise, just pause a couple seconds and retry */
3179 tsleep(&nmp->nm_state, (PZERO - 1), "nfsgrace", 2 * hz);
3180 }
3181 if (!(error = nfs_mount_state_wait_for_recovery(nmp))) {
3182 rlen = 0;
3183 goto writeagain;
3184 }
3185 }
3186 }
3187 #endif
3188 if (error) {
3189 SET(bp->nb_flags, NB_ERROR);
3190 bp->nb_error = error;
3191 }
3192 if (error || (nfsvers == NFS_VER2)) {
3193 goto out;
3194 }
3195 if (rlen <= 0) {
3196 SET(bp->nb_flags, NB_ERROR);
3197 bp->nb_error = error = EIO;
3198 goto out;
3199 }
3200
3201 /* save lowest commit level returned */
3202 if (committed < bp->nb_commitlevel) {
3203 bp->nb_commitlevel = committed;
3204 }
3205
3206 /* check the write verifier */
3207 if (!bp->nb_verf) {
3208 bp->nb_verf = wverf;
3209 } else if (bp->nb_verf != wverf) {
3210 /* verifier changed, so buffer will need to be rewritten */
3211 bp->nb_flags |= NB_STALEWVERF;
3212 bp->nb_commitlevel = NFS_WRITE_UNSTABLE;
3213 bp->nb_verf = wverf;
3214 }
3215
3216 if (!ISSET(bp->nb_flags, NB_STALEWVERF) && rlen > 0 && (bp->nb_offio < (offset + (int)rlen))) {
3217 bp->nb_offio = offset + rlen;
3218 }
3219
3220 /*
3221 * check for a short write
3222 *
3223 * If the server didn't write all the data, then we
3224 * need to issue another write for the rest of it.
3225 * (Don't bother if the buffer hit an error or stale wverf.)
3226 */
3227 if ((rlen < length) && !(bp->nb_flags & (NB_STALEWVERF | NB_ERROR))) {
3228 #if CONFIG_NFS4
3229 writeagain:
3230 #endif
3231 offset += rlen;
3232 length -= rlen;
3233
3234 auio = uio_create(1, NBOFF(bp) + offset, UIO_SYSSPACE, UIO_WRITE);
3235 uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
3236
3237 cb.rcb_args.offset = offset;
3238 cb.rcb_args.length = length;
3239 #if CONFIG_NFS4
3240 if (nmp->nm_vers >= NFS_VER4) {
3241 cb.rcb_args.stategenid = nmp->nm_stategenid;
3242 }
3243 #endif
3244 // XXX iomode should really match the original request
3245 error = nmp->nm_funcs->nf_write_rpc_async(np, auio, length, thd, cred,
3246 NFS_WRITE_FILESYNC, &cb, &wreq);
3247
3248 /* Free allocated uio buffer */
3249 uio_free(auio);
3250
3251 if (!error) {
3252 if (IS_VALID_CRED(cred)) {
3253 kauth_cred_unref(&cred);
3254 }
3255 if (!cb.rcb_func) {
3256 /* if !async we'll need to wait for this RPC to finish */
3257 req = wreq;
3258 wreq = NULL;
3259 goto finish;
3260 }
3261 nfs_request_rele(req);
3262 /*
3263 * We're done here.
3264 * Outstanding RPC count is unchanged.
3265 * Callback will be called when RPC is done.
3266 */
3267 return;
3268 }
3269 SET(bp->nb_flags, NB_ERROR);
3270 bp->nb_error = error;
3271 }
3272
3273 out:
3274 if (cb.rcb_func) {
3275 nfs_async_write_done(nmp);
3276 nfs_request_rele(req);
3277 }
3278 /*
3279 * Decrement outstanding RPC count on buffer
3280 * and call nfs_buf_write_finish on last RPC.
3281 *
3282 * (Note: when there are multiple async RPCs issued for a
3283 * buffer we need nfs_buffer_mutex to avoid problems when
3284 * aborting a partially-initiated set of RPCs)
3285 */
3286 multasyncrpc = ISSET(bp->nb_flags, NB_MULTASYNCRPC);
3287 if (multasyncrpc) {
3288 lck_mtx_lock(&nfs_buf_mutex);
3289 }
3290
3291 bp->nb_rpcs--;
3292 finished = (bp->nb_rpcs == 0);
3293
3294 if (multasyncrpc) {
3295 lck_mtx_unlock(&nfs_buf_mutex);
3296 }
3297
3298 if (finished) {
3299 if (multasyncrpc) {
3300 wakeme = &bp->nb_rpcs;
3301 }
3302 nfs_buf_write_finish(bp, thd, cred);
3303 if (wakeme) {
3304 wakeup(wakeme);
3305 }
3306 }
3307
3308 if (IS_VALID_CRED(cred)) {
3309 kauth_cred_unref(&cred);
3310 }
3311
3312 if (cb.rcb_func && np->n_needcommitcnt >= NFS_A_LOT_OF_NEEDCOMMITS) {
3313 nfs_flushcommits(np, 1);
3314 }
3315 }
3316
3317 /*
3318 * Send commit(s) for the given node's "needcommit" buffers
3319 */
3320 int
nfs_flushcommits(nfsnode_t np,int nowait)3321 nfs_flushcommits(nfsnode_t np, int nowait)
3322 {
3323 struct nfsmount *nmp;
3324 struct nfsbuf *bp, *prevlbp, *lbp;
3325 struct nfsbuflists blist, commitlist;
3326 int error = 0, retv, wcred_set, flags;
3327 u_quad_t off, endoff, toff;
3328 uint64_t wverf, count;
3329 kauth_cred_t wcred = NULL;
3330 nfsbufpgs dirty;
3331
3332 FSDBG_TOP(557, np, 0, 0, 0);
3333
3334 /*
3335 * A nb_flags == (NB_DELWRI | NB_NEEDCOMMIT) block has been written to the
3336 * server, but nas not been committed to stable storage on the server
3337 * yet. The byte range is worked out for as many nfsbufs as we can handle
3338 * and the commit rpc is done.
3339 */
3340 if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
3341 error = nfs_node_lock(np);
3342 if (error) {
3343 goto done;
3344 }
3345 np->n_flag |= NMODIFIED;
3346 nfs_node_unlock(np);
3347 }
3348
3349 off = (u_quad_t)-1;
3350 endoff = 0;
3351 wcred_set = 0;
3352 LIST_INIT(&commitlist);
3353
3354 nmp = NFSTONMP(np);
3355 if (nfs_mount_gone(nmp)) {
3356 error = ENXIO;
3357 goto done;
3358 }
3359 if (nmp->nm_vers == NFS_VER2) {
3360 error = EINVAL;
3361 goto done;
3362 }
3363
3364 flags = NBI_DIRTY;
3365 if (nowait) {
3366 flags |= NBI_NOWAIT;
3367 }
3368 lck_mtx_lock(&nfs_buf_mutex);
3369 wverf = nmp->nm_verf;
3370 if (!nfs_buf_iterprepare(np, &blist, flags)) {
3371 while ((bp = LIST_FIRST(&blist))) {
3372 LIST_REMOVE(bp, nb_vnbufs);
3373 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
3374 error = nfs_buf_acquire(bp, NBAC_NOWAIT, 0, 0);
3375 if (error) {
3376 continue;
3377 }
3378 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
3379 nfs_buf_check_write_verifier(np, bp);
3380 }
3381 if (((bp->nb_flags & (NB_DELWRI | NB_NEEDCOMMIT)) != (NB_DELWRI | NB_NEEDCOMMIT)) ||
3382 (bp->nb_verf != wverf)) {
3383 nfs_buf_drop(bp);
3384 continue;
3385 }
3386 nfs_buf_remfree(bp);
3387
3388 /* buffer UPLs will be grabbed *in order* below */
3389
3390 FSDBG(557, bp, bp->nb_flags, bp->nb_valid, bp->nb_dirty);
3391 FSDBG(557, bp->nb_validoff, bp->nb_validend,
3392 bp->nb_dirtyoff, bp->nb_dirtyend);
3393
3394 /*
3395 * Work out if all buffers are using the same cred
3396 * so we can deal with them all with one commit.
3397 *
3398 * Note: creds in bp's must be obtained by kauth_cred_ref
3399 * on the same original cred in order for them to be equal.
3400 */
3401 if (wcred_set == 0) {
3402 wcred = bp->nb_wcred;
3403 if (!IS_VALID_CRED(wcred)) {
3404 panic("nfs: needcommit w/out wcred");
3405 }
3406 wcred_set = 1;
3407 } else if ((wcred_set == 1) && wcred != bp->nb_wcred) {
3408 wcred_set = -1;
3409 }
3410 SET(bp->nb_flags, NB_WRITEINPROG);
3411
3412 /*
3413 * Add this buffer to the list of buffers we are committing.
3414 * Buffers are inserted into the list in ascending order so that
3415 * we can take the UPLs in order after the list is complete.
3416 */
3417 prevlbp = NULL;
3418 LIST_FOREACH(lbp, &commitlist, nb_vnbufs) {
3419 if (bp->nb_lblkno < lbp->nb_lblkno) {
3420 break;
3421 }
3422 prevlbp = lbp;
3423 }
3424 LIST_REMOVE(bp, nb_vnbufs);
3425 if (prevlbp) {
3426 LIST_INSERT_AFTER(prevlbp, bp, nb_vnbufs);
3427 } else {
3428 LIST_INSERT_HEAD(&commitlist, bp, nb_vnbufs);
3429 }
3430
3431 /* update commit range start, end */
3432 toff = NBOFF(bp) + bp->nb_dirtyoff;
3433 if (toff < off) {
3434 off = toff;
3435 }
3436 toff += (u_quad_t)(bp->nb_dirtyend - bp->nb_dirtyoff);
3437 if (toff > endoff) {
3438 endoff = toff;
3439 }
3440 }
3441 nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
3442 }
3443 lck_mtx_unlock(&nfs_buf_mutex);
3444
3445 if (LIST_EMPTY(&commitlist)) {
3446 error = ENOBUFS;
3447 goto done;
3448 }
3449
3450 /*
3451 * We need a UPL to prevent others from accessing the buffers during
3452 * our commit RPC(s).
3453 *
3454 * We used to also check for dirty pages here; if there were any we'd
3455 * abort the commit and force the entire buffer to be written again.
3456 * Instead of doing that, we just go ahead and commit the dirty range,
3457 * and then leave the buffer around with dirty pages that will be
3458 * written out later.
3459 */
3460 LIST_FOREACH(bp, &commitlist, nb_vnbufs) {
3461 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
3462 retv = nfs_buf_upl_setup(bp);
3463 if (retv) {
3464 /* Unable to create the UPL, the VM object probably no longer exists. */
3465 printf("nfs_flushcommits: upl create failed %d\n", retv);
3466 NBPGS_ERASE(&bp->nb_valid);
3467 NBPGS_ERASE(&bp->nb_dirty);
3468 }
3469 }
3470 nfs_buf_upl_check(bp);
3471 }
3472
3473 /*
3474 * Commit data on the server, as required.
3475 * If all bufs are using the same wcred, then use that with
3476 * one call for all of them, otherwise commit each one
3477 * separately.
3478 */
3479 if (wcred_set == 1) {
3480 /*
3481 * Note, it's possible the commit range could be >2^32-1.
3482 * If it is, we'll send one commit that covers the whole file.
3483 */
3484 if ((endoff - off) > 0xffffffff) {
3485 count = 0;
3486 } else {
3487 count = (endoff - off);
3488 }
3489 retv = nmp->nm_funcs->nf_commit_rpc(np, off, count, wcred, wverf);
3490 } else {
3491 retv = 0;
3492 LIST_FOREACH(bp, &commitlist, nb_vnbufs) {
3493 toff = NBOFF(bp) + bp->nb_dirtyoff;
3494 count = bp->nb_dirtyend - bp->nb_dirtyoff;
3495 retv = nmp->nm_funcs->nf_commit_rpc(np, toff, count, bp->nb_wcred, wverf);
3496 if (retv) {
3497 break;
3498 }
3499 }
3500 }
3501
3502 /*
3503 * Now, either mark the blocks I/O done or mark the
3504 * blocks dirty, depending on whether the commit
3505 * succeeded.
3506 */
3507 while ((bp = LIST_FIRST(&commitlist))) {
3508 LIST_REMOVE(bp, nb_vnbufs);
3509 FSDBG(557, bp, retv, bp->nb_flags, bp->nb_dirty);
3510 nfs_node_lock_force(np);
3511 CLR(bp->nb_flags, (NB_NEEDCOMMIT | NB_WRITEINPROG));
3512 np->n_needcommitcnt--;
3513 CHECK_NEEDCOMMITCNT(np);
3514 nfs_node_unlock(np);
3515
3516 if (retv) {
3517 /* move back to dirty list */
3518 lck_mtx_lock(&nfs_buf_mutex);
3519 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
3520 lck_mtx_unlock(&nfs_buf_mutex);
3521 nfs_buf_release(bp, 1);
3522 continue;
3523 }
3524
3525 nfs_node_lock_force(np);
3526 np->n_numoutput++;
3527 nfs_node_unlock(np);
3528 vnode_startwrite(NFSTOV(np));
3529 if (ISSET(bp->nb_flags, NB_DELWRI)) {
3530 lck_mtx_lock(&nfs_buf_mutex);
3531 nfs_nbdwrite--;
3532 NFSBUFCNTCHK();
3533 lck_mtx_unlock(&nfs_buf_mutex);
3534 wakeup(&nfs_nbdwrite);
3535 }
3536 CLR(bp->nb_flags, (NB_READ | NB_DONE | NB_ERROR | NB_DELWRI));
3537 /* if block still has dirty pages, we don't want it to */
3538 /* be released in nfs_buf_iodone(). So, don't set NB_ASYNC. */
3539 NBPGS_COPY(&dirty, &bp->nb_dirty);
3540 if (!nfs_buf_pgs_is_set(&dirty)) {
3541 SET(bp->nb_flags, NB_ASYNC);
3542 } else {
3543 CLR(bp->nb_flags, NB_ASYNC);
3544 }
3545
3546 /* move to clean list */
3547 lck_mtx_lock(&nfs_buf_mutex);
3548 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
3549 lck_mtx_unlock(&nfs_buf_mutex);
3550
3551 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
3552
3553 nfs_buf_iodone(bp);
3554 if (nfs_buf_pgs_is_set(&dirty)) {
3555 /* throw it back in as a delayed write buffer */
3556 CLR(bp->nb_flags, NB_DONE);
3557 nfs_buf_write_delayed(bp);
3558 }
3559 }
3560
3561 done:
3562 FSDBG_BOT(557, np, 0, 0, error);
3563 return error;
3564 }
3565
3566 /*
3567 * Flush all the blocks associated with a vnode.
3568 * Walk through the buffer pool and push any dirty pages
3569 * associated with the vnode.
3570 */
3571 int
nfs_flush(nfsnode_t np,int waitfor,thread_t thd,int ignore_writeerr)3572 nfs_flush(nfsnode_t np, int waitfor, thread_t thd, int ignore_writeerr)
3573 {
3574 struct nfsbuf *bp;
3575 struct nfsbuflists blist;
3576 struct nfsmount *nmp = NFSTONMP(np);
3577 int error = 0, error2, slptimeo = 0, slpflag = 0;
3578 int nfsvers, flags, passone = 1;
3579
3580 FSDBG_TOP(517, np, waitfor, ignore_writeerr, 0);
3581
3582 if (nfs_mount_gone(nmp)) {
3583 error = ENXIO;
3584 goto out;
3585 }
3586 nfsvers = nmp->nm_vers;
3587 if (NMFLAG(nmp, INTR)) {
3588 slpflag = PCATCH;
3589 }
3590
3591 if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
3592 nfs_node_lock_force(np);
3593 np->n_flag |= NMODIFIED;
3594 nfs_node_unlock(np);
3595 }
3596
3597 lck_mtx_lock(&nfs_buf_mutex);
3598 while (np->n_bflag & NBFLUSHINPROG) {
3599 np->n_bflag |= NBFLUSHWANT;
3600 error = msleep(&np->n_bflag, &nfs_buf_mutex, slpflag, "nfs_flush", NULL);
3601 if ((error && (error != EWOULDBLOCK)) ||
3602 ((error = nfs_sigintr(NFSTONMP(np), NULL, thd, 0)))) {
3603 lck_mtx_unlock(&nfs_buf_mutex);
3604 goto out;
3605 }
3606 }
3607 np->n_bflag |= NBFLUSHINPROG;
3608
3609 /*
3610 * On the first pass, start async/unstable writes on all
3611 * delayed write buffers. Then wait for all writes to complete
3612 * and call nfs_flushcommits() to commit any uncommitted buffers.
3613 * On all subsequent passes, start STABLE writes on any remaining
3614 * dirty buffers. Then wait for all writes to complete.
3615 */
3616 again:
3617 FSDBG(518, LIST_FIRST(&np->n_dirtyblkhd), np->n_flag, 0, 0);
3618 if (!NFSTONMP(np)) {
3619 lck_mtx_unlock(&nfs_buf_mutex);
3620 error = ENXIO;
3621 goto done;
3622 }
3623
3624 /* Start/do any write(s) that are required. */
3625 if (!nfs_buf_iterprepare(np, &blist, NBI_DIRTY)) {
3626 while ((bp = LIST_FIRST(&blist))) {
3627 LIST_REMOVE(bp, nb_vnbufs);
3628 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
3629 flags = (passone || !(waitfor == MNT_WAIT || waitfor == MNT_DWAIT)) ? NBAC_NOWAIT : 0;
3630 if (flags != NBAC_NOWAIT) {
3631 nfs_buf_refget(bp);
3632 }
3633 while ((error = nfs_buf_acquire(bp, flags, slpflag, slptimeo))) {
3634 FSDBG(524, bp, flags, bp->nb_lflags, bp->nb_flags);
3635 if (error == EBUSY) {
3636 break;
3637 }
3638 if (error) {
3639 error2 = nfs_sigintr(NFSTONMP(np), NULL, thd, 0);
3640 if (error2) {
3641 if (flags != NBAC_NOWAIT) {
3642 nfs_buf_refrele(bp);
3643 }
3644 nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
3645 lck_mtx_unlock(&nfs_buf_mutex);
3646 error = error2;
3647 goto done;
3648 }
3649 if (slpflag == PCATCH) {
3650 slpflag = 0;
3651 slptimeo = 2 * hz;
3652 }
3653 }
3654 }
3655 if (flags != NBAC_NOWAIT) {
3656 nfs_buf_refrele(bp);
3657 }
3658 if (error == EBUSY) {
3659 continue;
3660 }
3661 if (!bp->nb_np) {
3662 /* buffer is no longer valid */
3663 nfs_buf_drop(bp);
3664 continue;
3665 }
3666 if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
3667 nfs_buf_check_write_verifier(np, bp);
3668 }
3669 if (!ISSET(bp->nb_flags, NB_DELWRI)) {
3670 /* buffer is no longer dirty */
3671 nfs_buf_drop(bp);
3672 continue;
3673 }
3674 FSDBG(525, bp, passone, bp->nb_lflags, bp->nb_flags);
3675 if ((passone || !(waitfor == MNT_WAIT || waitfor == MNT_DWAIT)) &&
3676 ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
3677 nfs_buf_drop(bp);
3678 continue;
3679 }
3680 nfs_buf_remfree(bp);
3681 lck_mtx_unlock(&nfs_buf_mutex);
3682 if (ISSET(bp->nb_flags, NB_ERROR)) {
3683 nfs_node_lock_force(np);
3684 np->n_error = bp->nb_error ? bp->nb_error : EIO;
3685 np->n_flag |= NWRITEERR;
3686 nfs_node_unlock(np);
3687 nfs_buf_release(bp, 1);
3688 lck_mtx_lock(&nfs_buf_mutex);
3689 continue;
3690 }
3691 SET(bp->nb_flags, NB_ASYNC);
3692 if (!passone) {
3693 /* NB_STABLE forces this to be written FILESYNC */
3694 SET(bp->nb_flags, NB_STABLE);
3695 }
3696 nfs_buf_write(bp);
3697 lck_mtx_lock(&nfs_buf_mutex);
3698 }
3699 nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
3700 }
3701 lck_mtx_unlock(&nfs_buf_mutex);
3702
3703 if (waitfor == MNT_WAIT || waitfor == MNT_DWAIT) {
3704 while ((error = vnode_waitforwrites(NFSTOV(np), 0, slpflag, slptimeo, "nfsflush"))) {
3705 error2 = nfs_sigintr(NFSTONMP(np), NULL, thd, 0);
3706 if (error2) {
3707 error = error2;
3708 goto done;
3709 }
3710 if (slpflag == PCATCH) {
3711 slpflag = 0;
3712 slptimeo = 2 * hz;
3713 }
3714 }
3715 }
3716
3717 if (nfsvers != NFS_VER2) {
3718 /* loop while it looks like there are still buffers to be */
3719 /* commited and nfs_flushcommits() seems to be handling them. */
3720 while (np->n_needcommitcnt) {
3721 if (nfs_flushcommits(np, 0)) {
3722 break;
3723 }
3724 }
3725 }
3726
3727 if (passone) {
3728 passone = 0;
3729 if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
3730 nfs_node_lock_force(np);
3731 np->n_flag |= NMODIFIED;
3732 nfs_node_unlock(np);
3733 }
3734 lck_mtx_lock(&nfs_buf_mutex);
3735 goto again;
3736 }
3737
3738 if (waitfor == MNT_WAIT || waitfor == MNT_DWAIT) {
3739 if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
3740 nfs_node_lock_force(np);
3741 np->n_flag |= NMODIFIED;
3742 nfs_node_unlock(np);
3743 }
3744 lck_mtx_lock(&nfs_buf_mutex);
3745 if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
3746 goto again;
3747 }
3748 lck_mtx_unlock(&nfs_buf_mutex);
3749 nfs_node_lock_force(np);
3750 /*
3751 * OK, it looks like there are no dirty blocks. If we have no
3752 * writes in flight and no one in the write code, we can clear
3753 * the modified flag. In order to make sure we see the latest
3754 * attributes and size, we also invalidate the attributes and
3755 * advance the attribute cache XID to guarantee that attributes
3756 * newer than our clearing of NMODIFIED will get loaded next.
3757 * (If we don't do this, it's possible for the flush's final
3758 * write/commit (xid1) to be executed in parallel with a subsequent
3759 * getattr request (xid2). The getattr could return attributes
3760 * from *before* the write/commit completed but the stale attributes
3761 * would be preferred because of the xid ordering.)
3762 */
3763 if (!np->n_wrbusy && !np->n_numoutput) {
3764 np->n_flag &= ~NMODIFIED;
3765 NATTRINVALIDATE(np);
3766 nfs_get_xid(&np->n_xid);
3767 }
3768 } else {
3769 nfs_node_lock_force(np);
3770 }
3771
3772 FSDBG(526, np->n_flag, np->n_error, 0, 0);
3773 if (!ignore_writeerr && (np->n_flag & NWRITEERR)) {
3774 error = np->n_error;
3775 np->n_flag &= ~NWRITEERR;
3776 }
3777 nfs_node_unlock(np);
3778 done:
3779 lck_mtx_lock(&nfs_buf_mutex);
3780 flags = np->n_bflag;
3781 np->n_bflag &= ~(NBFLUSHINPROG | NBFLUSHWANT);
3782 lck_mtx_unlock(&nfs_buf_mutex);
3783 if (flags & NBFLUSHWANT) {
3784 wakeup(&np->n_bflag);
3785 }
3786 out:
3787 FSDBG_BOT(517, np, error, ignore_writeerr, 0);
3788 return error;
3789 }
3790
3791 /*
3792 * Flush out and invalidate all buffers associated with a vnode.
3793 * Called with the underlying object locked.
3794 */
3795 int
nfs_vinvalbuf_internal(nfsnode_t np,int flags,thread_t thd,kauth_cred_t cred,int slpflag,int slptimeo)3796 nfs_vinvalbuf_internal(
3797 nfsnode_t np,
3798 int flags,
3799 thread_t thd,
3800 kauth_cred_t cred,
3801 int slpflag,
3802 int slptimeo)
3803 {
3804 struct nfsbuf *bp;
3805 struct nfsbuflists blist;
3806 int list, error = 0;
3807
3808 if (flags & V_SAVE) {
3809 if ((error = nfs_flush(np, MNT_WAIT, thd, (flags & V_IGNORE_WRITEERR)))) {
3810 return error;
3811 }
3812 }
3813
3814 lck_mtx_lock(&nfs_buf_mutex);
3815 for (;;) {
3816 list = NBI_CLEAN;
3817 if (nfs_buf_iterprepare(np, &blist, list)) {
3818 list = NBI_DIRTY;
3819 if (nfs_buf_iterprepare(np, &blist, list)) {
3820 break;
3821 }
3822 }
3823 while ((bp = LIST_FIRST(&blist))) {
3824 LIST_REMOVE(bp, nb_vnbufs);
3825 if (list == NBI_CLEAN) {
3826 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
3827 } else {
3828 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
3829 }
3830 nfs_buf_refget(bp);
3831 while ((error = nfs_buf_acquire(bp, NBAC_REMOVE, slpflag, slptimeo))) {
3832 FSDBG(556, np, bp, NBOFF(bp), bp->nb_flags);
3833 if (error != EAGAIN) {
3834 FSDBG(554, np, bp, -1, error);
3835 nfs_buf_refrele(bp);
3836 nfs_buf_itercomplete(np, &blist, list);
3837 lck_mtx_unlock(&nfs_buf_mutex);
3838 return error;
3839 }
3840 }
3841 nfs_buf_refrele(bp);
3842 FSDBG(554, np, bp, NBOFF(bp), bp->nb_flags);
3843 lck_mtx_unlock(&nfs_buf_mutex);
3844 if ((flags & V_SAVE) && UBCINFOEXISTS(NFSTOV(np)) && bp->nb_np &&
3845 (NBOFF(bp) < (off_t)np->n_size)) {
3846 /* extra paranoia: make sure we're not */
3847 /* somehow leaving any dirty data around */
3848 nfsbufpgs pagemask;
3849 int mustwrite = 0;
3850 off_t end = (NBOFF(bp) + bp->nb_bufsize > (off_t)np->n_size) ?
3851 (np->n_size - NBOFF(bp)) : bp->nb_bufsize;
3852 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
3853 error = nfs_buf_upl_setup(bp);
3854 if (error == EINVAL) {
3855 /* vm object must no longer exist */
3856 /* hopefully we don't need to do */
3857 /* anything for this buffer */
3858 } else if (error) {
3859 printf("nfs_vinvalbuf_internal: upl setup failed %d\n", error);
3860 }
3861 NBPGS_ERASE(&bp->nb_valid);
3862 NBPGS_ERASE(&bp->nb_dirty);
3863 }
3864 nfs_buf_upl_check(bp);
3865 /* check for any dirty data before the EOF */
3866 if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < end)) {
3867 /* clip dirty range to EOF */
3868 if (bp->nb_dirtyend > end) {
3869 bp->nb_dirtyend = end;
3870 if (bp->nb_dirtyoff >= bp->nb_dirtyend) {
3871 bp->nb_dirtyoff = bp->nb_dirtyend = 0;
3872 }
3873 }
3874 if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < end)) {
3875 mustwrite++;
3876 }
3877 }
3878 nfs_buf_pgs_get_page_mask(&pagemask, round_page_64(end) / PAGE_SIZE);
3879 nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &bp->nb_dirty);
3880 if (nfs_buf_pgs_is_set(&bp->nb_dirty)) {
3881 mustwrite++;
3882 }
3883 /* also make sure we'll have a credential to do the write */
3884 if (mustwrite && !IS_VALID_CRED(bp->nb_wcred) && !IS_VALID_CRED(cred)) {
3885 printf("nfs_vinvalbuf_internal: found dirty buffer with no write creds\n");
3886 mustwrite = 0;
3887 }
3888 if (mustwrite) {
3889 FSDBG(554, np, bp, 0xd00dee, bp->nb_flags);
3890 if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
3891 panic("nfs_vinvalbuf_internal: dirty buffer without upl");
3892 }
3893 /* gotta write out dirty data before invalidating */
3894 /* (NB_STABLE indicates that data writes should be FILESYNC) */
3895 /* (NB_NOCACHE indicates buffer should be discarded) */
3896 CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL | NB_ASYNC));
3897 SET(bp->nb_flags, NB_STABLE | NB_NOCACHE);
3898 if (!IS_VALID_CRED(bp->nb_wcred)) {
3899 kauth_cred_ref(cred);
3900 bp->nb_wcred = cred;
3901 }
3902 error = nfs_buf_write(bp);
3903 // Note: bp has been released
3904 if (error) {
3905 FSDBG(554, bp, 0xd00dee, 0xbad, error);
3906 nfs_node_lock_force(np);
3907 if ((error != EINTR) && (error != ERESTART)) {
3908 np->n_error = error;
3909 np->n_flag |= NWRITEERR;
3910 }
3911 /*
3912 * There was a write error and we need to
3913 * invalidate attrs to sync with server.
3914 * (if this write was extending the file,
3915 * we may no longer know the correct size)
3916 */
3917 NATTRINVALIDATE(np);
3918 nfs_node_unlock(np);
3919 if ((error == EINTR) || (error == ERESTART)) {
3920 /*
3921 * Abort on EINTR. If we don't, we could
3922 * be stuck in this loop forever because
3923 * the buffer will continue to stay dirty.
3924 */
3925 lck_mtx_lock(&nfs_buf_mutex);
3926 nfs_buf_itercomplete(np, &blist, list);
3927 lck_mtx_unlock(&nfs_buf_mutex);
3928 return error;
3929 }
3930 error = 0;
3931 }
3932 lck_mtx_lock(&nfs_buf_mutex);
3933 continue;
3934 }
3935 }
3936 SET(bp->nb_flags, NB_INVAL);
3937 // hold off on FREEUPs until we're done here
3938 nfs_buf_release(bp, 0);
3939 lck_mtx_lock(&nfs_buf_mutex);
3940 }
3941 nfs_buf_itercomplete(np, &blist, list);
3942 }
3943 if (!LIST_EMPTY(&(np)->n_dirtyblkhd) || !LIST_EMPTY(&(np)->n_cleanblkhd)) {
3944 panic("nfs_vinvalbuf_internal: flush/inval failed");
3945 }
3946 lck_mtx_unlock(&nfs_buf_mutex);
3947 nfs_node_lock_force(np);
3948 if (!(flags & V_SAVE)) {
3949 np->n_flag &= ~NMODIFIED;
3950 }
3951 if (vnode_vtype(NFSTOV(np)) == VREG) {
3952 np->n_lastrahead = -1;
3953 }
3954 nfs_node_unlock(np);
3955 NFS_BUF_FREEUP();
3956 return 0;
3957 }
3958
3959
3960 /*
3961 * Flush and invalidate all dirty buffers. If another process is already
3962 * doing the flush, just wait for completion.
3963 */
3964 int
nfs_vinvalbuf1(vnode_t vp,int flags,vfs_context_t ctx,int intrflg)3965 nfs_vinvalbuf1(vnode_t vp, int flags, vfs_context_t ctx, int intrflg)
3966 {
3967 return nfs_vinvalbuf2(vp, flags, vfs_context_thread(ctx), vfs_context_ucred(ctx), intrflg);
3968 }
3969
3970 int
nfs_vinvalbuf2(vnode_t vp,int flags,thread_t thd,kauth_cred_t cred,int intrflg)3971 nfs_vinvalbuf2(vnode_t vp, int flags, thread_t thd, kauth_cred_t cred, int intrflg)
3972 {
3973 nfsnode_t np = VTONFS(vp);
3974 struct nfsmount *nmp = VTONMP(vp);
3975 int error, slpflag, slptimeo, nflags, retry = 0;
3976 int ubcflags = UBC_PUSHALL | UBC_SYNC | UBC_INVALIDATE;
3977 struct timespec ts = { .tv_sec = 2, .tv_nsec = 0 };
3978 off_t size;
3979
3980 FSDBG_TOP(554, np, flags, intrflg, 0);
3981
3982 if (nmp && !NMFLAG(nmp, INTR)) {
3983 intrflg = 0;
3984 }
3985 if (intrflg) {
3986 slpflag = PCATCH;
3987 slptimeo = 2 * hz;
3988 } else {
3989 slpflag = 0;
3990 slptimeo = 0;
3991 }
3992
3993 /* First wait for any other process doing a flush to complete. */
3994 lck_mtx_lock(&nfs_buf_mutex);
3995 while (np->n_bflag & NBINVALINPROG) {
3996 np->n_bflag |= NBINVALWANT;
3997 msleep(&np->n_bflag, &nfs_buf_mutex, slpflag, "nfs_vinvalbuf2", &ts);
3998 if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0))) {
3999 lck_mtx_unlock(&nfs_buf_mutex);
4000 return error;
4001 }
4002 if (np->n_bflag & NBINVALINPROG) {
4003 slpflag = 0;
4004 }
4005 }
4006 np->n_bflag |= NBINVALINPROG;
4007 lck_mtx_unlock(&nfs_buf_mutex);
4008
4009 /* Now, flush as required. */
4010 again:
4011 /* If the mount is gone no sense to try and write anything. and hang trying to do IO. */
4012 if (nfs_mount_gone(nmp)) {
4013 flags &= ~V_SAVE;
4014 }
4015
4016 error = nfs_vinvalbuf_internal(np, flags, thd, cred, slpflag, 0);
4017 while (error) {
4018 FSDBG(554, np, 0, 0, error);
4019 if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0))) {
4020 goto done;
4021 }
4022 error = nfs_vinvalbuf_internal(np, flags, thd, cred, 0, slptimeo);
4023 }
4024
4025 /* If the mount is gone no sense to try and write anything. and hang trying to do IO. */
4026 if (nfs_mount_gone(nmp)) {
4027 ubcflags &= ~UBC_PUSHALL;
4028 }
4029
4030 /* get the pages out of vm also */
4031 if (UBCINFOEXISTS(vp) && (size = ubc_getsize(vp))) {
4032 if ((error = ubc_msync(vp, 0, size, NULL, ubcflags))) {
4033 if (error == EINVAL) {
4034 panic("nfs_vinvalbuf2: ubc_msync failed!, error %d", error);
4035 }
4036 if (retry++ < 10) { /* retry invalidating a few times */
4037 if (retry > 1 || error == ENXIO) {
4038 ubcflags &= ~UBC_PUSHALL;
4039 }
4040 goto again;
4041 }
4042 /* give up */
4043 printf("nfs_vinvalbuf2: ubc_msync failed!, error %d\n", error);
4044 }
4045 }
4046 done:
4047 lck_mtx_lock(&nfs_buf_mutex);
4048 nflags = np->n_bflag;
4049 np->n_bflag &= ~(NBINVALINPROG | NBINVALWANT);
4050 lck_mtx_unlock(&nfs_buf_mutex);
4051 if (nflags & NBINVALWANT) {
4052 wakeup(&np->n_bflag);
4053 }
4054
4055 FSDBG_BOT(554, np, flags, intrflg, error);
4056 return error;
4057 }
4058
4059 /*
4060 * Wait for any busy buffers to complete.
4061 */
4062 void
nfs_wait_bufs(nfsnode_t np)4063 nfs_wait_bufs(nfsnode_t np)
4064 {
4065 struct nfsbuf *bp;
4066 struct nfsbuflists blist;
4067 int error = 0;
4068
4069 lck_mtx_lock(&nfs_buf_mutex);
4070 if (!nfs_buf_iterprepare(np, &blist, NBI_CLEAN)) {
4071 while ((bp = LIST_FIRST(&blist))) {
4072 LIST_REMOVE(bp, nb_vnbufs);
4073 LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
4074 nfs_buf_refget(bp);
4075 while ((error = nfs_buf_acquire(bp, 0, 0, 0))) {
4076 if (error != EAGAIN) {
4077 nfs_buf_refrele(bp);
4078 nfs_buf_itercomplete(np, &blist, NBI_CLEAN);
4079 lck_mtx_unlock(&nfs_buf_mutex);
4080 return;
4081 }
4082 }
4083 nfs_buf_refrele(bp);
4084 nfs_buf_drop(bp);
4085 }
4086 nfs_buf_itercomplete(np, &blist, NBI_CLEAN);
4087 }
4088 if (!nfs_buf_iterprepare(np, &blist, NBI_DIRTY)) {
4089 while ((bp = LIST_FIRST(&blist))) {
4090 LIST_REMOVE(bp, nb_vnbufs);
4091 LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
4092 nfs_buf_refget(bp);
4093 while ((error = nfs_buf_acquire(bp, 0, 0, 0))) {
4094 if (error != EAGAIN) {
4095 nfs_buf_refrele(bp);
4096 nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
4097 lck_mtx_unlock(&nfs_buf_mutex);
4098 return;
4099 }
4100 }
4101 nfs_buf_refrele(bp);
4102 nfs_buf_drop(bp);
4103 }
4104 nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
4105 }
4106 lck_mtx_unlock(&nfs_buf_mutex);
4107 }
4108
4109
4110 /*
4111 * Add an async I/O request to the mount's async I/O queue and make
4112 * sure that an nfsiod will service it.
4113 */
4114 void
nfs_asyncio_finish(struct nfsreq * req)4115 nfs_asyncio_finish(struct nfsreq *req)
4116 {
4117 struct nfsmount *nmp;
4118 struct nfsiod *niod;
4119 int started = 0;
4120
4121 FSDBG_TOP(552, nmp, 0, 0, 0);
4122 again:
4123 nmp = req->r_nmp;
4124
4125 if (nmp == NULL) {
4126 return;
4127 }
4128
4129 lck_mtx_lock(&nfsiod_mutex);
4130 niod = nmp->nm_niod;
4131
4132 /* grab an nfsiod if we don't have one already */
4133 if (!niod) {
4134 niod = TAILQ_FIRST(&nfsiodfree);
4135 if (niod) {
4136 TAILQ_REMOVE(&nfsiodfree, niod, niod_link);
4137 TAILQ_INSERT_TAIL(&nfsiodwork, niod, niod_link);
4138 niod->niod_nmp = nmp;
4139 } else if (((nfsiod_thread_count < NFSIOD_MAX) || (nfsiod_thread_count <= 0)) && (started < 4)) {
4140 /*
4141 * Try starting a new thread.
4142 * We may try a couple times if other callers
4143 * get the new threads before we do.
4144 */
4145 lck_mtx_unlock(&nfsiod_mutex);
4146 started++;
4147 if (!nfsiod_start()) {
4148 goto again;
4149 }
4150 lck_mtx_lock(&nfsiod_mutex);
4151 }
4152 }
4153
4154 /*
4155 * If we got here while being on the resendq we need to get off. This
4156 * happens when the timer fires and errors out requests from nfs_sigintr
4157 * or we receive a reply (UDP case) while being on the resend queue so
4158 * we're just finishing up and are not going to be resent.
4159 */
4160 lck_mtx_lock(&req->r_mtx);
4161 if (req->r_flags & R_RESENDQ) {
4162 lck_mtx_lock(&nmp->nm_lock);
4163 if ((req->r_flags & R_RESENDQ) && req->r_rchain.tqe_next != NFSREQNOLIST) {
4164 NFS_BIO_DBG("Proccessing async request on resendq. Removing");
4165 TAILQ_REMOVE(&nmp->nm_resendq, req, r_rchain);
4166 req->r_flags &= ~R_RESENDQ;
4167 req->r_rchain.tqe_next = NFSREQNOLIST;
4168 assert(req->r_refs > 1);
4169 /* Remove resendq reference */
4170 req->r_refs--;
4171 }
4172 lck_mtx_unlock(&nmp->nm_lock);
4173 }
4174 lck_mtx_unlock(&req->r_mtx);
4175
4176 if (req->r_achain.tqe_next == NFSREQNOLIST) {
4177 TAILQ_INSERT_TAIL(&nmp->nm_iodq, req, r_achain);
4178 }
4179
4180 /* If this mount doesn't already have an nfsiod working on it... */
4181 if (!nmp->nm_niod) {
4182 if (niod) { /* give it the nfsiod we just grabbed */
4183 nmp->nm_niod = niod;
4184 lck_mtx_unlock(&nfsiod_mutex);
4185 wakeup(niod);
4186 } else if (nfsiod_thread_count > 0) {
4187 /* just queue it up on nfsiod mounts queue if needed */
4188 if (nmp->nm_iodlink.tqe_next == NFSNOLIST) {
4189 TAILQ_INSERT_TAIL(&nfsiodmounts, nmp, nm_iodlink);
4190 }
4191 lck_mtx_unlock(&nfsiod_mutex);
4192 } else {
4193 printf("nfs_asyncio(): no nfsiods? %d %d (%d)\n", nfsiod_thread_count, NFSIOD_MAX, started);
4194 lck_mtx_unlock(&nfsiod_mutex);
4195 /* we have no other option but to be persistent */
4196 started = 0;
4197 goto again;
4198 }
4199 } else {
4200 lck_mtx_unlock(&nfsiod_mutex);
4201 }
4202
4203 FSDBG_BOT(552, nmp, 0, 0, 0);
4204 }
4205
4206 /*
4207 * queue up async I/O request for resend
4208 * Must be called with req->r_mtx locked.
4209 */
4210 void
nfs_asyncio_resend(struct nfsreq * req)4211 nfs_asyncio_resend(struct nfsreq *req)
4212 {
4213 struct nfsmount *nmp = req->r_nmp;
4214
4215 if (nfs_mount_gone(nmp)) {
4216 return;
4217 }
4218
4219 #if CONFIG_NFS_GSS
4220 nfs_gss_clnt_rpcdone(req);
4221 #endif
4222 lck_mtx_lock(&nmp->nm_lock);
4223 if (!(req->r_flags & R_RESENDQ)) {
4224 TAILQ_INSERT_TAIL(&nmp->nm_resendq, req, r_rchain);
4225 req->r_flags |= R_RESENDQ;
4226 /*
4227 * We take a reference on this request so that it can't be
4228 * destroyed while a resend is queued or in progress.
4229 */
4230 nfs_request_ref(req, 1);
4231 }
4232 nfs_mount_sock_thread_wake(nmp);
4233 lck_mtx_unlock(&nmp->nm_lock);
4234 }
4235
4236 /*
4237 * Read directory data into a buffer.
4238 *
4239 * Buffer will be filled (unless EOF is hit).
4240 * Buffers after this one may also be completely/partially filled.
4241 */
4242 int
nfs_buf_readdir(struct nfsbuf * bp,vfs_context_t ctx)4243 nfs_buf_readdir(struct nfsbuf *bp, vfs_context_t ctx)
4244 {
4245 nfsnode_t np = bp->nb_np;
4246 struct nfsmount *nmp = NFSTONMP(np);
4247 int error = 0;
4248
4249 if (nfs_mount_gone(nmp)) {
4250 return ENXIO;
4251 }
4252
4253 if (nmp->nm_vers < NFS_VER4) {
4254 error = nfs3_readdir_rpc(np, bp, ctx);
4255 }
4256 #if CONFIG_NFS4
4257 else {
4258 error = nfs4_readdir_rpc(np, bp, ctx);
4259 }
4260 #endif
4261 if (error && (error != NFSERR_DIRBUFDROPPED)) {
4262 SET(bp->nb_flags, NB_ERROR);
4263 bp->nb_error = error;
4264 }
4265 return error;
4266 }
4267
4268 #endif /* CONFIG_NFS_CLIENT */
4269