1 /*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1998 Apple Computer, Inc. All rights reserved.
29 *
30 * File: bsd/kern/kern_symfile.c
31 *
32 * HISTORY
33 */
34
35 #include <mach/vm_param.h>
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/signalvar.h>
40 #include <sys/resourcevar.h>
41 #include <sys/namei.h>
42 #include <sys/vnode_internal.h>
43 #include <sys/proc_internal.h>
44 #include <sys/kauth.h>
45 #include <sys/timeb.h>
46 #include <sys/times.h>
47 #include <sys/acct.h>
48 #include <sys/file_internal.h>
49 #include <sys/uio.h>
50 #include <sys/kernel.h>
51 #include <sys/stat.h>
52 #include <sys/disk.h>
53 #include <sys/conf.h>
54 #include <sys/content_protection.h>
55 #include <sys/fsctl.h>
56
57 #include <mach-o/loader.h>
58 #include <mach-o/nlist.h>
59
60 #include <kern/kalloc.h>
61 #include <vm/vm_kern.h>
62 #include <pexpert/pexpert.h>
63 #include <IOKit/IOPolledInterface.h>
64
65 #define HIBERNATE_MIN_PHYSICAL_LBA_512 (34)
66 #define HIBERNATE_MIN_PHYSICAL_LBA_4096 (6)
67 #define HIBERNATE_MIN_FILE_SIZE (1024*1024)
68
69 /* This function is called from kern_sysctl in the current process context;
70 * it is exported with the System6.0.exports, but this appears to be a legacy
71 * export, as there are no internal consumers.
72 */
73 int
74 get_kernel_symfile(__unused proc_t p, __unused char const **symfile);
75 int
get_kernel_symfile(__unused proc_t p,__unused char const ** symfile)76 get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
77 {
78 return KERN_FAILURE;
79 }
80
81 struct kern_direct_file_io_ref_t {
82 vfs_context_t ctx;
83 struct vnode * vp;
84 char * name;
85 size_t namesize;
86 dev_t device;
87 uint32_t blksize;
88 off_t filelength;
89 char cf;
90 char pinned;
91 char frozen;
92 char wbcranged;
93 };
94
95
96 static int
file_ioctl(void * p1,void * p2,u_long theIoctl,caddr_t result)97 file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result)
98 {
99 dev_t device = *(dev_t*) p1;
100
101 return (*bdevsw[major(device)].d_ioctl)
102 (device, theIoctl, result, S_IFBLK, p2);
103 }
104
105 static int
device_ioctl(void * p1,__unused void * p2,u_long theIoctl,caddr_t result)106 device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result)
107 {
108 return VNOP_IOCTL(p1, theIoctl, result, 0, p2);
109 }
110
111 static int
kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref,u_long theIoctl,off_t offset,off_t end)112 kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end)
113 {
114 int error = 0;
115 int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
116 void * p1;
117 void * p2;
118 uint64_t fileblk = 0;
119 size_t filechunk = 0;
120 dk_extent_t extent;
121 dk_unmap_t unmap;
122 _dk_cs_pin_t pin;
123
124 bzero(&extent, sizeof(dk_extent_t));
125 bzero(&unmap, sizeof(dk_unmap_t));
126 bzero(&pin, sizeof(pin));
127 if (ref->vp->v_type == VREG) {
128 p1 = &ref->device;
129 p2 = kernproc;
130 do_ioctl = &file_ioctl;
131 } else {
132 /* Partition. */
133 p1 = ref->vp;
134 p2 = ref->ctx;
135 do_ioctl = &device_ioctl;
136 }
137
138 if (_DKIOCCSPINEXTENT == theIoctl) {
139 /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */
140 pin.cp_extent.length = end;
141 pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE;
142 (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin);
143 } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
144 /* Tell CS hibernation is done, so it can stop blocking overlapping writes */
145 pin.cp_flags = _DKIOCCSPINDISCARDDENYLIST;
146 (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin);
147 }
148
149 for (; offset < end; offset += filechunk) {
150 if (ref->vp->v_type == VREG) {
151 daddr64_t blkno;
152 filechunk = 1 * 1024 * 1024 * 1024;
153 if (filechunk > (size_t)(end - offset)) {
154 filechunk = (size_t)(end - offset);
155 }
156 error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno,
157 &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
158 if (error) {
159 break;
160 }
161 if (-1LL == blkno) {
162 continue;
163 }
164 fileblk = blkno * ref->blksize;
165 } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
166 fileblk = offset;
167 filechunk = (unsigned long)((ref->filelength > ULONG_MAX) ? ULONG_MAX: ref->filelength);
168 }
169
170 if (DKIOCUNMAP == theIoctl) {
171 extent.offset = fileblk;
172 extent.length = filechunk;
173 unmap.extents = &extent;
174 unmap.extentsCount = 1;
175 error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap);
176 // printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length);
177 } else if (_DKIOCCSPINEXTENT == theIoctl) {
178 pin.cp_extent.offset = fileblk;
179 pin.cp_extent.length = filechunk;
180 pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
181 error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
182 if (error && (ENOTTY != error)) {
183 printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
184 }
185 } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
186 pin.cp_extent.offset = fileblk;
187 pin.cp_extent.length = filechunk;
188 pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
189 error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
190 if (error && (ENOTTY != error)) {
191 printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
192 }
193 } else {
194 error = EINVAL;
195 }
196
197 if (error) {
198 break;
199 }
200 }
201 return error;
202 }
203
204 extern uint32_t freespace_mb(vnode_t vp);
205
206 struct kern_direct_file_io_ref_t *
kern_open_file_for_direct_io(const char * name,uint32_t iflags,kern_get_file_extents_callback_t callback,void * callback_ref,off_t set_file_size_min,off_t set_file_size_max,off_t fs_free_size,off_t write_file_offset,void * write_file_addr,size_t write_file_len,dev_t * partition_device_result,dev_t * image_device_result,uint64_t * partitionbase_result,uint64_t * maxiocount_result,uint32_t * oflags)207 kern_open_file_for_direct_io(const char * name,
208 uint32_t iflags,
209 kern_get_file_extents_callback_t callback,
210 void * callback_ref,
211 off_t set_file_size_min,
212 off_t set_file_size_max,
213 off_t fs_free_size,
214 off_t write_file_offset,
215 void * write_file_addr,
216 size_t write_file_len,
217 dev_t * partition_device_result,
218 dev_t * image_device_result,
219 uint64_t * partitionbase_result,
220 uint64_t * maxiocount_result,
221 uint32_t * oflags)
222 {
223 struct kern_direct_file_io_ref_t * ref;
224
225 proc_t p;
226 struct vnode_attr va;
227 dk_apfs_wbc_range_t wbc_range;
228 int error;
229 off_t f_offset;
230 uint64_t fileblk = 0;
231 size_t filechunk = 0;
232 uint64_t physoffset, minoffset;
233 dev_t device;
234 dev_t target = 0;
235 int isssd = 0;
236 uint32_t flags = 0;
237 uint32_t blksize;
238 off_t maxiocount, count, segcount, wbctotal, set_file_size;
239 boolean_t locked = FALSE;
240 int fmode;
241 mode_t cmode;
242 struct nameidata nd;
243 u_int32_t ndflags;
244 off_t mpFree;
245
246 wbc_range.count = 0;
247
248 int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
249 do_ioctl = NULL;
250 void * p1 = NULL;
251 void * p2 = NULL;
252
253 error = EFAULT;
254
255 ref = kalloc_type(struct kern_direct_file_io_ref_t,
256 Z_WAITOK | Z_ZERO | Z_NOFAIL);
257
258 p = kernproc;
259 ref->ctx = vfs_context_kernel();
260 ref->namesize = strlen(name) + 1;
261 ref->name = kalloc_data(ref->namesize, Z_WAITOK | Z_NOFAIL);
262 strlcpy(ref->name, name, ref->namesize);
263
264 fmode = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE;
265 cmode = S_IRUSR | S_IWUSR;
266 ndflags = NOFOLLOW;
267 NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(ref->name), ref->ctx);
268 VATTR_INIT(&va);
269 VATTR_SET(&va, va_mode, cmode);
270 VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
271 VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
272 if ((error = vn_open_auth(&nd, &fmode, &va, NULLVP))) {
273 kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
274 goto out;
275 }
276
277 ref->vp = nd.ni_vp;
278 if (ref->vp->v_type == VREG) {
279 vnode_lock_spin(ref->vp);
280 SET(ref->vp->v_flag, VSWAP);
281 vnode_unlock(ref->vp);
282 }
283
284 if (write_file_addr && write_file_len) {
285 if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
286 kprintf("kern_write_file() failed with error: %d\n", error);
287 goto out;
288 }
289 }
290
291 VATTR_INIT(&va);
292 VATTR_WANTED(&va, va_rdev);
293 VATTR_WANTED(&va, va_fsid);
294 VATTR_WANTED(&va, va_devid);
295 VATTR_WANTED(&va, va_data_size);
296 VATTR_WANTED(&va, va_data_alloc);
297 VATTR_WANTED(&va, va_nlink);
298 error = EFAULT;
299 if (vnode_getattr(ref->vp, &va, ref->ctx)) {
300 goto out;
301 }
302
303 wbctotal = 0;
304 mpFree = freespace_mb(ref->vp);
305 mpFree <<= 20;
306 printf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n",
307 ref->name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);
308
309 if (ref->vp->v_type == VREG) {
310 /* Don't dump files with links. */
311 if (va.va_nlink != 1) {
312 goto out;
313 }
314
315 /* Don't dump on fs without backing device. */
316 if (!VATTR_IS_SUPPORTED(&va, va_devid)) {
317 kprintf("kern_direct_file(%s): Not backed by block device.\n", ref->name);
318 error = ENODEV;
319 goto out;
320 }
321 device = va.va_devid;
322 ref->filelength = va.va_data_size;
323
324 p1 = &device;
325 p2 = p;
326 do_ioctl = &file_ioctl;
327
328 if (kIOPolledFileHibernate & iflags) {
329 error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range);
330 ref->wbcranged = (error == 0);
331 }
332 if (ref->wbcranged) {
333 uint32_t idx;
334 assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0])));
335 for (idx = 0; idx < wbc_range.count; idx++) {
336 wbctotal += wbc_range.extents[idx].length;
337 }
338 kprintf("kern_direct_file(%s): wbc %qd\n", ref->name, wbctotal);
339 if (wbctotal) {
340 target = wbc_range.dev;
341 }
342 }
343
344 if ((set_file_size = set_file_size_max)) {
345 // set file size
346 if (wbctotal) {
347 // only hibernate
348 if (wbctotal >= set_file_size_min) {
349 set_file_size_min = HIBERNATE_MIN_FILE_SIZE;
350 } else {
351 set_file_size_min -= wbctotal;
352 if (set_file_size_min < HIBERNATE_MIN_FILE_SIZE) {
353 set_file_size_min = HIBERNATE_MIN_FILE_SIZE;
354 }
355 }
356 set_file_size = set_file_size_min;
357 }
358 if (fs_free_size) {
359 mpFree += va.va_data_alloc;
360 if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) {
361 set_file_size = mpFree - fs_free_size;
362 if (0 == set_file_size_min) {
363 // passing zero for set_file_size_min (coredumps)
364 // means caller only accepts set_file_size_max
365 error = ENOSPC;
366 goto out;
367 }
368 // if set_file_size_min is passed (hibernation),
369 // it does not check free space on disk
370 }
371 }
372 while (TRUE) {
373 if (set_file_size < set_file_size_min) {
374 set_file_size = set_file_size_min;
375 }
376 if (set_file_size < set_file_size_max) {
377 printf("kern_direct_file(%s): using reduced size %qd\n",
378 ref->name, set_file_size);
379 }
380 error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
381 if ((ENOSPC == error) && set_file_size_min && (set_file_size > set_file_size_min) && (set_file_size > fs_free_size)) {
382 set_file_size -= fs_free_size;
383 continue;
384 }
385 break;
386 }
387 if (error) {
388 goto out;
389 }
390 ref->filelength = set_file_size;
391 }
392 } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
393 /* Partition. */
394 device = va.va_rdev;
395
396 p1 = ref->vp;
397 p2 = ref->ctx;
398 do_ioctl = &device_ioctl;
399 } else {
400 /* Don't dump to non-regular files. */
401 error = EFAULT;
402 goto out;
403 }
404 ref->device = device;
405
406 // probe for CF
407 dk_corestorage_info_t cs_info;
408 memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
409 error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
410 ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);
411
412 // get block size
413
414 error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
415 if (error) {
416 goto out;
417 }
418
419 if (ref->blksize == 4096) {
420 minoffset = HIBERNATE_MIN_PHYSICAL_LBA_4096 * ref->blksize;
421 } else {
422 minoffset = HIBERNATE_MIN_PHYSICAL_LBA_512 * ref->blksize;
423 }
424
425 if (ref->vp->v_type != VREG) {
426 error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
427 if (error) {
428 goto out;
429 }
430 ref->filelength = fileblk * ref->blksize;
431 }
432
433 // pin logical extents, CS version
434
435 error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
436 if (error && (ENOTTY != error)) {
437 goto out;
438 }
439 ref->pinned = (error == 0);
440
441 // pin logical extents, apfs version
442
443 error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx);
444 if (error && (ENOTTY != error)) {
445 goto out;
446 }
447 ref->frozen = (error == 0);
448
449 // generate the block list
450
451 error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
452 if (error) {
453 goto out;
454 }
455 locked = TRUE;
456
457 f_offset = 0;
458 for (; f_offset < ref->filelength; f_offset += filechunk) {
459 if (ref->vp->v_type == VREG) {
460 filechunk = 1 * 1024 * 1024 * 1024;
461 daddr64_t blkno;
462
463 error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
464 &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
465 if (error) {
466 goto out;
467 }
468 if (-1LL == blkno) {
469 continue;
470 }
471 fileblk = blkno * ref->blksize;
472 } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
473 fileblk = f_offset;
474 filechunk = f_offset ? 0 : (unsigned long)ref->filelength;
475 }
476
477 physoffset = 0;
478 while (physoffset < filechunk) {
479 dk_physical_extent_t getphysreq;
480 bzero(&getphysreq, sizeof(getphysreq));
481
482 getphysreq.offset = fileblk + physoffset;
483 getphysreq.length = (filechunk - physoffset);
484 error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
485 if (error) {
486 goto out;
487 }
488 if (!target) {
489 target = getphysreq.dev;
490 } else if (target != getphysreq.dev) {
491 error = ENOTSUP;
492 goto out;
493 }
494
495 assert(getphysreq.offset >= minoffset);
496
497 #if HIBFRAGMENT
498 uint64_t rev;
499 for (rev = 4096; rev <= getphysreq.length; rev += 4096) {
500 callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
501 }
502 #else
503 callback(callback_ref, getphysreq.offset, getphysreq.length);
504 #endif
505 physoffset += getphysreq.length;
506 }
507 }
508 if (ref->wbcranged) {
509 uint32_t idx;
510 for (idx = 0; idx < wbc_range.count; idx++) {
511 assert(wbc_range.extents[idx].offset >= minoffset);
512 callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length);
513 }
514 }
515 callback(callback_ref, 0ULL, 0ULL);
516
517 if (ref->vp->v_type == VREG) {
518 p1 = ⌖
519 } else {
520 p1 = ⌖
521 p2 = p;
522 do_ioctl = &file_ioctl;
523 }
524
525 // get partition base
526
527 if (partitionbase_result) {
528 error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
529 if (error) {
530 goto out;
531 }
532 }
533
534 // get block size & constraints
535
536 error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
537 if (error) {
538 goto out;
539 }
540
541 maxiocount = 1 * 1024 * 1024 * 1024;
542
543 error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
544 if (error) {
545 count = 0;
546 }
547 count *= blksize;
548 if (count && (count < maxiocount)) {
549 maxiocount = count;
550 }
551
552 error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
553 if (error) {
554 count = 0;
555 }
556 count *= blksize;
557 if (count && (count < maxiocount)) {
558 maxiocount = count;
559 }
560
561 error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
562 if (error) {
563 count = 0;
564 }
565 if (count && (count < maxiocount)) {
566 maxiocount = count;
567 }
568
569 error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
570 if (error) {
571 count = 0;
572 }
573 if (count && (count < maxiocount)) {
574 maxiocount = count;
575 }
576
577 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
578 if (!error) {
579 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
580 }
581 if (error) {
582 count = segcount = 0;
583 }
584 count *= segcount;
585 if (count && (count < maxiocount)) {
586 maxiocount = count;
587 }
588
589 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
590 if (!error) {
591 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
592 }
593 if (error) {
594 count = segcount = 0;
595 }
596 count *= segcount;
597 if (count && (count < maxiocount)) {
598 maxiocount = count;
599 }
600
601 kprintf("max io 0x%qx bytes\n", maxiocount);
602 if (maxiocount_result) {
603 *maxiocount_result = maxiocount;
604 }
605
606 error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
607 if (!error && isssd) {
608 flags |= kIOPolledFileSSD;
609 }
610
611 if (partition_device_result) {
612 *partition_device_result = device;
613 }
614 if (image_device_result) {
615 *image_device_result = target;
616 }
617 if (oflags) {
618 *oflags = flags;
619 }
620
621 if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
622 vnode_close(ref->vp, FWRITE, ref->ctx);
623 ref->vp = NULLVP;
624 ref->ctx = NULL;
625 }
626
627 out:
628 printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);
629
630
631 if (error && locked) {
632 p1 = &device;
633 if (do_ioctl) {
634 (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
635 }
636 }
637
638 if (error && ref) {
639 if (ref->vp) {
640 (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
641
642 if (ref->frozen) {
643 (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
644 }
645 if (ref->wbcranged) {
646 if (do_ioctl) {
647 (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
648 }
649 }
650 vnode_close(ref->vp, FWRITE, ref->ctx);
651 ref->vp = NULLVP;
652 }
653 ref->ctx = NULL;
654 if (ref->name) {
655 kfree_data(ref->name, ref->namesize);
656 ref->name = NULL;
657 }
658 kfree_type(struct kern_direct_file_io_ref_t, ref);
659 ref = NULL;
660 }
661
662 return ref;
663 }
664
665 int
kern_write_file(struct kern_direct_file_io_ref_t * ref,off_t offset,void * addr,size_t len,int ioflag)666 kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
667 {
668 assert(len <= INT32_MAX);
669 return vn_rdwr(UIO_WRITE, ref->vp,
670 addr, (int)len, offset,
671 UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
672 vfs_context_ucred(ref->ctx), (int *) 0,
673 vfs_context_proc(ref->ctx));
674 }
675
676 int
kern_read_file(struct kern_direct_file_io_ref_t * ref,off_t offset,void * addr,size_t len,int ioflag)677 kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
678 {
679 assert(len <= INT32_MAX);
680 return vn_rdwr(UIO_READ, ref->vp,
681 addr, (int)len, offset,
682 UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
683 vfs_context_ucred(ref->ctx), (int *) 0,
684 vfs_context_proc(ref->ctx));
685 }
686
687
688 struct mount *
kern_file_mount(struct kern_direct_file_io_ref_t * ref)689 kern_file_mount(struct kern_direct_file_io_ref_t * ref)
690 {
691 return ref->vp->v_mount;
692 }
693
694 void
kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,off_t write_offset,void * addr,size_t write_length,off_t discard_offset,off_t discard_end,off_t set_file_size,bool unlink)695 kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
696 off_t write_offset, void * addr, size_t write_length,
697 off_t discard_offset, off_t discard_end, off_t set_file_size, bool unlink)
698 {
699 int error;
700 printf("kern_close_file_for_direct_io(%p) %qd\n", ref, set_file_size);
701
702 if (!ref) {
703 return;
704 }
705
706 if (ref->vp) {
707 int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
708 void * p1;
709 void * p2;
710
711 discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1));
712 discard_end = ((discard_end) & ~(((off_t) ref->blksize) - 1));
713
714 if (ref->vp->v_type == VREG) {
715 p1 = &ref->device;
716 p2 = kernproc;
717 do_ioctl = &file_ioctl;
718 } else {
719 /* Partition. */
720 p1 = ref->vp;
721 p2 = ref->ctx;
722 do_ioctl = &device_ioctl;
723 }
724 (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
725
726 //XXX If unmapping extents then don't also need to unpin; except ...
727 //XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
728 //XXX unmap is subset, so save extra walk over file extents (and the risk
729 //XXX that CF drain starts) vs leaving partial units pinned to SSD
730 //XXX (until whatever was sharing also unmaps). Err on cleaning up fully.
731 boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset);
732 boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);
733
734 (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);
735
736 if (will_unmap) {
737 (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
738 }
739
740 if (ref->frozen) {
741 (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
742 }
743 if (ref->wbcranged) {
744 (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
745 }
746
747 if (addr && write_length) {
748 (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
749 }
750 if (set_file_size) {
751 error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
752 }
753 error = vnode_close(ref->vp, FWRITE, ref->ctx);
754
755 ref->vp = NULLVP;
756 kprintf("vnode_close(%d)\n", error);
757
758
759 if (unlink) {
760 int unlink1(vfs_context_t, vnode_t, user_addr_t, enum uio_seg, int);
761
762 error = unlink1(ref->ctx, NULLVP, CAST_USER_ADDR_T(ref->name), UIO_SYSSPACE, 0);
763 kprintf("%s: unlink1(%d)\n", __func__, error);
764 }
765 }
766
767 ref->ctx = NULL;
768
769 kfree_data(ref->name, ref->namesize);
770 ref->name = NULL;
771
772 kfree_type(struct kern_direct_file_io_ref_t, ref);
773 }
774