1 /*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1982, 1986, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
67 */
68
69 #include <machine/atomic.h>
70
71 #include <sys/param.h>
72 #include <sys/systm.h>
73 #include <sys/proc_internal.h>
74 #include <sys/malloc.h>
75 #include <sys/queue.h>
76 #include <vm/pmap.h>
77 #include <sys/uio_internal.h>
78 #include <kern/kalloc.h>
79
80 #include <kdebug.h>
81
82 #include <sys/kdebug.h>
83 #define DBG_UIO_COPYOUT 16
84 #define DBG_UIO_COPYIN 17
85
86 #if DEBUG
87 #include <kern/simple_lock.h>
88
89 static uint32_t uio_t_count = 0;
90 #endif /* DEBUG */
91
92 #define IS_VALID_UIO_SEGFLG(segflg) \
93 ( (1 << segflg) & (UIOF_USERSPACE | \
94 UIOF_SYSSPACE | \
95 UIOF_USERSPACE32 | \
96 UIOF_USERSPACE64 | \
97 UIOF_SYSSPACE32 | \
98 UIOF_USERISPACE | \
99 UIOF_PHYS_USERSPACE | \
100 UIOF_PHYS_SYSSPACE | \
101 UIOF_USERISPACE32 | \
102 UIOF_PHYS_USERSPACE32 | \
103 UIOF_USERISPACE64 | \
104 UIOF_PHYS_USERSPACE64))
105
106 #define IS_SYS_OR_PHYS_SPACE_SEGFLG(segflg) \
107 ( (1 << segflg) & (UIOF_SYSSPACE | \
108 UIOF_PHYS_SYSSPACE | \
109 UIOF_SYSSPACE32 | \
110 UIOF_PHYS_USERSPACE | \
111 UIOF_PHYS_SYSSPACE | \
112 UIOF_PHYS_USERSPACE64 | \
113 UIOF_PHYS_USERSPACE32))
114
115 #define IS_PURE_USER_SPACE_SEGFLG(segflg) \
116 ( (1 << segflg) & (UIOF_USERSPACE | \
117 UIOF_USERSPACE32 | \
118 UIOF_USERSPACE64 | \
119 UIOF_USERISPACE | \
120 UIOF_USERISPACE32 | \
121 UIOF_USERISPACE64))
122
123 #define IS_SYS_SPACE_SEGFLG(segflg) \
124 ( (1 << segflg) & (UIOF_SYSSPACE | \
125 UIOF_SYSSPACE32))
126
127 #define IS_PHYS_USER_SPACE_SEGFLG(segflg) \
128 ( (1 << segflg) & (UIOF_PHYS_USERSPACE | \
129 UIOF_PHYS_USERSPACE64 | \
130 UIOF_PHYS_USERSPACE32))
131
132 #define IS_PHYS_SYS_SPACE_SEGFLG(segflg) \
133 ( (1 << segflg) & (UIOF_PHYS_SYSSPACE))
134
135 static void uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count);
136 static void uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count);
137 static user_size_t uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio);
138 static user_size_t uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio);
139
140 #if __has_feature(ptrauth_calls)
141 __attribute__((always_inline))
142 static u_int64_t
blend_iov_components(const struct kern_iovec * kiovp)143 blend_iov_components(const struct kern_iovec *kiovp)
144 {
145 return ptrauth_blend_discriminator(
146 (void *)((u_int64_t)&kiovp->iov_base ^ kiovp->iov_len),
147 ptrauth_string_discriminator("kiovp"));
148 }
149 #endif
150
151 __attribute__((always_inline))
152 static u_int64_t
kiovp_get_base(const struct kern_iovec * kiovp)153 kiovp_get_base(const struct kern_iovec *kiovp)
154 {
155 #if __has_feature(ptrauth_calls)
156 if (kiovp->iov_base == 0) {
157 return 0;
158 } else {
159 return (u_int64_t)ptrauth_auth_data((void *)kiovp->iov_base,
160 ptrauth_key_process_independent_data,
161 blend_iov_components(kiovp));
162 }
163 #else
164 return kiovp->iov_base;
165 #endif
166 }
167
168 __attribute__((always_inline))
169 static void
kiovp_set_base(struct kern_iovec * kiovp,u_int64_t addr)170 kiovp_set_base(struct kern_iovec *kiovp, u_int64_t addr)
171 {
172 #if __has_feature(ptrauth_calls)
173 if (addr == 0) {
174 kiovp->iov_base = 0;
175 } else {
176 kiovp->iov_base = (u_int64_t)ptrauth_sign_unauthenticated(
177 (void *)addr, ptrauth_key_process_independent_data,
178 blend_iov_components(kiovp));
179 }
180 #else
181 kiovp->iov_base = addr;
182 #endif
183 }
184
185 static struct kern_iovec *
uio_kiovp(uio_t uio)186 uio_kiovp(uio_t uio)
187 {
188 #if DEBUG
189 if (__improbable(!UIO_IS_SYS_SPACE(uio))) {
190 panic("%s: uio is not sys space", __func__);
191 }
192 #endif
193
194 return (struct kern_iovec *)uio->uio_iovs;
195 }
196
197 static struct user_iovec *
uio_uiovp(uio_t uio)198 uio_uiovp(uio_t uio)
199 {
200 return (struct user_iovec *)uio->uio_iovs;
201 }
202
203 static void *
uio_advance_user(uio_t uio)204 uio_advance_user(uio_t uio)
205 {
206 uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct user_iovec));
207
208 return uio->uio_iovs;
209 }
210
211 static void *
uio_advance_sys(uio_t uio)212 uio_advance_sys(uio_t uio)
213 {
214 uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct kern_iovec));
215
216 return uio->uio_iovs;
217 }
218
219 /*
220 * Returns: 0 Success
221 * uiomove64:EFAULT
222 *
223 * Notes: The first argument should be a caddr_t, but const poisoning
224 * for typedef'ed types doesn't work in gcc.
225 */
226 int
uiomove(const char * __counted_by (n)cp,int n,uio_t uio)227 uiomove(const char *__counted_by(n) cp, int n, uio_t uio)
228 {
229 return uiomove64((const addr64_t)(uintptr_t)cp, n, uio);
230 }
231
232 /*
233 * Returns: 0 Success
234 * EFAULT
235 * copyout:EFAULT
236 * copyin:EFAULT
237 * copywithin:EFAULT
238 * copypv:EFAULT
239 */
240 int
uiomove64(const addr64_t c_cp __sized_by (n),int n,struct uio * uio)241 uiomove64(const addr64_t c_cp __sized_by(n), int n, struct uio *uio)
242 {
243 if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) {
244 if (uio->uio_rw == UIO_READ) {
245 return uio_copyout_user((const char *)c_cp, n, uio);
246 } else {
247 return uio_copyin_user((const char *)c_cp, n, uio);
248 }
249 } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
250 if (uio->uio_rw == UIO_READ) {
251 return uio_copyout_sys((const char *)c_cp, n, uio);
252 } else {
253 return uio_copyin_sys((const char *)c_cp, n, uio);
254 }
255 } else if (IS_PHYS_USER_SPACE_SEGFLG(uio->uio_segflg)) {
256 if (uio->uio_rw == UIO_READ) {
257 return uio_copyout_phys_user((const char *)c_cp, n, uio);
258 } else {
259 return uio_copyin_phys_user((const char *)c_cp, n, uio);
260 }
261 } else if (IS_PHYS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
262 if (uio->uio_rw == UIO_READ) {
263 return uio_copyout_phys_sys((const char *)c_cp, n, uio);
264 } else {
265 return uio_copyin_phys_sys((const char *)c_cp, n, uio);
266 }
267 } else {
268 return EINVAL;
269 }
270 }
271
272 int
uio_copyout_user(const char * c_cp __sized_by (n),int n,uio_t uio)273 uio_copyout_user(const char *c_cp __sized_by(n), int n, uio_t uio)
274 {
275 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
276
277 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
278 struct user_iovec *uiovp;
279 uint64_t acnt;
280 int error;
281
282 uio_update_user(uio, 0);
283 acnt = uio_curriovlen_user(uio);
284 if (acnt == 0) {
285 continue;
286 }
287 if (n > 0 && acnt > (uint64_t)n) {
288 acnt = n;
289 }
290
291 uiovp = uio_uiovp(uio);
292
293 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
294 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0);
295
296 error = copyout(CAST_DOWN(caddr_t, cp), uiovp->iov_base, (size_t)acnt);
297
298 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
299 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0);
300
301 if (error) {
302 return error;
303 }
304
305 uio_update_user(uio, (user_size_t)acnt);
306 cp += acnt;
307 n -= acnt;
308 }
309 return 0;
310 }
311
312 int
uio_copyin_user(const char * c_cp __sized_by (n),int n,uio_t uio)313 uio_copyin_user(const char *c_cp __sized_by(n), int n, uio_t uio)
314 {
315 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
316
317 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
318 struct user_iovec *uiovp;
319 uint64_t acnt;
320 int error;
321
322 uio_update_user(uio, 0);
323 acnt = uio_curriovlen_user(uio);
324 if (acnt == 0) {
325 continue;
326 }
327 if (n > 0 && acnt > (uint64_t)n) {
328 acnt = n;
329 }
330
331 uiovp = uio_uiovp(uio);
332
333 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
334 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0);
335
336 error = copyin(uiovp->iov_base, CAST_DOWN(caddr_t, cp), (size_t)acnt);
337
338 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
339 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0);
340
341 if (error) {
342 return error;
343 }
344
345 uio_update_user(uio, (user_size_t)acnt);
346 cp += acnt;
347 n -= acnt;
348 }
349 return 0;
350 }
351
352 int
uio_copyout_sys(const char * c_cp __sized_by (n),int n,uio_t uio)353 uio_copyout_sys(const char *c_cp __sized_by(n), int n, uio_t uio)
354 {
355 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
356
357 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
358 struct kern_iovec *kiovp;
359 uint64_t acnt;
360
361 uio_update_sys(uio, 0);
362 acnt = uio_curriovlen_sys(uio);
363 if (acnt == 0) {
364 continue;
365 }
366 if (n > 0 && acnt > (uint64_t)n) {
367 acnt = n;
368 }
369
370 kiovp = uio_kiovp(uio);
371
372 copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, kiovp_get_base(kiovp)),
373 (size_t)acnt);
374
375 uio_update_sys(uio, (user_size_t)acnt);
376 cp += acnt;
377 n -= acnt;
378 }
379 return 0;
380 }
381
382 int
uio_copyin_sys(const char * c_cp __sized_by (n),int n,uio_t uio)383 uio_copyin_sys(const char *c_cp __sized_by(n), int n, uio_t uio)
384 {
385 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
386
387 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
388 struct kern_iovec *kiovp;
389 uint64_t acnt;
390
391 uio_update_sys(uio, 0);
392 acnt = uio_curriovlen_sys(uio);
393 if (acnt == 0) {
394 continue;
395 }
396 if (n > 0 && acnt > (uint64_t)n) {
397 acnt = n;
398 }
399
400 kiovp = uio_kiovp(uio);
401
402 copywithin(CAST_DOWN(caddr_t, kiovp_get_base(kiovp)), CAST_DOWN(caddr_t, cp),
403 (size_t)acnt);
404
405 uio_update_sys(uio, (user_size_t)acnt);
406 cp += acnt;
407 n -= acnt;
408 }
409 return 0;
410 }
411
412 int
uio_copyout_phys_user(const char * c_cp __sized_by (n),int n,uio_t uio)413 uio_copyout_phys_user(const char *c_cp __sized_by(n), int n, uio_t uio)
414 {
415 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
416
417 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
418 struct user_iovec *uiovp;
419 uint64_t acnt;
420 int error;
421
422 uio_update_user(uio, 0);
423 acnt = uio_curriovlen_user(uio);
424 if (acnt == 0) {
425 continue;
426 }
427 if (n > 0 && acnt > (uint64_t)n) {
428 acnt = n;
429 }
430
431 acnt = MIN(acnt, UINT_MAX);
432 uiovp = uio_uiovp(uio);
433
434 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
435 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0);
436
437 error = copypv((addr64_t)cp, uiovp->iov_base, (unsigned int)acnt, cppvPsrc | cppvNoRefSrc);
438
439 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
440 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0);
441
442 if (error) { /* Copy virtual to physical */
443 return EFAULT;
444 }
445
446 uio_update_user(uio, (user_size_t)acnt);
447 cp += acnt;
448 n -= acnt;
449 }
450 return 0;
451 }
452
453 int
uio_copyin_phys_user(const char * c_cp __sized_by (n),int n,uio_t uio)454 uio_copyin_phys_user(const char *c_cp __sized_by(n), int n, uio_t uio)
455 {
456 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
457
458 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
459 struct user_iovec *uiovp;
460 uint64_t acnt;
461 int error;
462
463 uio_update_user(uio, 0);
464 acnt = uio_curriovlen_user(uio);
465 if (acnt == 0) {
466 continue;
467 }
468 if (n > 0 && acnt > (uint64_t)n) {
469 acnt = n;
470 }
471
472 acnt = MIN(acnt, UINT_MAX);
473 uiovp = uio_uiovp(uio);
474
475 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
476 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0);
477
478 error = copypv(uiovp->iov_base, (addr64_t)cp, (unsigned int)acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
479
480 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
481 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0);
482
483 if (error) { /* Copy virtual to physical */
484 return EFAULT;
485 }
486
487 uio_update_user(uio, (user_size_t)acnt);
488 cp += acnt;
489 n -= acnt;
490 }
491 return 0;
492 }
493
494 int
uio_copyout_phys_sys(const char * c_cp __sized_by (n),int n,uio_t uio)495 uio_copyout_phys_sys(const char *c_cp __sized_by(n), int n, uio_t uio)
496 {
497 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
498
499 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
500 struct kern_iovec *kiovp;
501 uint64_t acnt;
502 int error;
503
504 uio_update_sys(uio, 0);
505 acnt = uio_curriovlen_sys(uio);
506 if (acnt == 0) {
507 continue;
508 }
509 if (n > 0 && acnt > (uint64_t)n) {
510 acnt = n;
511 }
512
513 acnt = MIN(acnt, UINT_MAX);
514 kiovp = uio_kiovp(uio);
515
516 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
517 (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0);
518
519 error = copypv((addr64_t)cp, (addr64_t)kiovp_get_base(kiovp), (unsigned int)acnt, cppvKmap | cppvPsrc | cppvNoRefSrc);
520
521 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
522 (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0);
523
524 if (error) { /* Copy virtual to physical */
525 return EFAULT;
526 }
527
528 uio_update_sys(uio, (user_size_t)acnt);
529 cp += acnt;
530 n -= acnt;
531 }
532 return 0;
533 }
534
535 int
uio_copyin_phys_sys(const char * c_cp __sized_by (n),int n,uio_t uio)536 uio_copyin_phys_sys(const char *c_cp __sized_by(n), int n, uio_t uio)
537 {
538 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
539
540 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(uio)) {
541 struct kern_iovec *kiovp;
542 uint64_t acnt;
543 int error;
544
545 uio_update_sys(uio, 0);
546 acnt = uio_curriovlen_sys(uio);
547 if (acnt == 0) {
548 continue;
549 }
550 if (n > 0 && acnt > (uint64_t)n) {
551 acnt = n;
552 }
553
554 acnt = MIN(acnt, UINT_MAX);
555 kiovp = uio_kiovp(uio);
556
557 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
558 (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0);
559
560 error = copypv((addr64_t)kiovp_get_base(kiovp), (addr64_t)cp, (unsigned int)acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
561
562 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
563 (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0);
564
565 if (error) { /* Copy virtual to physical */
566 return EFAULT;
567 }
568
569 uio_update_sys(uio, (user_size_t)acnt);
570 cp += acnt;
571 n -= acnt;
572 }
573 return 0;
574 }
575
576 /*
577 * Give next character to user as result of read.
578 */
579 int
ureadc(int c,struct uio * uio)580 ureadc(int c, struct uio *uio)
581 {
582 struct kern_iovec *kiovp;
583 struct user_iovec *uiovp;
584
585 if (__improbable(uio_resid(uio) <= 0)) {
586 panic("ureadc: non-positive resid");
587 }
588
589 if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) {
590 uio_update_user(uio, 0);
591
592 uiovp = uio_uiovp(uio);
593
594 if (subyte((user_addr_t)uiovp->iov_base, c) < 0) {
595 return EFAULT;
596 }
597
598 uio_update_user(uio, 1);
599 } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
600 uio_update_sys(uio, 0);
601
602 kiovp = uio_kiovp(uio);
603 *(CAST_DOWN(caddr_t, kiovp_get_base(kiovp))) = (char)c;
604
605 uio_update_sys(uio, 1);
606 }
607 return 0;
608 }
609
610
611 /*
612 * General routine to allocate a hash table.
613 */
614 static size_t __pure2
hashsize(int elements)615 hashsize(int elements)
616 {
617 if (__improbable(elements <= 0)) {
618 panic("hashsize: bad cnt");
619 }
620 return 1UL << (fls(elements) - 1);
621 }
622
623 void *
hashinit(int elements,int type __unused,u_long * hashmask)624 hashinit(int elements, int type __unused, u_long *hashmask)
625 {
626 struct generic_hash_head *hashtbl;
627 vm_size_t hash_size;
628
629 hash_size = hashsize(elements);
630 hashtbl = kalloc_type(struct generic_hash_head, hash_size, Z_WAITOK | Z_ZERO);
631 if (hashtbl != NULL) {
632 *hashmask = hash_size - 1;
633 }
634 return hashtbl;
635 }
636
637 void
hashinit_generic(int elements,struct generic_hash_head * __counted_by (* out_count)* out_ptr,size_t * out_count)638 hashinit_generic(int elements,
639 struct generic_hash_head *__counted_by(*out_count) *out_ptr,
640 size_t *out_count)
641 {
642 u_long hashmask = 0;
643 struct generic_hash_head *__unsafe_indexable hash = hashinit(elements, 0, &hashmask);
644 size_t count = hashmask + 1;
645 if (hash == NULL) {
646 return;
647 } else {
648 *out_count = count;
649 *out_ptr = __unsafe_forge_bidi_indexable(struct generic_hash_head *,
650 hash,
651 count * sizeof(struct generic_hash_head));
652 }
653 }
654
655 void
hashdestroy(void * hash,int type __unused,u_long hashmask)656 hashdestroy(void *hash, int type __unused, u_long hashmask)
657 {
658 assert(powerof2(hashmask + 1));
659 kfree_type(struct generic_hash_head, hashmask + 1, hash);
660 }
661
662 /*
663 * uio_resid - return the residual IO value for the given uio_t
664 */
665 user_ssize_t
uio_resid(uio_t a_uio)666 uio_resid( uio_t a_uio )
667 {
668 #if DEBUG
669 if (a_uio == NULL) {
670 printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
671 }
672 #endif /* DEBUG */
673
674 /* return 0 if there are no active iovecs */
675 if (a_uio == NULL) {
676 return 0;
677 }
678
679 return a_uio->uio_resid_64;
680 }
681
682 /*
683 * uio_setresid - set the residual IO value for the given uio_t
684 */
685 void
uio_setresid(uio_t a_uio,user_ssize_t a_value)686 uio_setresid( uio_t a_uio, user_ssize_t a_value )
687 {
688 #if DEBUG
689 if (__improbable(a_uio == NULL)) {
690 panic("invalid uio_t");
691 }
692 #endif /* DEBUG */
693
694 if (a_uio == NULL) {
695 return;
696 }
697
698 a_uio->uio_resid_64 = a_value;
699 return;
700 }
701
702 /*
703 * uio_curriovbase - return the base address of the current iovec associated
704 * with the given uio_t. May return 0.
705 */
706 user_addr_t
uio_curriovbase(uio_t a_uio)707 uio_curriovbase( uio_t a_uio )
708 {
709 struct kern_iovec *kiovp;
710 struct user_iovec *uiovp;
711
712 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
713 return 0;
714 }
715
716 if (UIO_IS_USER_SPACE(a_uio)) {
717 uiovp = uio_uiovp(a_uio);
718 return uiovp->iov_base;
719 }
720
721 kiovp = uio_kiovp(a_uio);
722 return (user_addr_t)kiovp_get_base(kiovp);
723 }
724
725 /*
726 * uio_curriovlen_user - return the length value of the current iovec associated
727 * with the given uio_t.
728 */
729 static user_size_t
uio_curriovlen_user(const uio_t a_uio)730 uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio)
731 {
732 return uio_uiovp(a_uio)->iov_len;
733 }
734
735 /*
736 * uio_curriovlen_sys - return the length value of the current iovec associated
737 * with the given uio_t.
738 */
739 static user_size_t
uio_curriovlen_sys(const uio_t a_uio)740 uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio )
741 {
742 return (user_size_t)uio_kiovp(a_uio)->iov_len;
743 }
744
745 /*
746 * uio_curriovlen - return the length value of the current iovec associated
747 * with the given uio_t.
748 */
749 user_size_t
uio_curriovlen(uio_t a_uio)750 uio_curriovlen( uio_t a_uio )
751 {
752 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
753 return 0;
754 }
755
756 if (UIO_IS_USER_SPACE(a_uio)) {
757 return uio_curriovlen_user(a_uio);
758 }
759
760 return uio_curriovlen_sys(a_uio);
761 }
762
763 /*
764 * uio_iovcnt - return count of active iovecs for the given uio_t
765 */
766 int
uio_iovcnt(uio_t a_uio)767 uio_iovcnt( uio_t a_uio )
768 {
769 if (a_uio == NULL) {
770 return 0;
771 }
772
773 return a_uio->uio_iovcnt;
774 }
775
776 /*
777 * uio_offset - return the current offset value for the given uio_t
778 */
779 off_t
uio_offset(uio_t a_uio)780 uio_offset( uio_t a_uio )
781 {
782 if (a_uio == NULL) {
783 return 0;
784 }
785 return a_uio->uio_offset;
786 }
787
788 /*
789 * uio_setoffset - set the current offset value for the given uio_t
790 */
791 void
uio_setoffset(uio_t a_uio,off_t a_offset)792 uio_setoffset( uio_t a_uio, off_t a_offset )
793 {
794 if (a_uio == NULL) {
795 return;
796 }
797 a_uio->uio_offset = a_offset;
798 return;
799 }
800
801 /*
802 * uio_rw - return the read / write flag for the given uio_t
803 */
804 int
uio_rw(uio_t a_uio)805 uio_rw( uio_t a_uio )
806 {
807 if (a_uio == NULL) {
808 return -1;
809 }
810 return a_uio->uio_rw;
811 }
812
813 /*
814 * uio_setrw - set the read / write flag for the given uio_t
815 */
816 void
uio_setrw(uio_t a_uio,int a_value)817 uio_setrw( uio_t a_uio, int a_value )
818 {
819 if (a_uio == NULL) {
820 return;
821 }
822
823 if (a_value == UIO_READ || a_value == UIO_WRITE) {
824 a_uio->uio_rw = a_value;
825 }
826 return;
827 }
828
829 /*
830 * uio_isuserspace - return non zero value if the address space
831 * flag is for a user address space (could be 32 or 64 bit).
832 */
833 int
uio_isuserspace(uio_t a_uio)834 uio_isuserspace( uio_t a_uio )
835 {
836 if (a_uio == NULL) {
837 return 0;
838 }
839
840 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
841 return 1;
842 }
843 return 0;
844 }
845
846 static void
uio_init(uio_t uio,int a_iovcount,off_t a_offset,int a_spacetype,int a_iodirection,void * iovecs)847 uio_init(uio_t uio,
848 int a_iovcount, /* number of iovecs */
849 off_t a_offset, /* current offset */
850 int a_spacetype, /* type of address space */
851 int a_iodirection, /* read or write flag */
852 void *iovecs) /* pointer to iovec array */
853 {
854 assert(a_iovcount >= 0 && a_iovcount <= UIO_MAXIOV);
855 assert(IS_VALID_UIO_SEGFLG(a_spacetype));
856 assert(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE);
857
858 /*
859 * we use uio_segflg to indicate if the uio_t is the new format or
860 * old (pre LP64 support) legacy format
861 * This if-statement should canonicalize incoming space type
862 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
863 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
864 */
865 if (__improbable((1 << a_spacetype) & (UIOF_USERSPACE | UIOF_SYSSPACE32 | UIOF_PHYS_USERSPACE))) {
866 if (a_spacetype == UIO_USERSPACE) {
867 uio->uio_segflg = UIO_USERSPACE32;
868 } else if (a_spacetype == UIO_SYSSPACE32) {
869 uio->uio_segflg = UIO_SYSSPACE;
870 } else if (a_spacetype == UIO_PHYS_USERSPACE) {
871 uio->uio_segflg = UIO_PHYS_USERSPACE32;
872 }
873 } else {
874 uio->uio_segflg = a_spacetype;
875 }
876
877 uio->uio_iovbase = iovecs;
878 uio->uio_iovs = iovecs;
879 uio->uio_max_iovs = a_iovcount;
880 uio->uio_offset = a_offset;
881 uio->uio_rw = a_iodirection;
882 uio->uio_flags = UIO_FLAGS_INITED;
883 }
884
885 static void *
uio_alloc_iov_array(int a_spacetype,size_t a_iovcount)886 uio_alloc_iov_array(int a_spacetype, size_t a_iovcount)
887 {
888 if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) {
889 return kalloc_type(struct kern_iovec, a_iovcount, Z_WAITOK | Z_ZERO);
890 }
891
892 size_t bytes = UIO_SIZEOF_IOVS(a_iovcount);
893 return kalloc_data(bytes, Z_WAITOK | Z_ZERO);
894 }
895
896 static void
uio_free_iov_array(int a_spacetype,void * iovs,size_t a_iovcount)897 uio_free_iov_array(int a_spacetype, void *iovs, size_t a_iovcount)
898 {
899 if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) {
900 kfree_type(struct kern_iovec, a_iovcount, iovs);
901 } else {
902 size_t bytes = UIO_SIZEOF_IOVS(a_iovcount);
903 kfree_data(iovs, bytes);
904 }
905 }
906
907 /*
908 * uio_create - create an uio_t.
909 * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t
910 * is not fully initialized until all iovecs are added using uio_addiov calls.
911 * a_iovcount is the maximum number of iovecs you may add.
912 */
913 uio_t
uio_create(int a_iovcount,off_t a_offset,int a_spacetype,int a_iodirection)914 uio_create( int a_iovcount, /* number of iovecs */
915 off_t a_offset, /* current offset */
916 int a_spacetype, /* type of address space */
917 int a_iodirection ) /* read or write flag */
918 {
919 uio_t uio;
920 void *iovecs;
921
922 if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) {
923 return NULL;
924 }
925
926 uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL);
927 iovecs = uio_alloc_iov_array(a_spacetype, (size_t)a_iovcount);
928
929 uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs);
930
931 /* leave a note that we allocated this uio_t */
932 uio->uio_flags |= UIO_FLAGS_WE_ALLOCED;
933 #if DEBUG
934 os_atomic_inc(&uio_t_count, relaxed);
935 #endif
936
937 return uio;
938 }
939
940
941 /*
942 * uio_createwithbuffer - create an uio_t.
943 * Create a uio_t using the given buffer. The uio_t
944 * is not fully initialized until all iovecs are added using uio_addiov calls.
945 * a_iovcount is the maximum number of iovecs you may add.
946 * This call may fail if the given buffer is not large enough.
947 */
948 __private_extern__ uio_t
uio_createwithbuffer(int a_iovcount,off_t a_offset,int a_spacetype,int a_iodirection,void * a_buf_p,size_t a_buffer_size)949 uio_createwithbuffer( int a_iovcount, /* number of iovecs */
950 off_t a_offset, /* current offset */
951 int a_spacetype, /* type of address space */
952 int a_iodirection, /* read or write flag */
953 void *a_buf_p, /* pointer to a uio_t buffer */
954 size_t a_buffer_size ) /* size of uio_t buffer */
955 {
956 uio_t uio = (uio_t) a_buf_p;
957 void *iovecs = NULL;
958
959 if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) {
960 return NULL;
961 }
962
963 if (a_buffer_size < UIO_SIZEOF(a_iovcount)) {
964 return NULL;
965 }
966
967 if (a_iovcount > 0) {
968 iovecs = (uint8_t *)uio + sizeof(struct uio);
969 }
970
971 bzero(a_buf_p, a_buffer_size);
972 uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs);
973
974 return uio;
975 }
976
977 /*
978 * uio_iovsaddr_user - get the address of the iovec array for the given uio_t.
979 * This returns the location of the iovecs within the uio.
980 * NOTE - for compatibility mode we just return the current value in uio_iovs
981 * which will increase as the IO is completed and is NOT embedded within the
982 * uio, it is a seperate array of one or more iovecs.
983 */
984 __private_extern__ struct user_iovec *
uio_iovsaddr_user(uio_t a_uio)985 uio_iovsaddr_user( uio_t a_uio )
986 {
987 if (a_uio == NULL) {
988 return NULL;
989 }
990
991 return uio_uiovp(a_uio);
992 }
993
994 static void
_uio_reset(uio_t a_uio,off_t a_offset,int a_iodirection)995 _uio_reset(uio_t a_uio,
996 off_t a_offset, /* current offset */
997 int a_iodirection) /* read or write flag */
998 {
999 void *my_iovs = a_uio->uio_iovbase;
1000 int my_max_iovs = a_uio->uio_max_iovs;
1001
1002 if (my_iovs != NULL) {
1003 bzero(my_iovs, UIO_SIZEOF_IOVS(my_max_iovs));
1004 }
1005
1006 a_uio->uio_iovs = my_iovs;
1007 a_uio->uio_iovcnt = 0;
1008 a_uio->uio_offset = a_offset;
1009 a_uio->uio_segflg = 0;
1010 a_uio->uio_rw = a_iodirection;
1011 a_uio->uio_resid_64 = 0;
1012 }
1013
1014 void
uio_reset_fast(uio_t a_uio,off_t a_offset,int a_spacetype,int a_iodirection)1015 uio_reset_fast( uio_t a_uio,
1016 off_t a_offset, /* current offset */
1017 int a_spacetype, /* type of address space */
1018 int a_iodirection ) /* read or write flag */
1019 {
1020 _uio_reset(a_uio, a_offset, a_iodirection);
1021
1022 a_uio->uio_segflg = a_spacetype;
1023 }
1024
1025 /*
1026 * uio_reset - reset an uio_t.
1027 * Reset the given uio_t to initial values. The uio_t is not fully initialized
1028 * until all iovecs are added using uio_addiov calls.
1029 * The a_iovcount value passed in the uio_create is the maximum number of
1030 * iovecs you may add.
1031 */
1032 void
uio_reset(uio_t a_uio,off_t a_offset,int a_spacetype,int a_iodirection)1033 uio_reset( uio_t a_uio,
1034 off_t a_offset, /* current offset */
1035 int a_spacetype, /* type of address space */
1036 int a_iodirection ) /* read or write flag */
1037 {
1038 if (a_uio == NULL) {
1039 return;
1040 }
1041
1042 _uio_reset(a_uio, a_offset, a_iodirection);
1043
1044 /*
1045 * we use uio_segflg to indicate if the uio_t is the new format or
1046 * old (pre LP64 support) legacy format
1047 * This switch statement should canonicalize incoming space type
1048 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
1049 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
1050 */
1051 switch (a_spacetype) {
1052 case UIO_USERSPACE:
1053 a_uio->uio_segflg = UIO_USERSPACE32;
1054 break;
1055 case UIO_SYSSPACE32:
1056 a_uio->uio_segflg = UIO_SYSSPACE;
1057 break;
1058 case UIO_PHYS_USERSPACE:
1059 a_uio->uio_segflg = UIO_PHYS_USERSPACE32;
1060 break;
1061 default:
1062 a_uio->uio_segflg = a_spacetype;
1063 break;
1064 }
1065 }
1066
1067 /*
1068 * uio_free - free a uio_t allocated via uio_init. this also frees all
1069 * associated iovecs.
1070 */
1071 void
uio_free(uio_t a_uio)1072 uio_free( uio_t a_uio )
1073 {
1074 #if DEBUG
1075 if (__improbable(a_uio == NULL)) {
1076 panic("passing NULL uio_t");
1077 }
1078 #endif
1079
1080 if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) {
1081 #if DEBUG
1082 if (__improbable(os_atomic_dec_orig(&uio_t_count, relaxed) == 0)) {
1083 panic("uio_t_count underflow");
1084 }
1085 #endif
1086 if (__improbable(a_uio->uio_max_iovs < 0 || a_uio->uio_max_iovs > UIO_MAXIOV)) {
1087 panic("%s: bad uio_max_iovs", __func__);
1088 }
1089
1090 uio_free_iov_array(a_uio->uio_segflg, a_uio->uio_iovbase,
1091 (size_t)a_uio->uio_max_iovs);
1092
1093 kfree_type(struct uio, a_uio);
1094 }
1095 }
1096
1097 /*
1098 * uio_addiov - add an iovec to the given uio_t. You may call this up to
1099 * the a_iovcount number that was passed to uio_create. This call will
1100 * increment the residual IO count as iovecs are added to the uio_t.
1101 * returns 0 if add was successful else non zero.
1102 */
1103 int
uio_addiov(uio_t a_uio,user_addr_t a_baseaddr,user_size_t a_length)1104 uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length )
1105 {
1106 int i;
1107 user_size_t resid;
1108 struct kern_iovec *kiovp;
1109 struct user_iovec *uiovp;
1110
1111 if (__improbable(a_uio == NULL)) {
1112 #if DEBUG
1113 panic("invalid uio_t");
1114 #endif
1115 return -1;
1116 }
1117
1118 if (__improbable(os_add_overflow(a_length, a_uio->uio_resid_64, &resid))) {
1119 #if DEBUG
1120 panic("invalid length %lu", (unsigned long)a_length);
1121 #endif
1122 return -1;
1123 }
1124
1125 if (UIO_IS_USER_SPACE(a_uio)) {
1126 uiovp = uio_uiovp(a_uio);
1127 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1128 if (uiovp[i].iov_len == 0 &&
1129 uiovp[i].iov_base == 0) {
1130 uiovp[i].iov_len = a_length;
1131 uiovp[i].iov_base = a_baseaddr;
1132 a_uio->uio_iovcnt++;
1133 a_uio->uio_resid_64 = resid;
1134 return 0;
1135 }
1136 }
1137 } else {
1138 kiovp = uio_kiovp(a_uio);
1139 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1140 if (kiovp[i].iov_len == 0 &&
1141 kiovp_get_base(&kiovp[i]) == 0) {
1142 kiovp[i].iov_len = (u_int64_t)a_length;
1143 kiovp_set_base(&kiovp[i], (u_int64_t)a_baseaddr);
1144 a_uio->uio_iovcnt++;
1145 a_uio->uio_resid_64 = resid;
1146 return 0;
1147 }
1148 }
1149 }
1150
1151 return -1;
1152 }
1153
1154 /*
1155 * uio_getiov - get iovec data associated with the given uio_t. Use
1156 * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)).
1157 * a_baseaddr_p and a_length_p may be NULL.
1158 * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t.
1159 * returns 0 when data is returned.
1160 */
1161 int
uio_getiov(uio_t a_uio,int a_index,user_addr_t * a_baseaddr_p,user_size_t * a_length_p)1162 uio_getiov( uio_t a_uio,
1163 int a_index,
1164 user_addr_t * a_baseaddr_p,
1165 user_size_t * a_length_p )
1166 {
1167 struct kern_iovec *kiovp;
1168 struct user_iovec *uiovp;
1169
1170 if (a_uio == NULL) {
1171 #if DEBUG
1172 panic("invalid uio_t");
1173 #endif /* DEBUG */
1174 return -1;
1175 }
1176 if (a_index < 0 || a_index >= a_uio->uio_iovcnt) {
1177 return -1;
1178 }
1179
1180 if (UIO_IS_USER_SPACE(a_uio)) {
1181 uiovp = uio_uiovp(a_uio);
1182
1183 if (a_baseaddr_p != NULL) {
1184 *a_baseaddr_p = uiovp[a_index].iov_base;
1185 }
1186 if (a_length_p != NULL) {
1187 *a_length_p = uiovp[a_index].iov_len;
1188 }
1189 } else {
1190 kiovp = uio_kiovp(a_uio);
1191
1192 if (a_baseaddr_p != NULL) {
1193 *a_baseaddr_p = (user_addr_t)kiovp_get_base(&kiovp[a_index]);
1194 }
1195 if (a_length_p != NULL) {
1196 *a_length_p = (user_size_t)kiovp[a_index].iov_len;
1197 }
1198 }
1199
1200 return 0;
1201 }
1202
1203 /*
1204 * uio_calculateresid_user - runs through all iovecs associated with this
1205 * uio_t and calculates (and sets) the residual IO count.
1206 */
1207 __private_extern__ int
uio_calculateresid_user(uio_t __attribute ((nonnull))a_uio)1208 uio_calculateresid_user(uio_t __attribute((nonnull))a_uio)
1209 {
1210 int i;
1211 u_int64_t resid = 0;
1212 struct user_iovec *uiovp;
1213
1214 a_uio->uio_iovcnt = a_uio->uio_max_iovs;
1215 uiovp = uio_uiovp(a_uio);
1216 a_uio->uio_resid_64 = 0;
1217 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1218 if (uiovp[i].iov_len != 0) {
1219 if (uiovp[i].iov_len > LONG_MAX) {
1220 return EINVAL;
1221 }
1222 resid += uiovp[i].iov_len;
1223 if (resid > LONG_MAX) {
1224 return EINVAL;
1225 }
1226 }
1227 }
1228 a_uio->uio_resid_64 = (user_size_t)resid;
1229
1230 /* position to first non zero length iovec (4235922) */
1231 while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) {
1232 a_uio->uio_iovcnt--;
1233 if (a_uio->uio_iovcnt > 0) {
1234 uiovp = uio_advance_user(a_uio);
1235 }
1236 }
1237
1238 return 0;
1239 }
1240
1241 /*
1242 * uio_update_user - update the given uio_t for a_count of completed IO.
1243 * This call decrements the current iovec length and residual IO value
1244 * and increments the current iovec base address and offset value.
1245 * If the current iovec length is 0 then advance to the next
1246 * iovec (if any).
1247 * If the a_count passed in is 0, than only do the advancement
1248 * over any 0 length iovec's.
1249 */
1250 static void
uio_update_user(uio_t a_uio,user_size_t a_count)1251 uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count)
1252 {
1253 struct user_iovec *uiovp;
1254
1255 uiovp = uio_uiovp(a_uio);
1256
1257 /*
1258 * if a_count == 0, then we are asking to skip over
1259 * any empty iovs
1260 */
1261 if (a_count) {
1262 if (a_count > uiovp->iov_len) {
1263 uiovp->iov_base += uiovp->iov_len;
1264 uiovp->iov_len = 0;
1265 } else {
1266 uiovp->iov_base += a_count;
1267 uiovp->iov_len -= a_count;
1268 }
1269 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1270 a_uio->uio_offset += a_uio->uio_resid_64;
1271 a_uio->uio_resid_64 = 0;
1272 } else {
1273 a_uio->uio_offset += a_count;
1274 a_uio->uio_resid_64 -= a_count;
1275 }
1276 }
1277 /*
1278 * advance to next iovec if current one is totally consumed
1279 */
1280 while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) {
1281 a_uio->uio_iovcnt--;
1282 if (a_uio->uio_iovcnt > 0) {
1283 uiovp = uio_advance_user(a_uio);
1284 }
1285 }
1286 }
1287
1288 /*
1289 * uio_update_sys - update the given uio_t for a_count of completed IO.
1290 * This call decrements the current iovec length and residual IO value
1291 * and increments the current iovec base address and offset value.
1292 * If the current iovec length is 0 then advance to the next
1293 * iovec (if any).
1294 * If the a_count passed in is 0, than only do the advancement
1295 * over any 0 length iovec's.
1296 */
1297 static void
uio_update_sys(uio_t a_uio,user_size_t a_count)1298 uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count)
1299 {
1300 struct kern_iovec *kiovp;
1301
1302 kiovp = uio_kiovp(a_uio);
1303
1304 /*
1305 * if a_count == 0, then we are asking to skip over
1306 * any empty iovs
1307 */
1308 if (a_count) {
1309 u_int64_t prev_base = kiovp_get_base(kiovp);
1310 if (a_count > kiovp->iov_len) {
1311 u_int64_t len = kiovp->iov_len;
1312 kiovp->iov_len = 0;
1313 kiovp_set_base(kiovp, prev_base + len);
1314 } else {
1315 kiovp->iov_len -= a_count;
1316 kiovp_set_base(kiovp, prev_base + a_count);
1317 }
1318 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1319 a_uio->uio_offset += a_uio->uio_resid_64;
1320 a_uio->uio_resid_64 = 0;
1321 } else {
1322 a_uio->uio_offset += a_count;
1323 a_uio->uio_resid_64 -= a_count;
1324 }
1325 }
1326 /*
1327 * advance to next iovec if current one is totally consumed
1328 */
1329 while (a_uio->uio_iovcnt > 0 && kiovp->iov_len == 0) {
1330 a_uio->uio_iovcnt--;
1331 if (a_uio->uio_iovcnt > 0) {
1332 kiovp = uio_advance_sys(a_uio);
1333 }
1334 }
1335 }
1336
1337 /*
1338 * uio_update - update the given uio_t for a_count of completed IO.
1339 * This call decrements the current iovec length and residual IO value
1340 * and increments the current iovec base address and offset value.
1341 * If the current iovec length is 0 then advance to the next
1342 * iovec (if any).
1343 * If the a_count passed in is 0, than only do the advancement
1344 * over any 0 length iovec's.
1345 */
1346 void
uio_update(uio_t a_uio,user_size_t a_count)1347 uio_update(uio_t a_uio, user_size_t a_count)
1348 {
1349 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
1350 return;
1351 }
1352
1353 if (UIO_IS_USER_SPACE(a_uio)) {
1354 uio_update_user(a_uio, a_count);
1355 } else {
1356 uio_update_sys(a_uio, a_count);
1357 }
1358 }
1359
1360 /*
1361 * uio_duplicate - allocate a new uio and make a copy of the given uio_t.
1362 * may return NULL.
1363 */
1364 uio_t
uio_duplicate(uio_t uio)1365 uio_duplicate(uio_t uio)
1366 {
1367 uio_t new_uio;
1368 size_t n;
1369 struct kern_iovec *kiovp;
1370 struct user_iovec *uiovp;
1371
1372 if (uio->uio_max_iovs < 0 || uio->uio_max_iovs > UIO_MAXIOV) {
1373 return NULL;
1374 }
1375
1376 new_uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1377 *new_uio = *uio;
1378
1379 if (new_uio->uio_max_iovs > 0) {
1380 new_uio->uio_iovbase = uio_alloc_iov_array(new_uio->uio_segflg,
1381 (size_t)new_uio->uio_max_iovs);
1382 new_uio->uio_iovs = new_uio->uio_iovbase;
1383
1384 n = UIO_SIZEOF_IOVS(new_uio->uio_iovcnt);
1385 bcopy((const void *)uio->uio_iovs, (void *)new_uio->uio_iovs, n);
1386 if (UIO_IS_SYS_SPACE(new_uio)) {
1387 struct kern_iovec *kiovp_old = uio_kiovp(uio);
1388
1389 kiovp = uio_kiovp(new_uio);
1390
1391 for (n = 0; n < new_uio->uio_max_iovs; ++n) {
1392 kiovp_set_base(&kiovp[n],
1393 kiovp_get_base(&kiovp_old[n]));
1394 }
1395 } else {
1396 uiovp = uio_uiovp(new_uio);
1397 }
1398
1399 /* advance to first nonzero iovec */
1400 for (n = 0; n < new_uio->uio_max_iovs; ++n) {
1401 if (UIO_IS_USER_SPACE(new_uio)) {
1402 if (uiovp->iov_len != 0) {
1403 break;
1404 }
1405
1406 uiovp = uio_advance_user(new_uio);
1407 } else {
1408 if (kiovp->iov_len != 0) {
1409 break;
1410 }
1411
1412 kiovp = uio_advance_sys(new_uio);
1413 }
1414 }
1415 } else {
1416 new_uio->uio_iovs = NULL;
1417 }
1418
1419 new_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED;
1420 #if DEBUG
1421 os_atomic_inc(&uio_t_count, relaxed);
1422 #endif
1423
1424 return new_uio;
1425 }
1426
1427 int
uio_restore(uio_t uio,uio_t snapshot_uio)1428 uio_restore(uio_t uio, uio_t snapshot_uio)
1429 {
1430 struct kern_iovec *kiovp;
1431 struct user_iovec *uiovp;
1432 size_t n;
1433
1434 if (uio->uio_max_iovs != snapshot_uio->uio_max_iovs) {
1435 return EINVAL;
1436 }
1437 if (uio->uio_max_iovs < 0 || uio->uio_max_iovs > UIO_MAXIOV) {
1438 return EINVAL;
1439 }
1440
1441 // printf("******* FBDP %s:%d uio %p (iovs %p cnt %d resid 0x%llx) snap %p (iovs %p cnt %d resid 0x%llx)\n", __FUNCTION__, __LINE__, uio, uio->uio_iovs, uio_iovcnt(uio), uio_resid(uio), snapshot_uio, snapshot_uio->uio_iovs, uio_iovcnt(snapshot_uio), uio_resid(snapshot_uio));
1442
1443 uio->uio_iovcnt = snapshot_uio->uio_iovcnt;
1444 uio->uio_offset = snapshot_uio->uio_offset;
1445 uio->uio_rw = snapshot_uio->uio_rw;
1446 uio->uio_resid_64 = snapshot_uio->uio_resid_64;
1447
1448 if (uio->uio_max_iovs > 0) {
1449 n = UIO_SIZEOF_IOVS(snapshot_uio->uio_max_iovs);
1450 bcopy((const void *)snapshot_uio->uio_iovbase, (void *)uio->uio_iovbase, n);
1451 if (UIO_IS_SYS_SPACE(uio)) {
1452 struct kern_iovec *kiovp_old = uio_kiovp(snapshot_uio);
1453
1454 kiovp = uio_kiovp(uio);
1455
1456 for (n = 0; n < snapshot_uio->uio_max_iovs; ++n) {
1457 kiovp_set_base(&kiovp[n],
1458 kiovp_get_base(&kiovp_old[n]));
1459 }
1460 } else {
1461 uiovp = uio_uiovp(uio);
1462 }
1463
1464 /* advance to first nonzero iovec */
1465 for (n = 0; n < uio->uio_max_iovs; ++n) {
1466 if (UIO_IS_USER_SPACE(uio)) {
1467 if (uiovp->iov_len != 0) {
1468 break;
1469 }
1470
1471 uiovp = uio_advance_user(uio);
1472 } else {
1473 if (kiovp->iov_len != 0) {
1474 break;
1475 }
1476
1477 kiovp = uio_advance_sys(uio);
1478 }
1479 }
1480
1481 uio->uio_iovs = uio->uio_iovbase;
1482 } else {
1483 assert(uio->uio_iovs == NULL);
1484 }
1485 return 0;
1486 }
1487
1488 int
copyin_user_iovec_array(user_addr_t uaddr,int spacetype,int count,struct user_iovec * dst)1489 copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst)
1490 {
1491 size_t size_of_iovec = (spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec));
1492 int error;
1493 int i;
1494
1495 // copyin to the front of "dst", without regard for putting records in the right places
1496 error = copyin(uaddr, dst, count * size_of_iovec);
1497 if (error) {
1498 return error;
1499 }
1500
1501 // now, unpack the entries in reverse order, so we don't overwrite anything
1502 for (i = count - 1; i >= 0; i--) {
1503 if (spacetype == UIO_USERSPACE64) {
1504 struct user64_iovec iovec = ((struct user64_iovec *)dst)[i];
1505 dst[i].iov_base = (user_addr_t)iovec.iov_base;
1506 dst[i].iov_len = (user_size_t)iovec.iov_len;
1507 } else {
1508 struct user32_iovec iovec = ((struct user32_iovec *)dst)[i];
1509 dst[i].iov_base = iovec.iov_base;
1510 dst[i].iov_len = iovec.iov_len;
1511 }
1512 }
1513
1514 return 0;
1515 }
1516