1 /*
2 * Copyright (c) 2012-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/cpu_data_internal.h>
30 #include <arm/misc_protos.h>
31 #include <kern/thread.h>
32 #include <kern/zalloc_internal.h>
33 #include <sys/errno.h>
34 #include <vm/pmap.h>
35 #include <vm/vm_map_xnu.h>
36 #include <vm/vm_memtag.h>
37 #include <san/kasan.h>
38 #include <arm/pmap.h>
39 #include <arm64/speculation.h>
40
41 #undef copyin
42 #undef copyout
43
44 extern int _bcopyin(const user_addr_t src, char *dst, vm_size_t len);
45 extern int _bcopyinstr(const user_addr_t src, char *dst, vm_size_t max, vm_size_t *actual);
46 extern int _bcopyout(const char *src, user_addr_t dst, vm_size_t len);
47 extern int _copyin_atomic32(const user_addr_t src, uint32_t *dst);
48 extern int _copyin_atomic32_wait_if_equals(const user_addr_t src, uint32_t value);
49 extern int _copyin_atomic64(const user_addr_t src, uint64_t *dst);
50 extern int _copyout_atomic32(uint32_t u32, user_addr_t dst);
51 extern int _copyout_atomic64(uint64_t u64, user_addr_t dst);
52
53
54 extern int copyoutstr_prevalidate(const void *kaddr, user_addr_t uaddr, size_t len);
55
56 extern const vm_map_address_t physmap_base;
57 extern const vm_map_address_t physmap_end;
58
59 /*!
60 * @typedef copyio_flags_t
61 *
62 * @const COPYIO_IN
63 * The copy is user -> kernel.
64 * One of COPYIO_IN or COPYIO_OUT should always be specified.
65 *
66 * @const COPYIO_OUT
67 * The copy is kernel -> user
68 * One of COPYIO_IN or COPYIO_OUT should always be specified.
69 *
70 * @const COPYIO_ALLOW_KERNEL_TO_KERNEL
71 * The "user_address" is allowed to be in the VA space of the kernel.
72 *
73 * @const COPYIO_VALIDATE_USER_ONLY
74 * There isn't really a kernel address used, and only the user address
75 * needs to be validated.
76 *
77 * @const COPYIO_ATOMIC
78 * The copyio operation is atomic, ensure that it is properly aligned.
79 */
80 __options_decl(copyio_flags_t, uint32_t, {
81 COPYIO_IN = 0x0001,
82 COPYIO_OUT = 0x0002,
83 COPYIO_ALLOW_KERNEL_TO_KERNEL = 0x0004,
84 COPYIO_VALIDATE_USER_ONLY = 0x0008,
85 COPYIO_ATOMIC = 0x0010,
86 });
87
88 typedef enum {
89 USER_ACCESS_READ,
90 USER_ACCESS_WRITE
91 } user_access_direction_t;
92
93 static inline void
user_access_enable(__unused user_access_direction_t user_access_direction,pmap_t __unused pmap)94 user_access_enable(__unused user_access_direction_t user_access_direction, pmap_t __unused pmap)
95 {
96 #if __ARM_PAN_AVAILABLE__
97 assert(__builtin_arm_rsr("pan") != 0);
98 __builtin_arm_wsr("pan", 0);
99 #endif /* __ARM_PAN_AVAILABLE__ */
100
101 }
102
103 static inline void
user_access_disable(__unused user_access_direction_t user_access_direction,pmap_t __unused pmap)104 user_access_disable(__unused user_access_direction_t user_access_direction, pmap_t __unused pmap)
105 {
106 #if __ARM_PAN_AVAILABLE__
107 __builtin_arm_wsr("pan", 1);
108 #endif /* __ARM_PAN_AVAILABLE__ */
109
110 }
111
112
113 #define WRAP_COPYIO_PAN(_dir, _map, _op) \
114 ({ \
115 int _ret; \
116 user_access_enable(_dir, (_map)->pmap); \
117 _ret = _op; \
118 user_access_disable(_dir, (_map)->pmap); \
119 _ret; \
120 })
121
122 #define WRAP_COPYIO(_dir, _map, _op) WRAP_COPYIO_PAN(_dir, _map, _op)
123
124 /*
125 * Copy sizes bigger than this value will cause a kernel panic.
126 *
127 * Yes, this is an arbitrary fixed limit, but it's almost certainly
128 * a programming error to be copying more than this amount between
129 * user and wired kernel memory in a single invocation on this
130 * platform.
131 */
132 const int copysize_limit_panic = (64 * 1024 * 1024);
133
134 static inline bool
is_kernel_to_kernel_copy(pmap_t pmap)135 is_kernel_to_kernel_copy(pmap_t pmap)
136 {
137 return pmap == kernel_pmap;
138 }
139
140 /**
141 * In order to prevent copies from speculatively targeting the wrong address
142 * space, force kernel-to-kernel copies to target the kernel address space
143 * (TTBR1) and non-kernel copies to target the user address space (TTBR0).
144 *
145 * This should have no non-speculative effect as any address which passes
146 * validation should already have bit 55 (the address space select bit) set
147 * appropriately. If the address would change (i.e. addr is invalid for the copy
148 * type), this function panics and so it must only be called after all other
149 * verification has completed.
150 */
151 static user_addr_t
copy_ensure_address_space_spec(vm_map_t map,const user_addr_t addr)152 copy_ensure_address_space_spec(vm_map_t map, const user_addr_t addr)
153 {
154 user_addr_t new_addr = 0;
155 user_addr_t kaddr = addr | BIT(55);
156 user_addr_t uaddr = addr & (~BIT(55));
157
158 /*
159 * new_addr = is_kernel_to_kernel_copy(...) ? kaddr : uaddr
160 *
161 * The check must be performed explicitly as the compiler lowering of the
162 * actual call may be subject to prediction.
163 */
164 SPECULATION_GUARD_SELECT_XXX(
165 /* out */ new_addr,
166 /* cmp_1 */ map->pmap, /* cmp_2 */ kernel_pmap,
167 /* cc */ "eq", /* sel_1 */ kaddr,
168 /* n_cc */ "ne", /* sel_2 */ uaddr);
169
170 /*
171 * Since we're modifying the address past the validation point, let's be
172 * sure we didn't erroneously change address spaces.
173 *
174 * We have to be careful to hide this check from the optimizer as if it
175 * learns that new_addr == addr, then it is free to (and, indeed, does) use
176 * addr everywhere that new_addr is referenced, which breaks our hardening.
177 */
178 user_addr_t new_addr_opt_hidden = new_addr;
179 __compiler_materialize_and_prevent_reordering_on(new_addr_opt_hidden);
180 if (new_addr_opt_hidden != addr) {
181 panic("copy_ensure_address_space_spec changed address: 0x%llx->0x%llx",
182 addr, new_addr);
183 }
184
185 return new_addr;
186 }
187
188 static int
copy_validate_user_addr(vm_map_t map,const user_addr_t user_addr,vm_size_t nbytes)189 copy_validate_user_addr(vm_map_t map, const user_addr_t user_addr, vm_size_t nbytes)
190 {
191 user_addr_t canonicalized_user_addr = user_addr;
192 user_addr_t user_addr_last;
193 bool is_kernel_to_kernel = is_kernel_to_kernel_copy(map->pmap);
194
195
196 if (__improbable(canonicalized_user_addr < vm_map_min(map) ||
197 os_add_overflow(canonicalized_user_addr, nbytes, &user_addr_last) ||
198 user_addr_last > vm_map_max(map))) {
199 return EFAULT;
200 }
201
202
203 if (!is_kernel_to_kernel) {
204 if (__improbable(canonicalized_user_addr & ARM_TBI_USER_MASK)) {
205 return EINVAL;
206 }
207 }
208
209 return 0;
210 }
211
212 static void
copy_validate_kernel_addr(uintptr_t kernel_addr,vm_size_t nbytes)213 copy_validate_kernel_addr(uintptr_t kernel_addr, vm_size_t nbytes)
214 {
215 uintptr_t kernel_addr_last;
216
217 if (__improbable(os_add_overflow(kernel_addr, nbytes, &kernel_addr_last))) {
218 panic("%s(%p, %lu) - kaddr not in kernel", __func__,
219 (void *)kernel_addr, nbytes);
220 }
221
222 bool in_kva = (VM_KERNEL_STRIP_UPTR(kernel_addr) >= VM_MIN_KERNEL_ADDRESS) &&
223 (VM_KERNEL_STRIP_UPTR(kernel_addr_last) <= VM_MAX_KERNEL_ADDRESS);
224 bool in_physmap = (VM_KERNEL_STRIP_UPTR(kernel_addr) >= physmap_base) &&
225 (VM_KERNEL_STRIP_UPTR(kernel_addr_last) <= physmap_end);
226
227 if (__improbable(!(in_kva || in_physmap))) {
228 panic("%s(%p, %lu) - kaddr not in kernel", __func__,
229 (void *)kernel_addr, nbytes);
230 }
231
232 zone_element_bounds_check(kernel_addr, nbytes);
233 }
234
235 /*
236 * Validate the arguments to copy{in,out} on this platform.
237 *
238 * Returns EXDEV when the current thread pmap is the kernel's
239 * which is non fatal for certain routines.
240 */
241 static inline __attribute__((always_inline)) int
copy_validate(vm_map_t map,const user_addr_t user_addr,uintptr_t kernel_addr,vm_size_t nbytes,copyio_flags_t flags)242 copy_validate(vm_map_t map, const user_addr_t user_addr, uintptr_t kernel_addr,
243 vm_size_t nbytes, copyio_flags_t flags)
244 {
245 int ret;
246
247 if (__improbable(nbytes > copysize_limit_panic)) {
248 return EINVAL;
249 }
250
251 ret = copy_validate_user_addr(map, user_addr, nbytes);
252 if (__improbable(ret)) {
253 return ret;
254 }
255
256 if (flags & COPYIO_ATOMIC) {
257 if (__improbable(user_addr & (nbytes - 1))) {
258 return EINVAL;
259 }
260 }
261
262 if ((flags & COPYIO_VALIDATE_USER_ONLY) == 0) {
263 copy_validate_kernel_addr(kernel_addr, nbytes);
264 #if KASAN
265 /* For user copies, asan-check the kernel-side buffer */
266 if (flags & COPYIO_IN) {
267 __asan_storeN(kernel_addr, nbytes);
268 } else {
269 __asan_loadN(kernel_addr, nbytes);
270 }
271 #endif
272 }
273
274 if (is_kernel_to_kernel_copy(map->pmap)) {
275 if (__improbable((flags & COPYIO_ALLOW_KERNEL_TO_KERNEL) == 0)) {
276 return EFAULT;
277 }
278 return EXDEV;
279 }
280
281 return 0;
282 }
283
284 int
copyin_kern(const user_addr_t user_addr,char * kernel_addr,vm_size_t nbytes)285 copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
286 {
287 bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes);
288
289 return 0;
290 }
291
292 int
copyout_kern(const char * kernel_addr,user_addr_t user_addr,vm_size_t nbytes)293 copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
294 {
295 bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes);
296
297 return 0;
298 }
299
300 int
copyin(const user_addr_t user_addr,void * kernel_addr,vm_size_t nbytes)301 copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
302 {
303 vm_map_t map = current_thread()->map;
304 user_addr_t guarded_user_addr;
305 int result;
306
307 if (__improbable(nbytes == 0)) {
308 return 0;
309 }
310
311 result = copy_validate(map, user_addr, (uintptr_t)kernel_addr, nbytes,
312 COPYIO_IN | COPYIO_ALLOW_KERNEL_TO_KERNEL);
313 if (result == EXDEV) {
314 guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
315 return copyin_kern(guarded_user_addr, kernel_addr, nbytes);
316 }
317 if (__improbable(result)) {
318 return result;
319 }
320
321 guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
322
323 return WRAP_COPYIO(USER_ACCESS_READ, map,
324 _bcopyin(guarded_user_addr, kernel_addr, nbytes));
325 }
326
327 /*
328 * copy{in,out}_atomic{32,64}
329 * Read or store an aligned value from userspace as a single memory transaction.
330 * These functions support userspace synchronization features
331 */
332 int
copyin_atomic32(const user_addr_t user_addr,uint32_t * kernel_addr)333 copyin_atomic32(const user_addr_t user_addr, uint32_t *kernel_addr)
334 {
335 vm_map_t map = current_thread()->map;
336 int result = copy_validate(map, user_addr, (uintptr_t)kernel_addr, 4,
337 COPYIO_IN | COPYIO_ATOMIC);
338 if (__improbable(result)) {
339 return result;
340 }
341
342 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
343
344 return WRAP_COPYIO(USER_ACCESS_READ, map,
345 _copyin_atomic32(guarded_user_addr, kernel_addr));
346 }
347
348
349 int
copyin_atomic32_wait_if_equals(const user_addr_t user_addr,uint32_t value)350 copyin_atomic32_wait_if_equals(const user_addr_t user_addr, uint32_t value)
351 {
352 vm_map_t map = current_thread()->map;
353 int result = copy_validate(map, user_addr, 0, 4,
354 COPYIO_OUT | COPYIO_ATOMIC | COPYIO_VALIDATE_USER_ONLY);
355 if (__improbable(result)) {
356 return result;
357 }
358
359 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
360
361 return WRAP_COPYIO(USER_ACCESS_READ, map,
362 _copyin_atomic32_wait_if_equals(guarded_user_addr, value));
363 }
364
365 int
copyin_atomic64(const user_addr_t user_addr,uint64_t * kernel_addr)366 copyin_atomic64(const user_addr_t user_addr, uint64_t *kernel_addr)
367 {
368 vm_map_t map = current_thread()->map;
369 int result = copy_validate(map, user_addr, (uintptr_t)kernel_addr, 8,
370 COPYIO_IN | COPYIO_ATOMIC);
371 if (__improbable(result)) {
372 return result;
373 }
374
375 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
376
377 return WRAP_COPYIO(USER_ACCESS_READ, map,
378 _copyin_atomic64(guarded_user_addr, kernel_addr));
379 }
380
381 int
copyout_atomic32(uint32_t value,user_addr_t user_addr)382 copyout_atomic32(uint32_t value, user_addr_t user_addr)
383 {
384 vm_map_t map = current_thread()->map;
385 int result = copy_validate(map, user_addr, 0, 4,
386 COPYIO_OUT | COPYIO_ATOMIC | COPYIO_VALIDATE_USER_ONLY);
387 if (__improbable(result)) {
388 return result;
389 }
390
391 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
392
393 return WRAP_COPYIO(USER_ACCESS_WRITE, map,
394 _copyout_atomic32(value, guarded_user_addr));
395 }
396
397 int
copyout_atomic64(uint64_t value,user_addr_t user_addr)398 copyout_atomic64(uint64_t value, user_addr_t user_addr)
399 {
400 vm_map_t map = current_thread()->map;
401 int result = copy_validate(map, user_addr, 0, 8,
402 COPYIO_OUT | COPYIO_ATOMIC | COPYIO_VALIDATE_USER_ONLY);
403 if (__improbable(result)) {
404 return result;
405 }
406
407 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
408
409 return WRAP_COPYIO(USER_ACCESS_WRITE, map,
410 _copyout_atomic64(value, guarded_user_addr));
411 }
412
413 int
copyinstr(const user_addr_t user_addr,char * kernel_addr,vm_size_t nbytes,vm_size_t * lencopied)414 copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
415 {
416 vm_map_t map = current_thread()->map;
417 int result;
418 vm_size_t bytes_copied = 0;
419
420 *lencopied = 0;
421 if (__improbable(nbytes == 0)) {
422 return ENAMETOOLONG;
423 }
424
425 result = copy_validate(map, user_addr, (uintptr_t)kernel_addr, nbytes, COPYIO_IN);
426 if (__improbable(result)) {
427 return result;
428 }
429
430 user_addr_t guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
431
432 result = WRAP_COPYIO(USER_ACCESS_READ, map,
433 _bcopyinstr(guarded_user_addr, kernel_addr, nbytes, &bytes_copied));
434
435 if (result != EFAULT) {
436 *lencopied = bytes_copied;
437 }
438 return result;
439 }
440
441 int
copyout(const void * kernel_addr,user_addr_t user_addr,vm_size_t nbytes)442 copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
443 {
444 vm_map_t map = current_thread()->map;
445 int result;
446 user_addr_t guarded_user_addr;
447
448 if (nbytes == 0) {
449 return 0;
450 }
451
452 result = copy_validate(map, user_addr, (uintptr_t)kernel_addr, nbytes,
453 COPYIO_OUT | COPYIO_ALLOW_KERNEL_TO_KERNEL);
454 if (result == EXDEV) {
455 guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
456 return copyout_kern(kernel_addr, guarded_user_addr, nbytes);
457 }
458 if (__improbable(result)) {
459 return result;
460 }
461
462 guarded_user_addr = copy_ensure_address_space_spec(map, user_addr);
463
464 return WRAP_COPYIO(USER_ACCESS_WRITE, map,
465 _bcopyout(kernel_addr, guarded_user_addr, nbytes));
466 }
467
468 int
copyoutstr_prevalidate(const void * __unused kaddr,user_addr_t __unused uaddr,size_t __unused len)469 copyoutstr_prevalidate(const void *__unused kaddr, user_addr_t __unused uaddr, size_t __unused len)
470 {
471 vm_map_t map = current_thread()->map;
472
473 if (__improbable(is_kernel_to_kernel_copy(map->pmap))) {
474 return EFAULT;
475 }
476
477 return 0;
478 }
479
480 #if (DEBUG || DEVELOPMENT)
481 int
verify_write(const void * source,void * dst,size_t size)482 verify_write(const void *source, void *dst, size_t size)
483 {
484 int rc;
485 disable_preemption();
486 rc = _bcopyout((const char*)source, (user_addr_t)dst, size);
487 enable_preemption();
488 return rc;
489 }
490 #endif
491