1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <mach/boolean.h>
91 #include <mach/kern_return.h>
92 #include <mach/mach_types.h> /* to get vm_address_t */
93 #include <mach/memory_object.h>
94 #include <mach/std_types.h> /* to get pointer_t */
95 #include <mach/upl.h>
96 #include <mach/vm_attributes.h>
97 #include <mach/vm_param.h>
98 #include <mach/vm_statistics.h>
99 #include <mach/mach_syscalls.h>
100 #include <mach/sdt.h>
101 #include <mach/memory_entry.h>
102
103 #include <mach/host_priv_server.h>
104 #include <mach/mach_vm_server.h>
105 #include <mach/memory_entry_server.h>
106 #include <mach/vm_map_server.h>
107
108 #include <kern/host.h>
109 #include <kern/kalloc.h>
110 #include <kern/task.h>
111 #include <kern/misc_protos.h>
112 #include <vm/vm_fault.h>
113 #include <vm/vm_map_internal.h>
114 #include <vm/vm_object_xnu.h>
115 #include <vm/vm_kern.h>
116 #include <vm/vm_page_internal.h>
117 #include <vm/memory_object_internal.h>
118 #include <vm/vm_pageout_internal.h>
119 #include <vm/vm_protos.h>
120 #include <vm/vm_purgeable_internal.h>
121 #include <vm/vm_memory_entry_xnu.h>
122 #include <vm/vm_kern_internal.h>
123 #include <vm/vm_iokit.h>
124 #include <vm/vm_sanitize_internal.h>
125 #if CONFIG_DEFERRED_RECLAIM
126 #include <vm/vm_reclaim_internal.h>
127 #endif /* CONFIG_DEFERRED_RECLAIM */
128 #include <vm/vm_init_xnu.h>
129
130 #include <san/kasan.h>
131
132 #include <libkern/OSDebug.h>
133 #include <IOKit/IOBSD.h>
134 #include <sys/kdebug_triage.h>
135
136 /*
137 * mach_vm_allocate allocates "zero fill" memory in the specfied
138 * map.
139 */
140 kern_return_t
mach_vm_allocate_external(vm_map_t map,mach_vm_offset_ut * addr,mach_vm_size_ut size,int flags)141 mach_vm_allocate_external(
142 vm_map_t map,
143 mach_vm_offset_ut *addr,
144 mach_vm_size_ut size,
145 int flags)
146 {
147 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
148
149 /* filter out any kernel-only flags */
150 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
151 ktriage_record(thread_tid(current_thread()),
152 KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM,
153 KDBG_TRIAGE_RESERVED,
154 KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADFLAGS_ERROR),
155 KERN_INVALID_ARGUMENT /* arg */);
156 return KERN_INVALID_ARGUMENT;
157 }
158
159 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
160
161 return mach_vm_allocate_kernel(map, addr, size, vmk_flags);
162 }
163
164 /*
165 * vm_allocate
166 * Legacy routine that allocates "zero fill" memory in the specfied
167 * map (which is limited to the same size as the kernel).
168 */
169 kern_return_t
vm_allocate_external(vm_map_t map,vm_offset_ut * addr,vm_size_ut size,int flags)170 vm_allocate_external(
171 vm_map_t map,
172 vm_offset_ut *addr,
173 vm_size_ut size,
174 int flags)
175 {
176 return mach_vm_allocate_external(map, addr, size, flags);
177 }
178
179 static inline kern_return_t
mach_vm_deallocate_sanitize(vm_map_t map,mach_vm_offset_ut start_u,mach_vm_size_ut size_u,mach_vm_offset_t * start,mach_vm_offset_t * end,mach_vm_size_t * size)180 mach_vm_deallocate_sanitize(
181 vm_map_t map,
182 mach_vm_offset_ut start_u,
183 mach_vm_size_ut size_u,
184 mach_vm_offset_t *start,
185 mach_vm_offset_t *end,
186 mach_vm_size_t *size)
187 {
188 return vm_sanitize_addr_size(start_u, size_u, VM_SANITIZE_CALLER_VM_DEALLOCATE,
189 map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, start,
190 end, size);
191 }
192
193 /*
194 * mach_vm_deallocate -
195 * deallocates the specified range of addresses in the
196 * specified address map.
197 */
198 kern_return_t
mach_vm_deallocate(vm_map_t map,mach_vm_offset_ut start_u,mach_vm_size_ut size_u)199 mach_vm_deallocate(
200 vm_map_t map,
201 mach_vm_offset_ut start_u,
202 mach_vm_size_ut size_u)
203 {
204 mach_vm_offset_t start, end;
205 mach_vm_size_t size;
206 kern_return_t kr;
207
208 if (map == VM_MAP_NULL) {
209 return KERN_INVALID_ARGUMENT;
210 }
211
212 kr = mach_vm_deallocate_sanitize(map,
213 start_u,
214 size_u,
215 &start,
216 &end,
217 &size);
218 if (__improbable(kr != KERN_SUCCESS)) {
219 return vm_sanitize_get_kr(kr);
220 }
221
222 return vm_map_remove_guard(map, start, end,
223 VM_MAP_REMOVE_NO_FLAGS,
224 KMEM_GUARD_NONE).kmr_return;
225 }
226
227 /*
228 * vm_deallocate -
229 * deallocates the specified range of addresses in the
230 * specified address map (limited to addresses the same
231 * size as the kernel).
232 */
233 kern_return_t
vm_deallocate(vm_map_t map,vm_offset_ut start,vm_size_ut size)234 vm_deallocate(
235 vm_map_t map,
236 vm_offset_ut start,
237 vm_size_ut size)
238 {
239 return mach_vm_deallocate(map, start, size);
240 }
241
242 /*
243 * mach_vm_inherit -
244 * Sets the inheritance of the specified range in the
245 * specified map.
246 */
247 kern_return_t
mach_vm_inherit(vm_map_t map,mach_vm_offset_t start,mach_vm_size_t size,vm_inherit_t new_inheritance)248 mach_vm_inherit(
249 vm_map_t map,
250 mach_vm_offset_t start,
251 mach_vm_size_t size,
252 vm_inherit_t new_inheritance)
253 {
254 if ((map == VM_MAP_NULL) || (start + size < start) ||
255 (new_inheritance > VM_INHERIT_LAST_VALID)) {
256 return KERN_INVALID_ARGUMENT;
257 }
258
259 if (size == 0) {
260 return KERN_SUCCESS;
261 }
262
263 return vm_map_inherit(map,
264 vm_map_trunc_page(start,
265 VM_MAP_PAGE_MASK(map)),
266 vm_map_round_page(start + size,
267 VM_MAP_PAGE_MASK(map)),
268 new_inheritance);
269 }
270
271 /*
272 * vm_inherit -
273 * Sets the inheritance of the specified range in the
274 * specified map (range limited to addresses
275 */
276 kern_return_t
vm_inherit(vm_map_t map,vm_offset_t start,vm_size_t size,vm_inherit_t new_inheritance)277 vm_inherit(
278 vm_map_t map,
279 vm_offset_t start,
280 vm_size_t size,
281 vm_inherit_t new_inheritance)
282 {
283 if ((map == VM_MAP_NULL) || (start + size < start) ||
284 (new_inheritance > VM_INHERIT_LAST_VALID)) {
285 return KERN_INVALID_ARGUMENT;
286 }
287
288 if (size == 0) {
289 return KERN_SUCCESS;
290 }
291
292 return vm_map_inherit(map,
293 vm_map_trunc_page(start,
294 VM_MAP_PAGE_MASK(map)),
295 vm_map_round_page(start + size,
296 VM_MAP_PAGE_MASK(map)),
297 new_inheritance);
298 }
299
300 /*
301 * mach_vm_protect -
302 * Sets the protection of the specified range in the
303 * specified map.
304 */
305
306 kern_return_t
mach_vm_protect(vm_map_t map,mach_vm_offset_t start,mach_vm_size_t size,boolean_t set_maximum,vm_prot_t new_protection)307 mach_vm_protect(
308 vm_map_t map,
309 mach_vm_offset_t start,
310 mach_vm_size_t size,
311 boolean_t set_maximum,
312 vm_prot_t new_protection)
313 {
314 if ((map == VM_MAP_NULL) || (start + size < start) ||
315 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) {
316 return KERN_INVALID_ARGUMENT;
317 }
318
319 if (size == 0) {
320 return KERN_SUCCESS;
321 }
322
323 return vm_map_protect(map,
324 vm_map_trunc_page(start,
325 VM_MAP_PAGE_MASK(map)),
326 vm_map_round_page(start + size,
327 VM_MAP_PAGE_MASK(map)),
328 new_protection,
329 set_maximum);
330 }
331
332 /*
333 * vm_protect -
334 * Sets the protection of the specified range in the
335 * specified map. Addressability of the range limited
336 * to the same size as the kernel.
337 */
338
339 kern_return_t
vm_protect(vm_map_t map,vm_offset_t start,vm_size_t size,boolean_t set_maximum,vm_prot_t new_protection)340 vm_protect(
341 vm_map_t map,
342 vm_offset_t start,
343 vm_size_t size,
344 boolean_t set_maximum,
345 vm_prot_t new_protection)
346 {
347 if ((map == VM_MAP_NULL) || (start + size < start) ||
348 (new_protection & ~VM_VALID_VMPROTECT_FLAGS)
349 #if defined(__x86_64__)
350 || ((new_protection & VM_PROT_UEXEC) && !pmap_supported_feature(map->pmap, PMAP_FEAT_UEXEC))
351 #endif
352 ) {
353 return KERN_INVALID_ARGUMENT;
354 }
355
356 if (size == 0) {
357 return KERN_SUCCESS;
358 }
359
360 return vm_map_protect(map,
361 vm_map_trunc_page(start,
362 VM_MAP_PAGE_MASK(map)),
363 vm_map_round_page(start + size,
364 VM_MAP_PAGE_MASK(map)),
365 new_protection,
366 set_maximum);
367 }
368
369 /*
370 * mach_vm_machine_attributes -
371 * Handle machine-specific attributes for a mapping, such
372 * as cachability, migrability, etc.
373 */
374 kern_return_t
mach_vm_machine_attribute(vm_map_t map,mach_vm_address_t addr,mach_vm_size_t size,vm_machine_attribute_t attribute,vm_machine_attribute_val_t * value)375 mach_vm_machine_attribute(
376 vm_map_t map,
377 mach_vm_address_t addr,
378 mach_vm_size_t size,
379 vm_machine_attribute_t attribute,
380 vm_machine_attribute_val_t* value) /* IN/OUT */
381 {
382 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
383 return KERN_INVALID_ARGUMENT;
384 }
385
386 if (size == 0) {
387 return KERN_SUCCESS;
388 }
389
390 return vm_map_machine_attribute(
391 map,
392 vm_map_trunc_page(addr,
393 VM_MAP_PAGE_MASK(map)),
394 vm_map_round_page(addr + size,
395 VM_MAP_PAGE_MASK(map)),
396 attribute,
397 value);
398 }
399
400 /*
401 * vm_machine_attribute -
402 * Handle machine-specific attributes for a mapping, such
403 * as cachability, migrability, etc. Limited addressability
404 * (same range limits as for the native kernel map).
405 */
406 kern_return_t
vm_machine_attribute(vm_map_t map,vm_address_t addr,vm_size_t size,vm_machine_attribute_t attribute,vm_machine_attribute_val_t * value)407 vm_machine_attribute(
408 vm_map_t map,
409 vm_address_t addr,
410 vm_size_t size,
411 vm_machine_attribute_t attribute,
412 vm_machine_attribute_val_t* value) /* IN/OUT */
413 {
414 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
415 return KERN_INVALID_ARGUMENT;
416 }
417
418 if (size == 0) {
419 return KERN_SUCCESS;
420 }
421
422 return vm_map_machine_attribute(
423 map,
424 vm_map_trunc_page(addr,
425 VM_MAP_PAGE_MASK(map)),
426 vm_map_round_page(addr + size,
427 VM_MAP_PAGE_MASK(map)),
428 attribute,
429 value);
430 }
431
432 /*
433 * mach_vm_read -
434 * Read/copy a range from one address space and return it to the caller.
435 *
436 * It is assumed that the address for the returned memory is selected by
437 * the IPC implementation as part of receiving the reply to this call.
438 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
439 * that gets returned.
440 *
441 * JMM - because of mach_msg_type_number_t, this call is limited to a
442 * single 4GB region at this time.
443 *
444 */
445 kern_return_t
mach_vm_read(vm_map_t map,mach_vm_address_ut addr,mach_vm_size_ut size,pointer_t * data,mach_msg_type_number_t * data_size)446 mach_vm_read(
447 vm_map_t map,
448 mach_vm_address_ut addr,
449 mach_vm_size_ut size,
450 pointer_t *data,
451 mach_msg_type_number_t *data_size)
452 {
453 kern_return_t error;
454 vm_map_copy_t ipc_address;
455
456 if (map == VM_MAP_NULL) {
457 return KERN_INVALID_ARGUMENT;
458 }
459
460 /*
461 * mach_msg_type_number_t is a signed int,
462 * make sure we do not overflow it.
463 */
464 if (!VM_SANITIZE_UNSAFE_FITS(size, mach_msg_type_number_t)) {
465 return KERN_INVALID_ARGUMENT;
466 }
467
468 error = vm_map_copyin(map, addr, size, FALSE, &ipc_address);
469
470 if (KERN_SUCCESS == error) {
471 *data = (pointer_t) ipc_address;
472 *data_size = (mach_msg_type_number_t)VM_SANITIZE_UNSAFE_UNWRAP(size);
473 }
474 return error;
475 }
476
477 /*
478 * vm_read -
479 * Read/copy a range from one address space and return it to the caller.
480 * Limited addressability (same range limits as for the native kernel map).
481 *
482 * It is assumed that the address for the returned memory is selected by
483 * the IPC implementation as part of receiving the reply to this call.
484 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
485 * that gets returned.
486 */
487 kern_return_t
vm_read(vm_map_t map,vm_address_ut addr,vm_size_ut size,pointer_t * data,mach_msg_type_number_t * data_size)488 vm_read(
489 vm_map_t map,
490 vm_address_ut addr,
491 vm_size_ut size,
492 pointer_t *data,
493 mach_msg_type_number_t *data_size)
494 {
495 return mach_vm_read(map, addr, size, data, data_size);
496 }
497
498 /*
499 * mach_vm_read_list -
500 * Read/copy a list of address ranges from specified map.
501 *
502 * MIG does not know how to deal with a returned array of
503 * vm_map_copy_t structures, so we have to do the copyout
504 * manually here.
505 */
506 kern_return_t
mach_vm_read_list(vm_map_t map,mach_vm_read_entry_t data_list,natural_t count)507 mach_vm_read_list(
508 vm_map_t map,
509 mach_vm_read_entry_t data_list,
510 natural_t count)
511 {
512 mach_msg_type_number_t i;
513 kern_return_t error;
514 vm_map_copy_t copy;
515
516 if (map == VM_MAP_NULL ||
517 count > VM_MAP_ENTRY_MAX) {
518 return KERN_INVALID_ARGUMENT;
519 }
520
521 error = KERN_SUCCESS;
522 for (i = 0; i < count; i++) {
523 vm_map_address_t map_addr;
524 vm_map_size_t map_size;
525
526 map_addr = (vm_map_address_t)(data_list[i].address);
527 map_size = (vm_map_size_t)(data_list[i].size);
528
529 if (map_size != 0) {
530 error = vm_map_copyin(map,
531 map_addr,
532 map_size,
533 FALSE, /* src_destroy */
534 ©);
535 if (KERN_SUCCESS == error) {
536 error = vm_map_copyout(
537 current_task()->map,
538 &map_addr,
539 copy);
540 if (KERN_SUCCESS == error) {
541 data_list[i].address = map_addr;
542 continue;
543 }
544 vm_map_copy_discard(copy);
545 }
546 }
547 data_list[i].address = (mach_vm_address_t)0;
548 data_list[i].size = (mach_vm_size_t)0;
549 }
550 return error;
551 }
552
553 /*
554 * vm_read_list -
555 * Read/copy a list of address ranges from specified map.
556 *
557 * MIG does not know how to deal with a returned array of
558 * vm_map_copy_t structures, so we have to do the copyout
559 * manually here.
560 *
561 * The source and destination ranges are limited to those
562 * that can be described with a vm_address_t (i.e. same
563 * size map as the kernel).
564 *
565 * JMM - If the result of the copyout is an address range
566 * that cannot be described with a vm_address_t (i.e. the
567 * caller had a larger address space but used this call
568 * anyway), it will result in a truncated address being
569 * returned (and a likely confused caller).
570 */
571
572 kern_return_t
vm_read_list(vm_map_t map,vm_read_entry_t data_list,natural_t count)573 vm_read_list(
574 vm_map_t map,
575 vm_read_entry_t data_list,
576 natural_t count)
577 {
578 mach_msg_type_number_t i;
579 kern_return_t error;
580 vm_map_copy_t copy;
581
582 if (map == VM_MAP_NULL ||
583 count > VM_MAP_ENTRY_MAX) {
584 return KERN_INVALID_ARGUMENT;
585 }
586
587 error = KERN_SUCCESS;
588 for (i = 0; i < count; i++) {
589 vm_map_address_t map_addr;
590 vm_map_size_t map_size;
591
592 map_addr = (vm_map_address_t)(data_list[i].address);
593 map_size = (vm_map_size_t)(data_list[i].size);
594
595 if (map_size != 0) {
596 error = vm_map_copyin(map,
597 map_addr,
598 map_size,
599 FALSE, /* src_destroy */
600 ©);
601 if (KERN_SUCCESS == error) {
602 error = vm_map_copyout(current_task()->map,
603 &map_addr,
604 copy);
605 if (KERN_SUCCESS == error) {
606 data_list[i].address =
607 CAST_DOWN(vm_offset_t, map_addr);
608 continue;
609 }
610 vm_map_copy_discard(copy);
611 }
612 }
613 data_list[i].address = (mach_vm_address_t)0;
614 data_list[i].size = (mach_vm_size_t)0;
615 }
616 return error;
617 }
618
619 /*
620 * mach_vm_read_overwrite -
621 * Overwrite a range of the current map with data from the specified
622 * map/address range.
623 *
624 * In making an assumption that the current thread is local, it is
625 * no longer cluster-safe without a fully supportive local proxy
626 * thread/task (but we don't support cluster's anymore so this is moot).
627 */
628
629 kern_return_t
mach_vm_read_overwrite(vm_map_t map,mach_vm_address_ut address,mach_vm_size_ut size,mach_vm_address_ut data,mach_vm_size_ut * data_size)630 mach_vm_read_overwrite(
631 vm_map_t map,
632 mach_vm_address_ut address,
633 mach_vm_size_ut size,
634 mach_vm_address_ut data,
635 mach_vm_size_ut *data_size)
636 {
637 kern_return_t error;
638 vm_map_copy_t copy;
639
640 if (map == VM_MAP_NULL) {
641 return KERN_INVALID_ARGUMENT;
642 }
643
644 error = vm_map_copyin(map, address, size, FALSE, ©);
645
646 if (KERN_SUCCESS == error) {
647 if (copy) {
648 assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
649 }
650
651 error = vm_map_copy_overwrite(current_thread()->map,
652 data, copy, size, FALSE);
653 if (KERN_SUCCESS == error) {
654 *data_size = size;
655 return error;
656 }
657 vm_map_copy_discard(copy);
658 }
659 return error;
660 }
661
662 /*
663 * vm_read_overwrite -
664 * Overwrite a range of the current map with data from the specified
665 * map/address range.
666 *
667 * This routine adds the additional limitation that the source and
668 * destination ranges must be describable with vm_address_t values
669 * (i.e. the same size address spaces as the kernel, or at least the
670 * the ranges are in that first portion of the respective address
671 * spaces).
672 */
673
674 kern_return_t
vm_read_overwrite(vm_map_t map,vm_address_ut address,vm_size_ut size,vm_address_ut data,vm_size_ut * data_size)675 vm_read_overwrite(
676 vm_map_t map,
677 vm_address_ut address,
678 vm_size_ut size,
679 vm_address_ut data,
680 vm_size_ut *data_size)
681 {
682 return mach_vm_read_overwrite(map, address, size, data, data_size);
683 }
684
685
686 /*
687 * mach_vm_write -
688 * Overwrite the specified address range with the data provided
689 * (from the current map).
690 */
691 kern_return_t
mach_vm_write(vm_map_t map,mach_vm_address_ut address,pointer_t data,mach_msg_type_number_t size)692 mach_vm_write(
693 vm_map_t map,
694 mach_vm_address_ut address,
695 pointer_t data,
696 mach_msg_type_number_t size)
697 {
698 if (map == VM_MAP_NULL) {
699 return KERN_INVALID_ARGUMENT;
700 }
701
702 return vm_map_copy_overwrite(map, address,
703 (vm_map_copy_t) data, size, FALSE /* interruptible XXX */);
704 }
705
706 /*
707 * vm_write -
708 * Overwrite the specified address range with the data provided
709 * (from the current map).
710 *
711 * The addressability of the range of addresses to overwrite is
712 * limited bu the use of a vm_address_t (same size as kernel map).
713 * Either the target map is also small, or the range is in the
714 * low addresses within it.
715 */
716 kern_return_t
vm_write(vm_map_t map,vm_address_ut address,pointer_t data,mach_msg_type_number_t size)717 vm_write(
718 vm_map_t map,
719 vm_address_ut address,
720 pointer_t data,
721 mach_msg_type_number_t size)
722 {
723 return mach_vm_write(map, address, data, size);
724 }
725
726 /*
727 * mach_vm_copy -
728 * Overwrite one range of the specified map with the contents of
729 * another range within that same map (i.e. both address ranges
730 * are "over there").
731 */
732 kern_return_t
mach_vm_copy(vm_map_t map,mach_vm_address_ut source_address,mach_vm_size_ut size,mach_vm_address_ut dest_address)733 mach_vm_copy(
734 vm_map_t map,
735 mach_vm_address_ut source_address,
736 mach_vm_size_ut size,
737 mach_vm_address_ut dest_address)
738 {
739 vm_map_copy_t copy;
740 kern_return_t kr;
741
742 if (map == VM_MAP_NULL) {
743 return KERN_INVALID_ARGUMENT;
744 }
745
746 kr = vm_map_copyin(map, source_address, size, FALSE, ©);
747
748 if (KERN_SUCCESS == kr) {
749 if (copy) {
750 assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
751 }
752
753 kr = vm_map_copy_overwrite(map, dest_address,
754 copy, size, FALSE /* interruptible XXX */);
755
756 if (KERN_SUCCESS != kr) {
757 vm_map_copy_discard(copy);
758 }
759 }
760 return kr;
761 }
762
763 kern_return_t
vm_copy(vm_map_t map,vm_address_ut source_address,vm_size_ut size,vm_address_ut dest_address)764 vm_copy(
765 vm_map_t map,
766 vm_address_ut source_address,
767 vm_size_ut size,
768 vm_address_ut dest_address)
769 {
770 return mach_vm_copy(map, source_address, size, dest_address);
771 }
772
773 /*
774 * mach_vm_map -
775 * Map some range of an object into an address space.
776 *
777 * The object can be one of several types of objects:
778 * NULL - anonymous memory
779 * a named entry - a range within another address space
780 * or a range within a memory object
781 * a whole memory object
782 *
783 */
784 kern_return_t
mach_vm_map_external(vm_map_t target_map,mach_vm_offset_ut * address,mach_vm_size_ut initial_size,mach_vm_offset_ut mask,int flags,ipc_port_t port,memory_object_offset_ut offset,boolean_t copy,vm_prot_ut cur_protection,vm_prot_ut max_protection,vm_inherit_ut inheritance)785 mach_vm_map_external(
786 vm_map_t target_map,
787 mach_vm_offset_ut *address,
788 mach_vm_size_ut initial_size,
789 mach_vm_offset_ut mask,
790 int flags,
791 ipc_port_t port,
792 memory_object_offset_ut offset,
793 boolean_t copy,
794 vm_prot_ut cur_protection,
795 vm_prot_ut max_protection,
796 vm_inherit_ut inheritance)
797 {
798 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
799
800 /* filter out any kernel-only flags */
801 if (flags & ~VM_FLAGS_USER_MAP) {
802 return KERN_INVALID_ARGUMENT;
803 }
804
805 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
806 /* range_id is set by mach_vm_map_kernel */
807 return mach_vm_map_kernel(target_map, address, initial_size, mask,
808 vmk_flags, port, offset, copy,
809 cur_protection, max_protection,
810 inheritance);
811 }
812
813 /* legacy interface */
814 __attribute__((always_inline))
815 kern_return_t
vm_map_64_external(vm_map_t target_map,vm_offset_ut * address,vm_size_ut size,vm_offset_ut mask,int flags,ipc_port_t port,memory_object_offset_ut offset,boolean_t copy,vm_prot_ut cur_protection,vm_prot_ut max_protection,vm_inherit_ut inheritance)816 vm_map_64_external(
817 vm_map_t target_map,
818 vm_offset_ut *address,
819 vm_size_ut size,
820 vm_offset_ut mask,
821 int flags,
822 ipc_port_t port,
823 memory_object_offset_ut offset,
824 boolean_t copy,
825 vm_prot_ut cur_protection,
826 vm_prot_ut max_protection,
827 vm_inherit_ut inheritance)
828 {
829 return mach_vm_map_external(target_map, address,
830 size, mask, flags, port, offset, copy,
831 cur_protection, max_protection, inheritance);
832 }
833
834 /* temporary, until world build */
835 __attribute__((always_inline))
836 kern_return_t
vm_map_external(vm_map_t target_map,vm_offset_ut * address,vm_size_ut size,vm_offset_ut mask,int flags,ipc_port_t port,vm_offset_ut offset,boolean_t copy,vm_prot_ut cur_protection,vm_prot_ut max_protection,vm_inherit_ut inheritance)837 vm_map_external(
838 vm_map_t target_map,
839 vm_offset_ut *address,
840 vm_size_ut size,
841 vm_offset_ut mask,
842 int flags,
843 ipc_port_t port,
844 vm_offset_ut offset,
845 boolean_t copy,
846 vm_prot_ut cur_protection,
847 vm_prot_ut max_protection,
848 vm_inherit_ut inheritance)
849 {
850 return mach_vm_map_external(target_map, address,
851 size, mask, flags, port, offset, copy,
852 cur_protection, max_protection, inheritance);
853 }
854
855 static inline kern_return_t
mach_vm_remap_new_external_sanitize(vm_map_t target_map,vm_prot_ut cur_protection_u,vm_prot_ut max_protection_u,vm_prot_t * cur_protection,vm_prot_t * max_protection)856 mach_vm_remap_new_external_sanitize(
857 vm_map_t target_map,
858 vm_prot_ut cur_protection_u,
859 vm_prot_ut max_protection_u,
860 vm_prot_t *cur_protection,
861 vm_prot_t *max_protection)
862 {
863 return vm_sanitize_cur_and_max_prots(cur_protection_u, max_protection_u,
864 VM_SANITIZE_CALLER_VM_MAP_REMAP, target_map,
865 cur_protection, max_protection);
866 }
867
868 /*
869 * mach_vm_remap_new -
870 * Behaves like mach_vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
871 * and {cur,max}_protection are in/out.
872 */
873 kern_return_t
mach_vm_remap_new_external(vm_map_t target_map,mach_vm_offset_ut * address,mach_vm_size_ut size,mach_vm_offset_ut mask,int flags,mach_port_t src_tport,mach_vm_offset_ut memory_address,boolean_t copy,vm_prot_ut * cur_protection_u,vm_prot_ut * max_protection_u,vm_inherit_ut inheritance)874 mach_vm_remap_new_external(
875 vm_map_t target_map,
876 mach_vm_offset_ut *address,
877 mach_vm_size_ut size,
878 mach_vm_offset_ut mask,
879 int flags,
880 mach_port_t src_tport,
881 mach_vm_offset_ut memory_address,
882 boolean_t copy,
883 vm_prot_ut *cur_protection_u, /* IN/OUT */
884 vm_prot_ut *max_protection_u, /* IN/OUT */
885 vm_inherit_ut inheritance)
886 {
887 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
888 vm_map_t src_map;
889 vm_prot_t cur_protection, max_protection;
890 kern_return_t kr;
891
892 if (target_map == VM_MAP_NULL) {
893 return KERN_INVALID_ARGUMENT;
894 }
895
896 /* filter out any kernel-only flags */
897 if (flags & ~VM_FLAGS_USER_REMAP) {
898 return KERN_INVALID_ARGUMENT;
899 }
900
901 vm_map_kernel_flags_set_vmflags(&vmk_flags,
902 flags | VM_FLAGS_RETURN_DATA_ADDR);
903
904 /*
905 * We don't need cur_protection here, but sanitizing it before
906 * enforcing W^X below matches historical error codes better.
907 */
908 kr = mach_vm_remap_new_external_sanitize(target_map,
909 *cur_protection_u,
910 *max_protection_u,
911 &cur_protection,
912 &max_protection);
913 if (__improbable(kr != KERN_SUCCESS)) {
914 return vm_sanitize_get_kr(kr);
915 }
916
917 if ((max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
918 (VM_PROT_WRITE | VM_PROT_EXECUTE)) {
919 /*
920 * XXX FBDP TODO
921 * enforce target's "wx" policies
922 */
923 return KERN_PROTECTION_FAILURE;
924 }
925
926 if (copy || max_protection == VM_PROT_READ || max_protection == VM_PROT_NONE) {
927 src_map = convert_port_to_map_read(src_tport);
928 } else {
929 src_map = convert_port_to_map(src_tport);
930 }
931
932 /* range_id is set by vm_map_remap */
933 kr = vm_map_remap(target_map,
934 address,
935 size,
936 mask,
937 vmk_flags,
938 src_map,
939 memory_address,
940 copy,
941 cur_protection_u, /* IN/OUT */
942 max_protection_u, /* IN/OUT */
943 inheritance);
944
945 vm_map_deallocate(src_map);
946
947 if (kr == KERN_SUCCESS) {
948 ipc_port_release_send(src_tport); /* consume on success */
949 }
950 return kr;
951 }
952
953 /*
954 * mach_vm_remap -
955 * Remap a range of memory from one task into another,
956 * to another address range within the same task, or
957 * over top of itself (with altered permissions and/or
958 * as an in-place copy of itself).
959 */
960 kern_return_t
mach_vm_remap_external(vm_map_t target_map,mach_vm_offset_ut * address,mach_vm_size_ut size,mach_vm_offset_ut mask,int flags,vm_map_t src_map,mach_vm_offset_ut memory_address,boolean_t copy,vm_prot_ut * cur_protection,vm_prot_ut * max_protection,vm_inherit_ut inheritance)961 mach_vm_remap_external(
962 vm_map_t target_map,
963 mach_vm_offset_ut *address,
964 mach_vm_size_ut size,
965 mach_vm_offset_ut mask,
966 int flags,
967 vm_map_t src_map,
968 mach_vm_offset_ut memory_address,
969 boolean_t copy,
970 vm_prot_ut *cur_protection, /* OUT */
971 vm_prot_ut *max_protection, /* OUT */
972 vm_inherit_ut inheritance)
973 {
974 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
975
976 /* filter out any kernel-only flags */
977 if (flags & ~VM_FLAGS_USER_REMAP) {
978 return KERN_INVALID_ARGUMENT;
979 }
980
981 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
982
983 *cur_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
984 *max_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
985
986 /* range_id is set by vm_map_remap */
987 return vm_map_remap(target_map,
988 address,
989 size,
990 mask,
991 vmk_flags,
992 src_map,
993 memory_address,
994 copy,
995 cur_protection,
996 max_protection,
997 inheritance);
998 }
999
1000 /*
1001 * vm_remap_new -
1002 * Behaves like vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
1003 * and {cur,max}_protection are in/out.
1004 */
1005 kern_return_t
vm_remap_new_external(vm_map_t target_map,vm_offset_ut * address,vm_size_ut size,vm_offset_ut mask,int flags,mach_port_t src_tport,vm_offset_ut memory_address,boolean_t copy,vm_prot_ut * cur_protection,vm_prot_ut * max_protection,vm_inherit_ut inheritance)1006 vm_remap_new_external(
1007 vm_map_t target_map,
1008 vm_offset_ut *address,
1009 vm_size_ut size,
1010 vm_offset_ut mask,
1011 int flags,
1012 mach_port_t src_tport,
1013 vm_offset_ut memory_address,
1014 boolean_t copy,
1015 vm_prot_ut *cur_protection, /* IN/OUT */
1016 vm_prot_ut *max_protection, /* IN/OUT */
1017 vm_inherit_ut inheritance)
1018 {
1019 return mach_vm_remap_new_external(target_map,
1020 address,
1021 size,
1022 mask,
1023 flags,
1024 src_tport,
1025 memory_address,
1026 copy,
1027 cur_protection, /* IN/OUT */
1028 max_protection, /* IN/OUT */
1029 inheritance);
1030 }
1031
1032 /*
1033 * vm_remap -
1034 * Remap a range of memory from one task into another,
1035 * to another address range within the same task, or
1036 * over top of itself (with altered permissions and/or
1037 * as an in-place copy of itself).
1038 *
1039 * The addressability of the source and target address
1040 * range is limited by the size of vm_address_t (in the
1041 * kernel context).
1042 */
1043 kern_return_t
vm_remap_external(vm_map_t target_map,vm_offset_ut * address,vm_size_ut size,vm_offset_ut mask,int flags,vm_map_t src_map,vm_offset_ut memory_address,boolean_t copy,vm_prot_ut * cur_protection,vm_prot_ut * max_protection,vm_inherit_ut inheritance)1044 vm_remap_external(
1045 vm_map_t target_map,
1046 vm_offset_ut *address,
1047 vm_size_ut size,
1048 vm_offset_ut mask,
1049 int flags,
1050 vm_map_t src_map,
1051 vm_offset_ut memory_address,
1052 boolean_t copy,
1053 vm_prot_ut *cur_protection, /* OUT */
1054 vm_prot_ut *max_protection, /* OUT */
1055 vm_inherit_ut inheritance)
1056 {
1057 return mach_vm_remap_external(target_map, address,
1058 size, mask, flags, src_map, memory_address, copy,
1059 cur_protection, max_protection, inheritance);
1060 }
1061
1062 /*
1063 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1064 * when mach_vm_wire and vm_wire are changed to use ledgers.
1065 */
1066 #include <mach/mach_host_server.h>
1067 /*
1068 * mach_vm_wire
1069 * Specify that the range of the virtual address space
1070 * of the target task must not cause page faults for
1071 * the indicated accesses.
1072 *
1073 * [ To unwire the pages, specify VM_PROT_NONE. ]
1074 */
1075 kern_return_t
mach_vm_wire_external(host_priv_t host_priv,vm_map_t map,mach_vm_address_ut start,mach_vm_size_ut size,vm_prot_ut access)1076 mach_vm_wire_external(
1077 host_priv_t host_priv,
1078 vm_map_t map,
1079 mach_vm_address_ut start,
1080 mach_vm_size_ut size,
1081 vm_prot_ut access)
1082 {
1083 kern_return_t rc;
1084 mach_vm_offset_ut end;
1085
1086 if (host_priv == HOST_PRIV_NULL) {
1087 return KERN_INVALID_HOST;
1088 }
1089
1090 if (map == VM_MAP_NULL) {
1091 return KERN_INVALID_TASK;
1092 }
1093
1094 end = vm_sanitize_compute_unsafe_end(start, size);
1095 if (VM_SANITIZE_UNSAFE_IS_ZERO(access)) {
1096 rc = vm_map_unwire_impl(map, start, end, true,
1097 VM_SANITIZE_CALLER_VM_UNWIRE_USER);
1098 } else {
1099 rc = vm_map_wire_impl(map, start, end, access,
1100 VM_KERN_MEMORY_MLOCK, true, NULL, VM_SANITIZE_CALLER_VM_WIRE_USER);
1101 }
1102
1103 return rc;
1104 }
1105
1106 /*
1107 * vm_wire -
1108 * Specify that the range of the virtual address space
1109 * of the target task must not cause page faults for
1110 * the indicated accesses.
1111 *
1112 * [ To unwire the pages, specify VM_PROT_NONE. ]
1113 */
1114 kern_return_t
vm_wire(host_priv_t host_priv,vm_map_t map,vm_offset_ut start,vm_size_ut size,vm_prot_ut access)1115 vm_wire(
1116 host_priv_t host_priv,
1117 vm_map_t map,
1118 vm_offset_ut start,
1119 vm_size_ut size,
1120 vm_prot_ut access)
1121 {
1122 return mach_vm_wire_external(host_priv, map, start, size, access);
1123 }
1124
1125 /*
1126 * vm_msync
1127 *
1128 * Synchronises the memory range specified with its backing store
1129 * image by either flushing or cleaning the contents to the appropriate
1130 * memory manager.
1131 *
1132 * interpretation of sync_flags
1133 * VM_SYNC_INVALIDATE - discard pages, only return precious
1134 * pages to manager.
1135 *
1136 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1137 * - discard pages, write dirty or precious
1138 * pages back to memory manager.
1139 *
1140 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1141 * - write dirty or precious pages back to
1142 * the memory manager.
1143 *
1144 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1145 * is a hole in the region, and we would
1146 * have returned KERN_SUCCESS, return
1147 * KERN_INVALID_ADDRESS instead.
1148 *
1149 * RETURNS
1150 * KERN_INVALID_TASK Bad task parameter
1151 * KERN_INVALID_ARGUMENT both sync and async were specified.
1152 * KERN_SUCCESS The usual.
1153 * KERN_INVALID_ADDRESS There was a hole in the region.
1154 */
1155
1156 kern_return_t
mach_vm_msync(vm_map_t map,mach_vm_address_t address,mach_vm_size_t size,vm_sync_t sync_flags)1157 mach_vm_msync(
1158 vm_map_t map,
1159 mach_vm_address_t address,
1160 mach_vm_size_t size,
1161 vm_sync_t sync_flags)
1162 {
1163 if (map == VM_MAP_NULL) {
1164 return KERN_INVALID_TASK;
1165 }
1166
1167 return vm_map_msync(map, (vm_map_address_t)address,
1168 (vm_map_size_t)size, sync_flags);
1169 }
1170
1171 /*
1172 * vm_msync
1173 *
1174 * Synchronises the memory range specified with its backing store
1175 * image by either flushing or cleaning the contents to the appropriate
1176 * memory manager.
1177 *
1178 * interpretation of sync_flags
1179 * VM_SYNC_INVALIDATE - discard pages, only return precious
1180 * pages to manager.
1181 *
1182 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1183 * - discard pages, write dirty or precious
1184 * pages back to memory manager.
1185 *
1186 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1187 * - write dirty or precious pages back to
1188 * the memory manager.
1189 *
1190 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1191 * is a hole in the region, and we would
1192 * have returned KERN_SUCCESS, return
1193 * KERN_INVALID_ADDRESS instead.
1194 *
1195 * The addressability of the range is limited to that which can
1196 * be described by a vm_address_t.
1197 *
1198 * RETURNS
1199 * KERN_INVALID_TASK Bad task parameter
1200 * KERN_INVALID_ARGUMENT both sync and async were specified.
1201 * KERN_SUCCESS The usual.
1202 * KERN_INVALID_ADDRESS There was a hole in the region.
1203 */
1204
1205 kern_return_t
vm_msync(vm_map_t map,vm_address_t address,vm_size_t size,vm_sync_t sync_flags)1206 vm_msync(
1207 vm_map_t map,
1208 vm_address_t address,
1209 vm_size_t size,
1210 vm_sync_t sync_flags)
1211 {
1212 if (map == VM_MAP_NULL) {
1213 return KERN_INVALID_TASK;
1214 }
1215
1216 return vm_map_msync(map, (vm_map_address_t)address,
1217 (vm_map_size_t)size, sync_flags);
1218 }
1219
1220
1221 int
vm_toggle_entry_reuse(int toggle,int * old_value)1222 vm_toggle_entry_reuse(int toggle, int *old_value)
1223 {
1224 vm_map_t map = current_map();
1225
1226 assert(!map->is_nested_map);
1227 if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) {
1228 *old_value = map->disable_vmentry_reuse;
1229 } else if (toggle == VM_TOGGLE_SET) {
1230 vm_map_entry_t map_to_entry;
1231
1232 vm_map_lock(map);
1233 vm_map_disable_hole_optimization(map);
1234 map->disable_vmentry_reuse = TRUE;
1235 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1236 if (map->first_free == map_to_entry) {
1237 map->highest_entry_end = vm_map_min(map);
1238 } else {
1239 map->highest_entry_end = map->first_free->vme_end;
1240 }
1241 vm_map_unlock(map);
1242 } else if (toggle == VM_TOGGLE_CLEAR) {
1243 vm_map_lock(map);
1244 map->disable_vmentry_reuse = FALSE;
1245 vm_map_unlock(map);
1246 } else {
1247 return KERN_INVALID_ARGUMENT;
1248 }
1249
1250 return KERN_SUCCESS;
1251 }
1252
1253 /*
1254 * mach_vm_behavior_set
1255 *
1256 * Sets the paging behavior attribute for the specified range
1257 * in the specified map.
1258 *
1259 * This routine will fail with KERN_INVALID_ADDRESS if any address
1260 * in [start,start+size) is not a valid allocated memory region.
1261 */
1262 kern_return_t
mach_vm_behavior_set(vm_map_t map,mach_vm_offset_t start,mach_vm_size_t size,vm_behavior_t new_behavior)1263 mach_vm_behavior_set(
1264 vm_map_t map,
1265 mach_vm_offset_t start,
1266 mach_vm_size_t size,
1267 vm_behavior_t new_behavior)
1268 {
1269 vm_map_offset_t align_mask;
1270
1271 if ((map == VM_MAP_NULL) || (start + size < start)) {
1272 return KERN_INVALID_ARGUMENT;
1273 }
1274
1275 if (size == 0) {
1276 return KERN_SUCCESS;
1277 }
1278
1279 switch (new_behavior) {
1280 case VM_BEHAVIOR_REUSABLE:
1281 case VM_BEHAVIOR_REUSE:
1282 case VM_BEHAVIOR_CAN_REUSE:
1283 case VM_BEHAVIOR_ZERO:
1284 /*
1285 * Align to the hardware page size, to allow
1286 * malloc() to maximize the amount of re-usability,
1287 * even on systems with larger software page size.
1288 */
1289 align_mask = PAGE_MASK;
1290 break;
1291 default:
1292 align_mask = VM_MAP_PAGE_MASK(map);
1293 break;
1294 }
1295
1296 return vm_map_behavior_set(map,
1297 vm_map_trunc_page(start, align_mask),
1298 vm_map_round_page(start + size, align_mask),
1299 new_behavior);
1300 }
1301
1302 /*
1303 * vm_behavior_set
1304 *
1305 * Sets the paging behavior attribute for the specified range
1306 * in the specified map.
1307 *
1308 * This routine will fail with KERN_INVALID_ADDRESS if any address
1309 * in [start,start+size) is not a valid allocated memory region.
1310 *
1311 * This routine is potentially limited in addressibility by the
1312 * use of vm_offset_t (if the map provided is larger than the
1313 * kernel's).
1314 */
1315 kern_return_t
vm_behavior_set(vm_map_t map,vm_offset_t start,vm_size_t size,vm_behavior_t new_behavior)1316 vm_behavior_set(
1317 vm_map_t map,
1318 vm_offset_t start,
1319 vm_size_t size,
1320 vm_behavior_t new_behavior)
1321 {
1322 if (start + size < start) {
1323 return KERN_INVALID_ARGUMENT;
1324 }
1325
1326 return mach_vm_behavior_set(map,
1327 (mach_vm_offset_t) start,
1328 (mach_vm_size_t) size,
1329 new_behavior);
1330 }
1331
1332 /*
1333 * mach_vm_region:
1334 *
1335 * User call to obtain information about a region in
1336 * a task's address map. Currently, only one flavor is
1337 * supported.
1338 *
1339 * XXX The reserved and behavior fields cannot be filled
1340 * in until the vm merge from the IK is completed, and
1341 * vm_reserve is implemented.
1342 *
1343 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1344 */
1345
1346 kern_return_t
mach_vm_region(vm_map_t map,mach_vm_offset_t * address,mach_vm_size_t * size,vm_region_flavor_t flavor,vm_region_info_t info,mach_msg_type_number_t * count,mach_port_t * object_name)1347 mach_vm_region(
1348 vm_map_t map,
1349 mach_vm_offset_t *address, /* IN/OUT */
1350 mach_vm_size_t *size, /* OUT */
1351 vm_region_flavor_t flavor, /* IN */
1352 vm_region_info_t info, /* OUT */
1353 mach_msg_type_number_t *count, /* IN/OUT */
1354 mach_port_t *object_name) /* OUT */
1355 {
1356 vm_map_offset_t map_addr;
1357 vm_map_size_t map_size;
1358 kern_return_t kr;
1359
1360 if (VM_MAP_NULL == map) {
1361 return KERN_INVALID_ARGUMENT;
1362 }
1363
1364 map_addr = (vm_map_offset_t)*address;
1365 map_size = (vm_map_size_t)*size;
1366
1367 /* legacy conversion */
1368 if (VM_REGION_BASIC_INFO == flavor) {
1369 flavor = VM_REGION_BASIC_INFO_64;
1370 }
1371
1372 kr = vm_map_region(map,
1373 &map_addr, &map_size,
1374 flavor, info, count,
1375 object_name);
1376
1377 *address = map_addr;
1378 *size = map_size;
1379 return kr;
1380 }
1381
1382 /*
1383 * vm_region_64 and vm_region:
1384 *
1385 * User call to obtain information about a region in
1386 * a task's address map. Currently, only one flavor is
1387 * supported.
1388 *
1389 * XXX The reserved and behavior fields cannot be filled
1390 * in until the vm merge from the IK is completed, and
1391 * vm_reserve is implemented.
1392 *
1393 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1394 */
1395
1396 kern_return_t
vm_region_64(vm_map_t map,vm_offset_t * address,vm_size_t * size,vm_region_flavor_t flavor,vm_region_info_t info,mach_msg_type_number_t * count,mach_port_t * object_name)1397 vm_region_64(
1398 vm_map_t map,
1399 vm_offset_t *address, /* IN/OUT */
1400 vm_size_t *size, /* OUT */
1401 vm_region_flavor_t flavor, /* IN */
1402 vm_region_info_t info, /* OUT */
1403 mach_msg_type_number_t *count, /* IN/OUT */
1404 mach_port_t *object_name) /* OUT */
1405 {
1406 vm_map_offset_t map_addr;
1407 vm_map_size_t map_size;
1408 kern_return_t kr;
1409
1410 if (VM_MAP_NULL == map) {
1411 return KERN_INVALID_ARGUMENT;
1412 }
1413
1414 map_addr = (vm_map_offset_t)*address;
1415 map_size = (vm_map_size_t)*size;
1416
1417 /* legacy conversion */
1418 if (VM_REGION_BASIC_INFO == flavor) {
1419 flavor = VM_REGION_BASIC_INFO_64;
1420 }
1421
1422 kr = vm_map_region(map,
1423 &map_addr, &map_size,
1424 flavor, info, count,
1425 object_name);
1426
1427 *address = CAST_DOWN(vm_offset_t, map_addr);
1428 *size = CAST_DOWN(vm_size_t, map_size);
1429
1430 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1431 return KERN_INVALID_ADDRESS;
1432 }
1433 return kr;
1434 }
1435
1436 kern_return_t
vm_region(vm_map_t map,vm_address_t * address,vm_size_t * size,vm_region_flavor_t flavor,vm_region_info_t info,mach_msg_type_number_t * count,mach_port_t * object_name)1437 vm_region(
1438 vm_map_t map,
1439 vm_address_t *address, /* IN/OUT */
1440 vm_size_t *size, /* OUT */
1441 vm_region_flavor_t flavor, /* IN */
1442 vm_region_info_t info, /* OUT */
1443 mach_msg_type_number_t *count, /* IN/OUT */
1444 mach_port_t *object_name) /* OUT */
1445 {
1446 vm_map_address_t map_addr;
1447 vm_map_size_t map_size;
1448 kern_return_t kr;
1449
1450 if (VM_MAP_NULL == map) {
1451 return KERN_INVALID_ARGUMENT;
1452 }
1453
1454 map_addr = (vm_map_address_t)*address;
1455 map_size = (vm_map_size_t)*size;
1456
1457 kr = vm_map_region(map,
1458 &map_addr, &map_size,
1459 flavor, info, count,
1460 object_name);
1461
1462 *address = CAST_DOWN(vm_address_t, map_addr);
1463 *size = CAST_DOWN(vm_size_t, map_size);
1464
1465 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1466 return KERN_INVALID_ADDRESS;
1467 }
1468 return kr;
1469 }
1470
1471 /*
1472 * vm_region_recurse: A form of vm_region which follows the
1473 * submaps in a target map
1474 *
1475 */
1476 kern_return_t
mach_vm_region_recurse(vm_map_t map,mach_vm_address_t * address,mach_vm_size_t * size,uint32_t * depth,vm_region_recurse_info_t info,mach_msg_type_number_t * infoCnt)1477 mach_vm_region_recurse(
1478 vm_map_t map,
1479 mach_vm_address_t *address,
1480 mach_vm_size_t *size,
1481 uint32_t *depth,
1482 vm_region_recurse_info_t info,
1483 mach_msg_type_number_t *infoCnt)
1484 {
1485 vm_map_address_t map_addr;
1486 vm_map_size_t map_size;
1487 kern_return_t kr;
1488
1489 if (VM_MAP_NULL == map) {
1490 return KERN_INVALID_ARGUMENT;
1491 }
1492
1493 map_addr = (vm_map_address_t)*address;
1494 map_size = (vm_map_size_t)*size;
1495
1496 kr = vm_map_region_recurse_64(
1497 map,
1498 &map_addr,
1499 &map_size,
1500 depth,
1501 (vm_region_submap_info_64_t)info,
1502 infoCnt);
1503
1504 *address = map_addr;
1505 *size = map_size;
1506 return kr;
1507 }
1508
1509 /*
1510 * vm_region_recurse: A form of vm_region which follows the
1511 * submaps in a target map
1512 *
1513 */
1514 kern_return_t
vm_region_recurse_64(vm_map_t map,vm_address_t * address,vm_size_t * size,uint32_t * depth,vm_region_recurse_info_64_t info,mach_msg_type_number_t * infoCnt)1515 vm_region_recurse_64(
1516 vm_map_t map,
1517 vm_address_t *address,
1518 vm_size_t *size,
1519 uint32_t *depth,
1520 vm_region_recurse_info_64_t info,
1521 mach_msg_type_number_t *infoCnt)
1522 {
1523 vm_map_address_t map_addr;
1524 vm_map_size_t map_size;
1525 kern_return_t kr;
1526
1527 if (VM_MAP_NULL == map) {
1528 return KERN_INVALID_ARGUMENT;
1529 }
1530
1531 map_addr = (vm_map_address_t)*address;
1532 map_size = (vm_map_size_t)*size;
1533
1534 kr = vm_map_region_recurse_64(
1535 map,
1536 &map_addr,
1537 &map_size,
1538 depth,
1539 (vm_region_submap_info_64_t)info,
1540 infoCnt);
1541
1542 *address = CAST_DOWN(vm_address_t, map_addr);
1543 *size = CAST_DOWN(vm_size_t, map_size);
1544
1545 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1546 return KERN_INVALID_ADDRESS;
1547 }
1548 return kr;
1549 }
1550
1551 kern_return_t
vm_region_recurse(vm_map_t map,vm_offset_t * address,vm_size_t * size,natural_t * depth,vm_region_recurse_info_t info32,mach_msg_type_number_t * infoCnt)1552 vm_region_recurse(
1553 vm_map_t map,
1554 vm_offset_t *address, /* IN/OUT */
1555 vm_size_t *size, /* OUT */
1556 natural_t *depth, /* IN/OUT */
1557 vm_region_recurse_info_t info32, /* IN/OUT */
1558 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1559 {
1560 vm_region_submap_info_data_64_t info64;
1561 vm_region_submap_info_t info;
1562 vm_map_address_t map_addr;
1563 vm_map_size_t map_size;
1564 kern_return_t kr;
1565
1566 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) {
1567 return KERN_INVALID_ARGUMENT;
1568 }
1569
1570
1571 map_addr = (vm_map_address_t)*address;
1572 map_size = (vm_map_size_t)*size;
1573 info = (vm_region_submap_info_t)info32;
1574 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1575
1576 kr = vm_map_region_recurse_64(map, &map_addr, &map_size,
1577 depth, &info64, infoCnt);
1578
1579 info->protection = info64.protection;
1580 info->max_protection = info64.max_protection;
1581 info->inheritance = info64.inheritance;
1582 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1583 info->user_tag = info64.user_tag;
1584 info->pages_resident = info64.pages_resident;
1585 info->pages_shared_now_private = info64.pages_shared_now_private;
1586 info->pages_swapped_out = info64.pages_swapped_out;
1587 info->pages_dirtied = info64.pages_dirtied;
1588 info->ref_count = info64.ref_count;
1589 info->shadow_depth = info64.shadow_depth;
1590 info->external_pager = info64.external_pager;
1591 info->share_mode = info64.share_mode;
1592 info->is_submap = info64.is_submap;
1593 info->behavior = info64.behavior;
1594 info->object_id = info64.object_id;
1595 info->user_wired_count = info64.user_wired_count;
1596
1597 *address = CAST_DOWN(vm_address_t, map_addr);
1598 *size = CAST_DOWN(vm_size_t, map_size);
1599 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1600
1601 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1602 return KERN_INVALID_ADDRESS;
1603 }
1604 return kr;
1605 }
1606
1607 kern_return_t
mach_vm_purgable_control(vm_map_t map,mach_vm_offset_t address,vm_purgable_t control,int * state)1608 mach_vm_purgable_control(
1609 vm_map_t map,
1610 mach_vm_offset_t address,
1611 vm_purgable_t control,
1612 int *state)
1613 {
1614 if (VM_MAP_NULL == map) {
1615 return KERN_INVALID_ARGUMENT;
1616 }
1617
1618 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1619 /* not allowed from user-space */
1620 return KERN_INVALID_ARGUMENT;
1621 }
1622
1623 return vm_map_purgable_control(map,
1624 vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
1625 control,
1626 state);
1627 }
1628
1629 kern_return_t
mach_vm_purgable_control_external(mach_port_t target_tport,mach_vm_offset_t address,vm_purgable_t control,int * state)1630 mach_vm_purgable_control_external(
1631 mach_port_t target_tport,
1632 mach_vm_offset_t address,
1633 vm_purgable_t control,
1634 int *state)
1635 {
1636 vm_map_t map;
1637 kern_return_t kr;
1638
1639 if (control == VM_PURGABLE_GET_STATE) {
1640 map = convert_port_to_map_read(target_tport);
1641 } else {
1642 map = convert_port_to_map(target_tport);
1643 }
1644
1645 kr = mach_vm_purgable_control(map, address, control, state);
1646 vm_map_deallocate(map);
1647
1648 return kr;
1649 }
1650
1651 kern_return_t
vm_purgable_control(vm_map_t map,vm_offset_t address,vm_purgable_t control,int * state)1652 vm_purgable_control(
1653 vm_map_t map,
1654 vm_offset_t address,
1655 vm_purgable_t control,
1656 int *state)
1657 {
1658 if (VM_MAP_NULL == map) {
1659 return KERN_INVALID_ARGUMENT;
1660 }
1661
1662 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1663 /* not allowed from user-space */
1664 return KERN_INVALID_ARGUMENT;
1665 }
1666
1667 return vm_map_purgable_control(map,
1668 vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
1669 control,
1670 state);
1671 }
1672
1673 kern_return_t
vm_purgable_control_external(mach_port_t target_tport,vm_offset_t address,vm_purgable_t control,int * state)1674 vm_purgable_control_external(
1675 mach_port_t target_tport,
1676 vm_offset_t address,
1677 vm_purgable_t control,
1678 int *state)
1679 {
1680 vm_map_t map;
1681 kern_return_t kr;
1682
1683 if (control == VM_PURGABLE_GET_STATE) {
1684 map = convert_port_to_map_read(target_tport);
1685 } else {
1686 map = convert_port_to_map(target_tport);
1687 }
1688
1689 kr = vm_purgable_control(map, address, control, state);
1690 vm_map_deallocate(map);
1691
1692 return kr;
1693 }
1694
1695
1696 kern_return_t
mach_vm_page_query(vm_map_t map,mach_vm_offset_t offset,int * disposition,int * ref_count)1697 mach_vm_page_query(
1698 vm_map_t map,
1699 mach_vm_offset_t offset,
1700 int *disposition,
1701 int *ref_count)
1702 {
1703 if (VM_MAP_NULL == map) {
1704 return KERN_INVALID_ARGUMENT;
1705 }
1706
1707 return vm_map_page_query_internal(
1708 map,
1709 vm_map_trunc_page(offset, PAGE_MASK),
1710 disposition, ref_count);
1711 }
1712
1713 kern_return_t
vm_map_page_query(vm_map_t map,vm_offset_t offset,int * disposition,int * ref_count)1714 vm_map_page_query(
1715 vm_map_t map,
1716 vm_offset_t offset,
1717 int *disposition,
1718 int *ref_count)
1719 {
1720 if (VM_MAP_NULL == map) {
1721 return KERN_INVALID_ARGUMENT;
1722 }
1723
1724 return vm_map_page_query_internal(
1725 map,
1726 vm_map_trunc_page(offset, PAGE_MASK),
1727 disposition, ref_count);
1728 }
1729
1730 kern_return_t
mach_vm_page_range_query(vm_map_t map,mach_vm_offset_t address,mach_vm_size_t size,mach_vm_address_t dispositions_addr,mach_vm_size_t * dispositions_count)1731 mach_vm_page_range_query(
1732 vm_map_t map,
1733 mach_vm_offset_t address,
1734 mach_vm_size_t size,
1735 mach_vm_address_t dispositions_addr,
1736 mach_vm_size_t *dispositions_count)
1737 {
1738 kern_return_t kr = KERN_SUCCESS;
1739 int num_pages = 0, i = 0;
1740 mach_vm_size_t curr_sz = 0, copy_sz = 0;
1741 mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
1742 mach_msg_type_number_t count = 0;
1743
1744 void *info = NULL;
1745 void *local_disp = NULL;
1746 vm_map_size_t info_size = 0, local_disp_size = 0;
1747 mach_vm_offset_t start = 0, end = 0;
1748 int effective_page_shift, effective_page_size, effective_page_mask;
1749
1750 if (map == VM_MAP_NULL || dispositions_count == NULL) {
1751 return KERN_INVALID_ARGUMENT;
1752 }
1753
1754 effective_page_shift = vm_self_region_page_shift_safely(map);
1755 if (effective_page_shift == -1) {
1756 return KERN_INVALID_ARGUMENT;
1757 }
1758 effective_page_size = (1 << effective_page_shift);
1759 effective_page_mask = effective_page_size - 1;
1760
1761 if (os_mul_overflow(*dispositions_count, sizeof(int), &disp_buf_req_size)) {
1762 return KERN_INVALID_ARGUMENT;
1763 }
1764
1765 start = vm_map_trunc_page(address, effective_page_mask);
1766 end = vm_map_round_page(address + size, effective_page_mask);
1767
1768 if (end < start) {
1769 return KERN_INVALID_ARGUMENT;
1770 }
1771
1772 if ((end - start) < size) {
1773 /*
1774 * Aligned size is less than unaligned size.
1775 */
1776 return KERN_INVALID_ARGUMENT;
1777 }
1778
1779 if (disp_buf_req_size == 0 || (end == start)) {
1780 return KERN_SUCCESS;
1781 }
1782
1783 /*
1784 * For large requests, we will go through them
1785 * MAX_PAGE_RANGE_QUERY chunk at a time.
1786 */
1787
1788 curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
1789 num_pages = (int) (curr_sz >> effective_page_shift);
1790
1791 info_size = num_pages * sizeof(vm_page_info_basic_data_t);
1792 info = kalloc_data(info_size, Z_WAITOK);
1793
1794 local_disp_size = num_pages * sizeof(int);
1795 local_disp = kalloc_data(local_disp_size, Z_WAITOK);
1796
1797 if (info == NULL || local_disp == NULL) {
1798 kr = KERN_RESOURCE_SHORTAGE;
1799 goto out;
1800 }
1801
1802 while (size) {
1803 count = VM_PAGE_INFO_BASIC_COUNT;
1804 kr = vm_map_page_range_info_internal(
1805 map,
1806 start,
1807 vm_map_round_page(start + curr_sz, effective_page_mask),
1808 effective_page_shift,
1809 VM_PAGE_INFO_BASIC,
1810 (vm_page_info_t) info,
1811 &count);
1812
1813 assert(kr == KERN_SUCCESS);
1814
1815 for (i = 0; i < num_pages; i++) {
1816 ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
1817 }
1818
1819 copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */);
1820 kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
1821
1822 start += curr_sz;
1823 disp_buf_req_size -= copy_sz;
1824 disp_buf_total_size += copy_sz;
1825
1826 if (kr != 0) {
1827 break;
1828 }
1829
1830 if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
1831 /*
1832 * We might have inspected the full range OR
1833 * more than it esp. if the user passed in
1834 * non-page aligned start/size and/or if we
1835 * descended into a submap. We are done here.
1836 */
1837
1838 size = 0;
1839 } else {
1840 dispositions_addr += copy_sz;
1841
1842 size -= curr_sz;
1843
1844 curr_sz = MIN(vm_map_round_page(size, effective_page_mask), MAX_PAGE_RANGE_QUERY);
1845 num_pages = (int)(curr_sz >> effective_page_shift);
1846 }
1847 }
1848
1849 *dispositions_count = disp_buf_total_size / sizeof(int);
1850
1851 out:
1852 kfree_data(local_disp, local_disp_size);
1853 kfree_data(info, info_size);
1854 return kr;
1855 }
1856
1857 kern_return_t
mach_vm_page_info(vm_map_t map,mach_vm_address_t address,vm_page_info_flavor_t flavor,vm_page_info_t info,mach_msg_type_number_t * count)1858 mach_vm_page_info(
1859 vm_map_t map,
1860 mach_vm_address_t address,
1861 vm_page_info_flavor_t flavor,
1862 vm_page_info_t info,
1863 mach_msg_type_number_t *count)
1864 {
1865 kern_return_t kr;
1866
1867 if (map == VM_MAP_NULL) {
1868 return KERN_INVALID_ARGUMENT;
1869 }
1870
1871 kr = vm_map_page_info(map, address, flavor, info, count);
1872 return kr;
1873 }
1874
1875 /*
1876 * task_wire
1877 *
1878 * Set or clear the map's wiring_required flag. This flag, if set,
1879 * will cause all future virtual memory allocation to allocate
1880 * user wired memory. Unwiring pages wired down as a result of
1881 * this routine is done with the vm_wire interface.
1882 */
1883 kern_return_t
task_wire(vm_map_t map,boolean_t must_wire __unused)1884 task_wire(
1885 vm_map_t map,
1886 boolean_t must_wire __unused)
1887 {
1888 if (map == VM_MAP_NULL) {
1889 return KERN_INVALID_ARGUMENT;
1890 }
1891
1892 return KERN_NOT_SUPPORTED;
1893 }
1894
1895 kern_return_t
vm_map_exec_lockdown(vm_map_t map)1896 vm_map_exec_lockdown(
1897 vm_map_t map)
1898 {
1899 if (map == VM_MAP_NULL) {
1900 return KERN_INVALID_ARGUMENT;
1901 }
1902
1903 vm_map_lock(map);
1904 map->map_disallow_new_exec = TRUE;
1905 vm_map_unlock(map);
1906
1907 return KERN_SUCCESS;
1908 }
1909
1910 #if XNU_PLATFORM_MacOSX
1911 /*
1912 * Now a kernel-private interface (for BootCache
1913 * use only). Need a cleaner way to create an
1914 * empty vm_map() and return a handle to it.
1915 */
1916
1917 kern_return_t
vm_region_object_create(vm_map_t target_map,vm_size_t size,ipc_port_t * object_handle)1918 vm_region_object_create(
1919 vm_map_t target_map,
1920 vm_size_t size,
1921 ipc_port_t *object_handle)
1922 {
1923 vm_named_entry_t user_entry;
1924 vm_map_t new_map;
1925
1926 user_entry = mach_memory_entry_allocate(object_handle);
1927
1928 /* Create a named object based on a submap of specified size */
1929
1930 new_map = vm_map_create_options(PMAP_NULL, VM_MAP_MIN_ADDRESS,
1931 vm_map_round_page(size, VM_MAP_PAGE_MASK(target_map)),
1932 VM_MAP_CREATE_PAGEABLE);
1933 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
1934
1935 user_entry->backing.map = new_map;
1936 user_entry->internal = TRUE;
1937 user_entry->is_sub_map = TRUE;
1938 user_entry->offset = 0;
1939 user_entry->protection = VM_PROT_ALL;
1940 user_entry->size = size;
1941
1942 return KERN_SUCCESS;
1943 }
1944 #endif /* XNU_PLATFORM_MacOSX */
1945
1946 kern_return_t
mach_vm_deferred_reclamation_buffer_init(task_t task,mach_vm_offset_t * address,mach_vm_size_t size)1947 mach_vm_deferred_reclamation_buffer_init(
1948 task_t task,
1949 mach_vm_offset_t *address,
1950 mach_vm_size_t size)
1951 {
1952 #if CONFIG_DEFERRED_RECLAIM
1953 return vm_deferred_reclamation_buffer_init_internal(task, address, size);
1954 #else
1955 (void) task;
1956 (void) address;
1957 (void) size;
1958 (void) indices;
1959 return KERN_NOT_SUPPORTED;
1960 #endif /* CONFIG_DEFERRED_RECLAIM */
1961 }
1962
1963 kern_return_t
mach_vm_deferred_reclamation_buffer_synchronize(task_t task,mach_vm_size_t num_entries_to_reclaim)1964 mach_vm_deferred_reclamation_buffer_synchronize(
1965 task_t task,
1966 mach_vm_size_t num_entries_to_reclaim)
1967 {
1968 #if CONFIG_DEFERRED_RECLAIM
1969 return vm_deferred_reclamation_buffer_synchronize_internal(task, num_entries_to_reclaim);
1970 #else
1971 (void) task;
1972 (void) num_entries_to_reclaim;
1973 return KERN_NOT_SUPPORTED;
1974 #endif /* CONFIG_DEFERRED_RECLAIM */
1975 }
1976
1977 kern_return_t
mach_vm_deferred_reclamation_buffer_update_reclaimable_bytes(task_t task,mach_vm_size_t reclaimable_bytes)1978 mach_vm_deferred_reclamation_buffer_update_reclaimable_bytes(task_t task, mach_vm_size_t reclaimable_bytes)
1979 {
1980 #if CONFIG_DEFERRED_RECLAIM
1981 return vm_deferred_reclamation_buffer_update_reclaimable_bytes_internal(task, reclaimable_bytes);
1982 #else
1983 (void) task;
1984 (void) reclaimable_bytes;
1985 return KERN_NOT_SUPPORTED;
1986 #endif /* CONFIG_DEFERRED_RECLAIM */
1987 }
1988
1989 #if CONFIG_MAP_RANGES
1990
1991 extern void qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
1992
1993 static int
vm_map_user_range_cmp(const void * e1,const void * e2)1994 vm_map_user_range_cmp(const void *e1, const void *e2)
1995 {
1996 const struct vm_map_user_range *r1 = e1;
1997 const struct vm_map_user_range *r2 = e2;
1998
1999 if (r1->vmur_min_address != r2->vmur_min_address) {
2000 return r1->vmur_min_address < r2->vmur_min_address ? -1 : 1;
2001 }
2002
2003 return 0;
2004 }
2005
2006 static int
mach_vm_range_recipe_v1_cmp(const void * e1,const void * e2)2007 mach_vm_range_recipe_v1_cmp(const void *e1, const void *e2)
2008 {
2009 const mach_vm_range_recipe_v1_t *r1 = e1;
2010 const mach_vm_range_recipe_v1_t *r2 = e2;
2011
2012 if (r1->range.min_address != r2->range.min_address) {
2013 return r1->range.min_address < r2->range.min_address ? -1 : 1;
2014 }
2015
2016 return 0;
2017 }
2018
2019 /*!
2020 * @function mach_vm_range_create_v1()
2021 *
2022 * @brief
2023 * Handle the backend for mach_vm_range_create() for the
2024 * MACH_VM_RANGE_FLAVOR_V1 flavor.
2025 *
2026 * @description
2027 * This call allows to create "ranges" in the map of a task
2028 * that have special semantics/policies around placement of
2029 * new allocations (in the vm_map_locate_space() sense).
2030 *
2031 * @returns
2032 * - KERN_SUCCESS on success
2033 * - KERN_INVALID_ARGUMENT for incorrect arguments
2034 * - KERN_NO_SPACE if the maximum amount of ranges would be exceeded
2035 * - KERN_MEMORY_PRESENT if any of the requested ranges
2036 * overlaps with existing ranges or allocations in the map.
2037 */
2038 static kern_return_t
mach_vm_range_create_v1(vm_map_t map,mach_vm_range_recipe_v1_t * recipe,uint32_t new_count)2039 mach_vm_range_create_v1(
2040 vm_map_t map,
2041 mach_vm_range_recipe_v1_t *recipe,
2042 uint32_t new_count)
2043 {
2044 const vm_offset_t mask = VM_MAP_PAGE_MASK(map);
2045 vm_map_user_range_t table;
2046 kern_return_t kr = KERN_SUCCESS;
2047 uint16_t count;
2048
2049 struct mach_vm_range void1 = {
2050 .min_address = map->default_range.max_address,
2051 .max_address = map->data_range.min_address,
2052 };
2053 struct mach_vm_range void2 = {
2054 .min_address = map->data_range.max_address,
2055 #if XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT
2056 .max_address = MACH_VM_JUMBO_ADDRESS,
2057 #else /* !XNU_TARGET_OS_IOS || !EXTENDED_USER_VA_SUPPORT */
2058 .max_address = vm_map_max(map),
2059 #endif /* XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT */
2060 };
2061
2062 qsort(recipe, new_count, sizeof(mach_vm_range_recipe_v1_t),
2063 mach_vm_range_recipe_v1_cmp);
2064
2065 /*
2066 * Step 1: Validate that the recipes have no intersections.
2067 */
2068
2069 for (size_t i = 0; i < new_count; i++) {
2070 mach_vm_range_t r = &recipe[i].range;
2071 mach_vm_size_t s;
2072
2073 if (recipe[i].flags) {
2074 return KERN_INVALID_ARGUMENT;
2075 }
2076
2077 static_assert(UMEM_RANGE_ID_FIXED == MACH_VM_RANGE_FIXED);
2078 switch (recipe[i].range_tag) {
2079 case MACH_VM_RANGE_FIXED:
2080 break;
2081 default:
2082 return KERN_INVALID_ARGUMENT;
2083 }
2084
2085 if (!VM_MAP_PAGE_ALIGNED(r->min_address, mask) ||
2086 !VM_MAP_PAGE_ALIGNED(r->max_address, mask) ||
2087 r->min_address >= r->max_address) {
2088 return KERN_INVALID_ARGUMENT;
2089 }
2090
2091 s = mach_vm_range_size(r);
2092 if (!mach_vm_range_contains(&void1, r->min_address, s) &&
2093 !mach_vm_range_contains(&void2, r->min_address, s)) {
2094 return KERN_INVALID_ARGUMENT;
2095 }
2096
2097 if (i > 0 && recipe[i - 1].range.max_address >
2098 recipe[i].range.min_address) {
2099 return KERN_INVALID_ARGUMENT;
2100 }
2101 }
2102
2103 vm_map_lock(map);
2104
2105 table = map->extra_ranges;
2106 count = map->extra_ranges_count;
2107
2108 if (count + new_count > VM_MAP_EXTRA_RANGES_MAX) {
2109 kr = KERN_NO_SPACE;
2110 goto out_unlock;
2111 }
2112
2113 /*
2114 * Step 2: Check that there is no intersection with existing ranges.
2115 */
2116
2117 for (size_t i = 0, j = 0; i < new_count && j < count;) {
2118 mach_vm_range_t r1 = &recipe[i].range;
2119 vm_map_user_range_t r2 = &table[j];
2120
2121 if (r1->max_address <= r2->vmur_min_address) {
2122 i++;
2123 } else if (r2->vmur_max_address <= r1->min_address) {
2124 j++;
2125 } else {
2126 kr = KERN_MEMORY_PRESENT;
2127 goto out_unlock;
2128 }
2129 }
2130
2131 /*
2132 * Step 4: commit the new ranges.
2133 */
2134
2135 static_assert(VM_MAP_EXTRA_RANGES_MAX * sizeof(struct vm_map_user_range) <=
2136 KALLOC_SAFE_ALLOC_SIZE);
2137
2138 table = krealloc_data(table,
2139 count * sizeof(struct vm_map_user_range),
2140 (count + new_count) * sizeof(struct vm_map_user_range),
2141 Z_ZERO | Z_WAITOK | Z_NOFAIL);
2142
2143 for (size_t i = 0; i < new_count; i++) {
2144 static_assert(MACH_VM_MAX_ADDRESS < (1ull << 56));
2145
2146 table[count + i] = (struct vm_map_user_range){
2147 .vmur_min_address = recipe[i].range.min_address,
2148 .vmur_max_address = recipe[i].range.max_address,
2149 .vmur_range_id = (vm_map_range_id_t)recipe[i].range_tag,
2150 };
2151 }
2152
2153 qsort(table, count + new_count,
2154 sizeof(struct vm_map_user_range), vm_map_user_range_cmp);
2155
2156 map->extra_ranges_count += new_count;
2157 map->extra_ranges = table;
2158
2159 out_unlock:
2160 vm_map_unlock(map);
2161
2162 if (kr == KERN_SUCCESS) {
2163 for (size_t i = 0; i < new_count; i++) {
2164 vm_map_kernel_flags_t vmk_flags = {
2165 .vmf_fixed = true,
2166 .vmf_overwrite = true,
2167 .vmkf_overwrite_immutable = true,
2168 .vm_tag = recipe[i].vm_tag,
2169 };
2170 __assert_only kern_return_t kr2;
2171
2172 kr2 = vm_map_enter(map, &recipe[i].range.min_address,
2173 mach_vm_range_size(&recipe[i].range),
2174 0, vmk_flags, VM_OBJECT_NULL, 0, FALSE,
2175 VM_PROT_NONE, VM_PROT_ALL,
2176 VM_INHERIT_DEFAULT);
2177 assert(kr2 == KERN_SUCCESS);
2178 }
2179 }
2180 return kr;
2181 }
2182
2183 kern_return_t
mach_vm_range_create(vm_map_t map,mach_vm_range_flavor_t flavor,mach_vm_range_recipes_raw_t recipe,natural_t size)2184 mach_vm_range_create(
2185 vm_map_t map,
2186 mach_vm_range_flavor_t flavor,
2187 mach_vm_range_recipes_raw_t recipe,
2188 natural_t size)
2189 {
2190 if (map != current_map()) {
2191 return KERN_INVALID_ARGUMENT;
2192 }
2193
2194 if (!map->uses_user_ranges) {
2195 return KERN_NOT_SUPPORTED;
2196 }
2197
2198 if (size == 0) {
2199 return KERN_SUCCESS;
2200 }
2201
2202 if (flavor == MACH_VM_RANGE_FLAVOR_V1) {
2203 mach_vm_range_recipe_v1_t *array;
2204
2205 if (size % sizeof(mach_vm_range_recipe_v1_t)) {
2206 return KERN_INVALID_ARGUMENT;
2207 }
2208
2209 size /= sizeof(mach_vm_range_recipe_v1_t);
2210 if (size > VM_MAP_EXTRA_RANGES_MAX) {
2211 return KERN_NO_SPACE;
2212 }
2213
2214 array = (mach_vm_range_recipe_v1_t *)recipe;
2215 return mach_vm_range_create_v1(map, array, size);
2216 }
2217
2218 return KERN_INVALID_ARGUMENT;
2219 }
2220
2221 #else /* !CONFIG_MAP_RANGES */
2222
2223 kern_return_t
mach_vm_range_create(vm_map_t map,mach_vm_range_flavor_t flavor,mach_vm_range_recipes_raw_t recipe,natural_t size)2224 mach_vm_range_create(
2225 vm_map_t map,
2226 mach_vm_range_flavor_t flavor,
2227 mach_vm_range_recipes_raw_t recipe,
2228 natural_t size)
2229 {
2230 #pragma unused(map, flavor, recipe, size)
2231 return KERN_NOT_SUPPORTED;
2232 }
2233
2234 #endif /* !CONFIG_MAP_RANGES */
2235
2236 /*
2237 * These symbols are looked up at runtime by vmware, VirtualBox,
2238 * despite not being exported in the symbol sets.
2239 */
2240
2241 #if defined(__x86_64__)
2242
2243 extern typeof(mach_vm_remap_external) mach_vm_remap;
2244 extern typeof(mach_vm_map_external) mach_vm_map;
2245 extern typeof(vm_map_external) vm_map;
2246
2247 kern_return_t
mach_vm_map(vm_map_t target_map,mach_vm_offset_ut * address,mach_vm_size_ut initial_size,mach_vm_offset_ut mask,int flags,ipc_port_t port,memory_object_offset_ut offset,boolean_t copy,vm_prot_ut cur_protection,vm_prot_ut max_protection,vm_inherit_ut inheritance)2248 mach_vm_map(
2249 vm_map_t target_map,
2250 mach_vm_offset_ut *address,
2251 mach_vm_size_ut initial_size,
2252 mach_vm_offset_ut mask,
2253 int flags,
2254 ipc_port_t port,
2255 memory_object_offset_ut offset,
2256 boolean_t copy,
2257 vm_prot_ut cur_protection,
2258 vm_prot_ut max_protection,
2259 vm_inherit_ut inheritance)
2260 {
2261 return mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
2262 offset, copy, cur_protection, max_protection, inheritance);
2263 }
2264
2265 kern_return_t
mach_vm_remap(vm_map_t target_map,mach_vm_offset_ut * address,mach_vm_size_ut size,mach_vm_offset_ut mask,int flags,vm_map_t src_map,mach_vm_offset_ut memory_address,boolean_t copy,vm_prot_ut * cur_protection,vm_prot_ut * max_protection,vm_inherit_ut inheritance)2266 mach_vm_remap(
2267 vm_map_t target_map,
2268 mach_vm_offset_ut *address,
2269 mach_vm_size_ut size,
2270 mach_vm_offset_ut mask,
2271 int flags,
2272 vm_map_t src_map,
2273 mach_vm_offset_ut memory_address,
2274 boolean_t copy,
2275 vm_prot_ut *cur_protection, /* OUT */
2276 vm_prot_ut *max_protection, /* OUT */
2277 vm_inherit_ut inheritance)
2278 {
2279 return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
2280 copy, cur_protection, max_protection, inheritance);
2281 }
2282
2283 kern_return_t
vm_map(vm_map_t target_map,vm_offset_ut * address,vm_size_ut size,vm_offset_ut mask,int flags,ipc_port_t port,vm_offset_ut offset,boolean_t copy,vm_prot_ut cur_protection,vm_prot_ut max_protection,vm_inherit_ut inheritance)2284 vm_map(
2285 vm_map_t target_map,
2286 vm_offset_ut *address,
2287 vm_size_ut size,
2288 vm_offset_ut mask,
2289 int flags,
2290 ipc_port_t port,
2291 vm_offset_ut offset,
2292 boolean_t copy,
2293 vm_prot_ut cur_protection,
2294 vm_prot_ut max_protection,
2295 vm_inherit_ut inheritance)
2296 {
2297 return mach_vm_map(target_map, address,
2298 size, mask, flags, port, offset, copy,
2299 cur_protection, max_protection, inheritance);
2300 }
2301
2302 #endif /* __x86_64__ */
2303