/* * Copyright (c) 2000-2021 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ /* */ /* * File: vm/vm_user.c * Author: Avadis Tevanian, Jr., Michael Wayne Young * * User-exported virtual memory functions. */ /* * There are three implementations of the "XXX_allocate" functionality in * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate * (for a task with the same address space size, especially the current task), * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate * in the kernel should only be used on the kernel_task. vm32_vm_allocate only * makes sense on platforms where a user task can either be 32 or 64, or the kernel * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred * for new code. * * The entrypoints into the kernel are more complex. All platforms support a * mach_vm_allocate-style API (subsystem 4800) which operates with the largest * size types for the platform. On platforms that only support U32/K32, * subsystem 4800 is all you need. On platforms that support both U32 and U64, * subsystem 3800 is used disambiguate the size of parameters, and they will * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms, * the MIG glue should never call into vm_allocate directly, because the calling * task and kernel_task are unlikely to use the same size parameters * * New VM call implementations should be added here and to mach_vm.defs * (subsystem 4800), and use mach_vm_* "wide" types. */ #include #include #include #include /* to get vm_address_t */ #include #include /* to get pointer_t */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if CONFIG_DEFERRED_RECLAIM #include #endif /* CONFIG_DEFERRED_RECLAIM */ #include #include #include #include #include /* * mach_vm_allocate allocates "zero fill" memory in the specfied * map. */ kern_return_t mach_vm_allocate_external( vm_map_t map, mach_vm_offset_ut *addr, mach_vm_size_ut size, int flags) { vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; /* filter out any kernel-only flags */ if (flags & ~VM_FLAGS_USER_ALLOCATE) { ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADFLAGS_ERROR), KERN_INVALID_ARGUMENT /* arg */); return KERN_INVALID_ARGUMENT; } vm_map_kernel_flags_set_vmflags(&vmk_flags, flags); return mach_vm_allocate_kernel(map, addr, size, vmk_flags); } /* * vm_allocate * Legacy routine that allocates "zero fill" memory in the specfied * map (which is limited to the same size as the kernel). */ kern_return_t vm_allocate_external( vm_map_t map, vm_offset_ut *addr, vm_size_ut size, int flags) { return mach_vm_allocate_external(map, addr, size, flags); } static inline kern_return_t mach_vm_deallocate_sanitize( vm_map_t map, mach_vm_offset_ut start_u, mach_vm_size_ut size_u, mach_vm_offset_t *start, mach_vm_offset_t *end, mach_vm_size_t *size) { return vm_sanitize_addr_size(start_u, size_u, VM_SANITIZE_CALLER_VM_DEALLOCATE, map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, start, end, size); } /* * mach_vm_deallocate - * deallocates the specified range of addresses in the * specified address map. */ kern_return_t mach_vm_deallocate( vm_map_t map, mach_vm_offset_ut start_u, mach_vm_size_ut size_u) { mach_vm_offset_t start, end; mach_vm_size_t size; kern_return_t kr; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } kr = mach_vm_deallocate_sanitize(map, start_u, size_u, &start, &end, &size); if (__improbable(kr != KERN_SUCCESS)) { return vm_sanitize_get_kr(kr); } return vm_map_remove_guard(map, start, end, VM_MAP_REMOVE_NO_FLAGS, KMEM_GUARD_NONE).kmr_return; } /* * vm_deallocate - * deallocates the specified range of addresses in the * specified address map (limited to addresses the same * size as the kernel). */ kern_return_t vm_deallocate( vm_map_t map, vm_offset_ut start, vm_size_ut size) { return mach_vm_deallocate(map, start, size); } /* * mach_vm_inherit - * Sets the inheritance of the specified range in the * specified map. */ kern_return_t mach_vm_inherit( vm_map_t map, mach_vm_offset_t start, mach_vm_size_t size, vm_inherit_t new_inheritance) { if ((map == VM_MAP_NULL) || (start + size < start) || (new_inheritance > VM_INHERIT_LAST_VALID)) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_inherit(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start + size, VM_MAP_PAGE_MASK(map)), new_inheritance); } /* * vm_inherit - * Sets the inheritance of the specified range in the * specified map (range limited to addresses */ kern_return_t vm_inherit( vm_map_t map, vm_offset_t start, vm_size_t size, vm_inherit_t new_inheritance) { if ((map == VM_MAP_NULL) || (start + size < start) || (new_inheritance > VM_INHERIT_LAST_VALID)) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_inherit(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start + size, VM_MAP_PAGE_MASK(map)), new_inheritance); } /* * mach_vm_protect - * Sets the protection of the specified range in the * specified map. */ kern_return_t mach_vm_protect( vm_map_t map, mach_vm_offset_t start, mach_vm_size_t size, boolean_t set_maximum, vm_prot_t new_protection) { if ((map == VM_MAP_NULL) || (start + size < start) || (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_protect(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start + size, VM_MAP_PAGE_MASK(map)), new_protection, set_maximum); } /* * vm_protect - * Sets the protection of the specified range in the * specified map. Addressability of the range limited * to the same size as the kernel. */ kern_return_t vm_protect( vm_map_t map, vm_offset_t start, vm_size_t size, boolean_t set_maximum, vm_prot_t new_protection) { if ((map == VM_MAP_NULL) || (start + size < start) || (new_protection & ~VM_VALID_VMPROTECT_FLAGS) #if defined(__x86_64__) || ((new_protection & VM_PROT_UEXEC) && !pmap_supported_feature(map->pmap, PMAP_FEAT_UEXEC)) #endif ) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_protect(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start + size, VM_MAP_PAGE_MASK(map)), new_protection, set_maximum); } /* * mach_vm_machine_attributes - * Handle machine-specific attributes for a mapping, such * as cachability, migrability, etc. */ kern_return_t mach_vm_machine_attribute( vm_map_t map, mach_vm_address_t addr, mach_vm_size_t size, vm_machine_attribute_t attribute, vm_machine_attribute_val_t* value) /* IN/OUT */ { if ((map == VM_MAP_NULL) || (addr + size < addr)) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_machine_attribute( map, vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(map)), vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(map)), attribute, value); } /* * vm_machine_attribute - * Handle machine-specific attributes for a mapping, such * as cachability, migrability, etc. Limited addressability * (same range limits as for the native kernel map). */ kern_return_t vm_machine_attribute( vm_map_t map, vm_address_t addr, vm_size_t size, vm_machine_attribute_t attribute, vm_machine_attribute_val_t* value) /* IN/OUT */ { if ((map == VM_MAP_NULL) || (addr + size < addr)) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } return vm_map_machine_attribute( map, vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(map)), vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(map)), attribute, value); } /* * mach_vm_read - * Read/copy a range from one address space and return it to the caller. * * It is assumed that the address for the returned memory is selected by * the IPC implementation as part of receiving the reply to this call. * If IPC isn't used, the caller must deal with the vm_map_copy_t object * that gets returned. * * JMM - because of mach_msg_type_number_t, this call is limited to a * single 4GB region at this time. * */ kern_return_t mach_vm_read( vm_map_t map, mach_vm_address_ut addr, mach_vm_size_ut size, pointer_t *data, mach_msg_type_number_t *data_size) { kern_return_t error; vm_map_copy_t ipc_address; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } /* * mach_msg_type_number_t is a signed int, * make sure we do not overflow it. */ if (!VM_SANITIZE_UNSAFE_FITS(size, mach_msg_type_number_t)) { return KERN_INVALID_ARGUMENT; } error = vm_map_copyin(map, addr, size, FALSE, &ipc_address); if (KERN_SUCCESS == error) { *data = (pointer_t) ipc_address; *data_size = (mach_msg_type_number_t)VM_SANITIZE_UNSAFE_UNWRAP(size); } return error; } /* * vm_read - * Read/copy a range from one address space and return it to the caller. * Limited addressability (same range limits as for the native kernel map). * * It is assumed that the address for the returned memory is selected by * the IPC implementation as part of receiving the reply to this call. * If IPC isn't used, the caller must deal with the vm_map_copy_t object * that gets returned. */ kern_return_t vm_read( vm_map_t map, vm_address_ut addr, vm_size_ut size, pointer_t *data, mach_msg_type_number_t *data_size) { return mach_vm_read(map, addr, size, data, data_size); } /* * mach_vm_read_list - * Read/copy a list of address ranges from specified map. * * MIG does not know how to deal with a returned array of * vm_map_copy_t structures, so we have to do the copyout * manually here. */ kern_return_t mach_vm_read_list( vm_map_t map, mach_vm_read_entry_t data_list, natural_t count) { mach_msg_type_number_t i; kern_return_t error; vm_map_copy_t copy; if (map == VM_MAP_NULL || count > VM_MAP_ENTRY_MAX) { return KERN_INVALID_ARGUMENT; } error = KERN_SUCCESS; for (i = 0; i < count; i++) { vm_map_address_t map_addr; vm_map_size_t map_size; map_addr = (vm_map_address_t)(data_list[i].address); map_size = (vm_map_size_t)(data_list[i].size); if (map_size != 0) { error = vm_map_copyin(map, map_addr, map_size, FALSE, /* src_destroy */ ©); if (KERN_SUCCESS == error) { error = vm_map_copyout( current_task()->map, &map_addr, copy); if (KERN_SUCCESS == error) { data_list[i].address = map_addr; continue; } vm_map_copy_discard(copy); } } data_list[i].address = (mach_vm_address_t)0; data_list[i].size = (mach_vm_size_t)0; } return error; } /* * vm_read_list - * Read/copy a list of address ranges from specified map. * * MIG does not know how to deal with a returned array of * vm_map_copy_t structures, so we have to do the copyout * manually here. * * The source and destination ranges are limited to those * that can be described with a vm_address_t (i.e. same * size map as the kernel). * * JMM - If the result of the copyout is an address range * that cannot be described with a vm_address_t (i.e. the * caller had a larger address space but used this call * anyway), it will result in a truncated address being * returned (and a likely confused caller). */ kern_return_t vm_read_list( vm_map_t map, vm_read_entry_t data_list, natural_t count) { mach_msg_type_number_t i; kern_return_t error; vm_map_copy_t copy; if (map == VM_MAP_NULL || count > VM_MAP_ENTRY_MAX) { return KERN_INVALID_ARGUMENT; } error = KERN_SUCCESS; for (i = 0; i < count; i++) { vm_map_address_t map_addr; vm_map_size_t map_size; map_addr = (vm_map_address_t)(data_list[i].address); map_size = (vm_map_size_t)(data_list[i].size); if (map_size != 0) { error = vm_map_copyin(map, map_addr, map_size, FALSE, /* src_destroy */ ©); if (KERN_SUCCESS == error) { error = vm_map_copyout(current_task()->map, &map_addr, copy); if (KERN_SUCCESS == error) { data_list[i].address = CAST_DOWN(vm_offset_t, map_addr); continue; } vm_map_copy_discard(copy); } } data_list[i].address = (mach_vm_address_t)0; data_list[i].size = (mach_vm_size_t)0; } return error; } /* * mach_vm_read_overwrite - * Overwrite a range of the current map with data from the specified * map/address range. * * In making an assumption that the current thread is local, it is * no longer cluster-safe without a fully supportive local proxy * thread/task (but we don't support cluster's anymore so this is moot). */ kern_return_t mach_vm_read_overwrite( vm_map_t map, mach_vm_address_ut address, mach_vm_size_ut size, mach_vm_address_ut data, mach_vm_size_ut *data_size) { kern_return_t error; vm_map_copy_t copy; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } error = vm_map_copyin(map, address, size, FALSE, ©); if (KERN_SUCCESS == error) { if (copy) { assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size)); } error = vm_map_copy_overwrite(current_thread()->map, data, copy, size, FALSE); if (KERN_SUCCESS == error) { *data_size = size; return error; } vm_map_copy_discard(copy); } return error; } /* * vm_read_overwrite - * Overwrite a range of the current map with data from the specified * map/address range. * * This routine adds the additional limitation that the source and * destination ranges must be describable with vm_address_t values * (i.e. the same size address spaces as the kernel, or at least the * the ranges are in that first portion of the respective address * spaces). */ kern_return_t vm_read_overwrite( vm_map_t map, vm_address_ut address, vm_size_ut size, vm_address_ut data, vm_size_ut *data_size) { return mach_vm_read_overwrite(map, address, size, data, data_size); } /* * mach_vm_write - * Overwrite the specified address range with the data provided * (from the current map). */ kern_return_t mach_vm_write( vm_map_t map, mach_vm_address_ut address, pointer_t data, mach_msg_type_number_t size) { if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } return vm_map_copy_overwrite(map, address, (vm_map_copy_t) data, size, FALSE /* interruptible XXX */); } /* * vm_write - * Overwrite the specified address range with the data provided * (from the current map). * * The addressability of the range of addresses to overwrite is * limited bu the use of a vm_address_t (same size as kernel map). * Either the target map is also small, or the range is in the * low addresses within it. */ kern_return_t vm_write( vm_map_t map, vm_address_ut address, pointer_t data, mach_msg_type_number_t size) { return mach_vm_write(map, address, data, size); } /* * mach_vm_copy - * Overwrite one range of the specified map with the contents of * another range within that same map (i.e. both address ranges * are "over there"). */ kern_return_t mach_vm_copy( vm_map_t map, mach_vm_address_ut source_address, mach_vm_size_ut size, mach_vm_address_ut dest_address) { vm_map_copy_t copy; kern_return_t kr; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } kr = vm_map_copyin(map, source_address, size, FALSE, ©); if (KERN_SUCCESS == kr) { if (copy) { assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size)); } kr = vm_map_copy_overwrite(map, dest_address, copy, size, FALSE /* interruptible XXX */); if (KERN_SUCCESS != kr) { vm_map_copy_discard(copy); } } return kr; } kern_return_t vm_copy( vm_map_t map, vm_address_ut source_address, vm_size_ut size, vm_address_ut dest_address) { return mach_vm_copy(map, source_address, size, dest_address); } /* * mach_vm_map - * Map some range of an object into an address space. * * The object can be one of several types of objects: * NULL - anonymous memory * a named entry - a range within another address space * or a range within a memory object * a whole memory object * */ kern_return_t mach_vm_map_external( vm_map_t target_map, mach_vm_offset_ut *address, mach_vm_size_ut initial_size, mach_vm_offset_ut mask, int flags, ipc_port_t port, memory_object_offset_ut offset, boolean_t copy, vm_prot_ut cur_protection, vm_prot_ut max_protection, vm_inherit_ut inheritance) { vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; /* filter out any kernel-only flags */ if (flags & ~VM_FLAGS_USER_MAP) { return KERN_INVALID_ARGUMENT; } vm_map_kernel_flags_set_vmflags(&vmk_flags, flags); /* range_id is set by mach_vm_map_kernel */ return mach_vm_map_kernel(target_map, address, initial_size, mask, vmk_flags, port, offset, copy, cur_protection, max_protection, inheritance); } /* legacy interface */ __attribute__((always_inline)) kern_return_t vm_map_64_external( vm_map_t target_map, vm_offset_ut *address, vm_size_ut size, vm_offset_ut mask, int flags, ipc_port_t port, memory_object_offset_ut offset, boolean_t copy, vm_prot_ut cur_protection, vm_prot_ut max_protection, vm_inherit_ut inheritance) { return mach_vm_map_external(target_map, address, size, mask, flags, port, offset, copy, cur_protection, max_protection, inheritance); } /* temporary, until world build */ __attribute__((always_inline)) kern_return_t vm_map_external( vm_map_t target_map, vm_offset_ut *address, vm_size_ut size, vm_offset_ut mask, int flags, ipc_port_t port, vm_offset_ut offset, boolean_t copy, vm_prot_ut cur_protection, vm_prot_ut max_protection, vm_inherit_ut inheritance) { return mach_vm_map_external(target_map, address, size, mask, flags, port, offset, copy, cur_protection, max_protection, inheritance); } static inline kern_return_t mach_vm_remap_new_external_sanitize( vm_map_t target_map, vm_prot_ut cur_protection_u, vm_prot_ut max_protection_u, vm_prot_t *cur_protection, vm_prot_t *max_protection) { return vm_sanitize_cur_and_max_prots(cur_protection_u, max_protection_u, VM_SANITIZE_CALLER_VM_MAP_REMAP, target_map, cur_protection, max_protection); } /* * mach_vm_remap_new - * Behaves like mach_vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set * and {cur,max}_protection are in/out. */ kern_return_t mach_vm_remap_new_external( vm_map_t target_map, mach_vm_offset_ut *address, mach_vm_size_ut size, mach_vm_offset_ut mask, int flags, mach_port_t src_tport, mach_vm_offset_ut memory_address, boolean_t copy, vm_prot_ut *cur_protection_u, /* IN/OUT */ vm_prot_ut *max_protection_u, /* IN/OUT */ vm_inherit_ut inheritance) { vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; vm_map_t src_map; vm_prot_t cur_protection, max_protection; kern_return_t kr; if (target_map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } /* filter out any kernel-only flags */ if (flags & ~VM_FLAGS_USER_REMAP) { return KERN_INVALID_ARGUMENT; } vm_map_kernel_flags_set_vmflags(&vmk_flags, flags | VM_FLAGS_RETURN_DATA_ADDR); /* * We don't need cur_protection here, but sanitizing it before * enforcing W^X below matches historical error codes better. */ kr = mach_vm_remap_new_external_sanitize(target_map, *cur_protection_u, *max_protection_u, &cur_protection, &max_protection); if (__improbable(kr != KERN_SUCCESS)) { return vm_sanitize_get_kr(kr); } if ((max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) == (VM_PROT_WRITE | VM_PROT_EXECUTE)) { /* * XXX FBDP TODO * enforce target's "wx" policies */ return KERN_PROTECTION_FAILURE; } if (copy || max_protection == VM_PROT_READ || max_protection == VM_PROT_NONE) { src_map = convert_port_to_map_read(src_tport); } else { src_map = convert_port_to_map(src_tport); } /* range_id is set by vm_map_remap */ kr = vm_map_remap(target_map, address, size, mask, vmk_flags, src_map, memory_address, copy, cur_protection_u, /* IN/OUT */ max_protection_u, /* IN/OUT */ inheritance); vm_map_deallocate(src_map); if (kr == KERN_SUCCESS) { ipc_port_release_send(src_tport); /* consume on success */ } return kr; } /* * mach_vm_remap - * Remap a range of memory from one task into another, * to another address range within the same task, or * over top of itself (with altered permissions and/or * as an in-place copy of itself). */ kern_return_t mach_vm_remap_external( vm_map_t target_map, mach_vm_offset_ut *address, mach_vm_size_ut size, mach_vm_offset_ut mask, int flags, vm_map_t src_map, mach_vm_offset_ut memory_address, boolean_t copy, vm_prot_ut *cur_protection, /* OUT */ vm_prot_ut *max_protection, /* OUT */ vm_inherit_ut inheritance) { vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; /* filter out any kernel-only flags */ if (flags & ~VM_FLAGS_USER_REMAP) { return KERN_INVALID_ARGUMENT; } vm_map_kernel_flags_set_vmflags(&vmk_flags, flags); *cur_protection = vm_sanitize_wrap_prot(VM_PROT_NONE); *max_protection = vm_sanitize_wrap_prot(VM_PROT_NONE); /* range_id is set by vm_map_remap */ return vm_map_remap(target_map, address, size, mask, vmk_flags, src_map, memory_address, copy, cur_protection, max_protection, inheritance); } /* * vm_remap_new - * Behaves like vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set * and {cur,max}_protection are in/out. */ kern_return_t vm_remap_new_external( vm_map_t target_map, vm_offset_ut *address, vm_size_ut size, vm_offset_ut mask, int flags, mach_port_t src_tport, vm_offset_ut memory_address, boolean_t copy, vm_prot_ut *cur_protection, /* IN/OUT */ vm_prot_ut *max_protection, /* IN/OUT */ vm_inherit_ut inheritance) { return mach_vm_remap_new_external(target_map, address, size, mask, flags, src_tport, memory_address, copy, cur_protection, /* IN/OUT */ max_protection, /* IN/OUT */ inheritance); } /* * vm_remap - * Remap a range of memory from one task into another, * to another address range within the same task, or * over top of itself (with altered permissions and/or * as an in-place copy of itself). * * The addressability of the source and target address * range is limited by the size of vm_address_t (in the * kernel context). */ kern_return_t vm_remap_external( vm_map_t target_map, vm_offset_ut *address, vm_size_ut size, vm_offset_ut mask, int flags, vm_map_t src_map, vm_offset_ut memory_address, boolean_t copy, vm_prot_ut *cur_protection, /* OUT */ vm_prot_ut *max_protection, /* OUT */ vm_inherit_ut inheritance) { return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address, copy, cur_protection, max_protection, inheritance); } /* * NOTE: these routine (and this file) will no longer require mach_host_server.h * when mach_vm_wire and vm_wire are changed to use ledgers. */ #include /* * mach_vm_wire * Specify that the range of the virtual address space * of the target task must not cause page faults for * the indicated accesses. * * [ To unwire the pages, specify VM_PROT_NONE. ] */ kern_return_t mach_vm_wire_external( host_priv_t host_priv, vm_map_t map, mach_vm_address_ut start, mach_vm_size_ut size, vm_prot_ut access) { kern_return_t rc; mach_vm_offset_ut end; if (host_priv == HOST_PRIV_NULL) { return KERN_INVALID_HOST; } if (map == VM_MAP_NULL) { return KERN_INVALID_TASK; } end = vm_sanitize_compute_unsafe_end(start, size); if (VM_SANITIZE_UNSAFE_IS_ZERO(access)) { rc = vm_map_unwire_impl(map, start, end, true, VM_SANITIZE_CALLER_VM_UNWIRE_USER); } else { rc = vm_map_wire_impl(map, start, end, access, VM_KERN_MEMORY_MLOCK, true, NULL, VM_SANITIZE_CALLER_VM_WIRE_USER); } return rc; } /* * vm_wire - * Specify that the range of the virtual address space * of the target task must not cause page faults for * the indicated accesses. * * [ To unwire the pages, specify VM_PROT_NONE. ] */ kern_return_t vm_wire( host_priv_t host_priv, vm_map_t map, vm_offset_ut start, vm_size_ut size, vm_prot_ut access) { return mach_vm_wire_external(host_priv, map, start, size, access); } /* * vm_msync * * Synchronises the memory range specified with its backing store * image by either flushing or cleaning the contents to the appropriate * memory manager. * * interpretation of sync_flags * VM_SYNC_INVALIDATE - discard pages, only return precious * pages to manager. * * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS) * - discard pages, write dirty or precious * pages back to memory manager. * * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS * - write dirty or precious pages back to * the memory manager. * * VM_SYNC_CONTIGUOUS - does everything normally, but if there * is a hole in the region, and we would * have returned KERN_SUCCESS, return * KERN_INVALID_ADDRESS instead. * * RETURNS * KERN_INVALID_TASK Bad task parameter * KERN_INVALID_ARGUMENT both sync and async were specified. * KERN_SUCCESS The usual. * KERN_INVALID_ADDRESS There was a hole in the region. */ kern_return_t mach_vm_msync( vm_map_t map, mach_vm_address_t address, mach_vm_size_t size, vm_sync_t sync_flags) { if (map == VM_MAP_NULL) { return KERN_INVALID_TASK; } return vm_map_msync(map, (vm_map_address_t)address, (vm_map_size_t)size, sync_flags); } /* * vm_msync * * Synchronises the memory range specified with its backing store * image by either flushing or cleaning the contents to the appropriate * memory manager. * * interpretation of sync_flags * VM_SYNC_INVALIDATE - discard pages, only return precious * pages to manager. * * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS) * - discard pages, write dirty or precious * pages back to memory manager. * * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS * - write dirty or precious pages back to * the memory manager. * * VM_SYNC_CONTIGUOUS - does everything normally, but if there * is a hole in the region, and we would * have returned KERN_SUCCESS, return * KERN_INVALID_ADDRESS instead. * * The addressability of the range is limited to that which can * be described by a vm_address_t. * * RETURNS * KERN_INVALID_TASK Bad task parameter * KERN_INVALID_ARGUMENT both sync and async were specified. * KERN_SUCCESS The usual. * KERN_INVALID_ADDRESS There was a hole in the region. */ kern_return_t vm_msync( vm_map_t map, vm_address_t address, vm_size_t size, vm_sync_t sync_flags) { if (map == VM_MAP_NULL) { return KERN_INVALID_TASK; } return vm_map_msync(map, (vm_map_address_t)address, (vm_map_size_t)size, sync_flags); } int vm_toggle_entry_reuse(int toggle, int *old_value) { vm_map_t map = current_map(); assert(!map->is_nested_map); if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) { *old_value = map->disable_vmentry_reuse; } else if (toggle == VM_TOGGLE_SET) { vm_map_entry_t map_to_entry; vm_map_lock(map); vm_map_disable_hole_optimization(map); map->disable_vmentry_reuse = TRUE; __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map)); if (map->first_free == map_to_entry) { map->highest_entry_end = vm_map_min(map); } else { map->highest_entry_end = map->first_free->vme_end; } vm_map_unlock(map); } else if (toggle == VM_TOGGLE_CLEAR) { vm_map_lock(map); map->disable_vmentry_reuse = FALSE; vm_map_unlock(map); } else { return KERN_INVALID_ARGUMENT; } return KERN_SUCCESS; } /* * mach_vm_behavior_set * * Sets the paging behavior attribute for the specified range * in the specified map. * * This routine will fail with KERN_INVALID_ADDRESS if any address * in [start,start+size) is not a valid allocated memory region. */ kern_return_t mach_vm_behavior_set( vm_map_t map, mach_vm_offset_t start, mach_vm_size_t size, vm_behavior_t new_behavior) { vm_map_offset_t align_mask; if ((map == VM_MAP_NULL) || (start + size < start)) { return KERN_INVALID_ARGUMENT; } if (size == 0) { return KERN_SUCCESS; } switch (new_behavior) { case VM_BEHAVIOR_REUSABLE: case VM_BEHAVIOR_REUSE: case VM_BEHAVIOR_CAN_REUSE: case VM_BEHAVIOR_ZERO: /* * Align to the hardware page size, to allow * malloc() to maximize the amount of re-usability, * even on systems with larger software page size. */ align_mask = PAGE_MASK; break; default: align_mask = VM_MAP_PAGE_MASK(map); break; } return vm_map_behavior_set(map, vm_map_trunc_page(start, align_mask), vm_map_round_page(start + size, align_mask), new_behavior); } /* * vm_behavior_set * * Sets the paging behavior attribute for the specified range * in the specified map. * * This routine will fail with KERN_INVALID_ADDRESS if any address * in [start,start+size) is not a valid allocated memory region. * * This routine is potentially limited in addressibility by the * use of vm_offset_t (if the map provided is larger than the * kernel's). */ kern_return_t vm_behavior_set( vm_map_t map, vm_offset_t start, vm_size_t size, vm_behavior_t new_behavior) { if (start + size < start) { return KERN_INVALID_ARGUMENT; } return mach_vm_behavior_set(map, (mach_vm_offset_t) start, (mach_vm_size_t) size, new_behavior); } /* * mach_vm_region: * * User call to obtain information about a region in * a task's address map. Currently, only one flavor is * supported. * * XXX The reserved and behavior fields cannot be filled * in until the vm merge from the IK is completed, and * vm_reserve is implemented. * * XXX Dependency: syscall_vm_region() also supports only one flavor. */ kern_return_t mach_vm_region( vm_map_t map, mach_vm_offset_t *address, /* IN/OUT */ mach_vm_size_t *size, /* OUT */ vm_region_flavor_t flavor, /* IN */ vm_region_info_t info, /* OUT */ mach_msg_type_number_t *count, /* IN/OUT */ mach_port_t *object_name) /* OUT */ { vm_map_offset_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_offset_t)*address; map_size = (vm_map_size_t)*size; /* legacy conversion */ if (VM_REGION_BASIC_INFO == flavor) { flavor = VM_REGION_BASIC_INFO_64; } kr = vm_map_region(map, &map_addr, &map_size, flavor, info, count, object_name); *address = map_addr; *size = map_size; return kr; } /* * vm_region_64 and vm_region: * * User call to obtain information about a region in * a task's address map. Currently, only one flavor is * supported. * * XXX The reserved and behavior fields cannot be filled * in until the vm merge from the IK is completed, and * vm_reserve is implemented. * * XXX Dependency: syscall_vm_region() also supports only one flavor. */ kern_return_t vm_region_64( vm_map_t map, vm_offset_t *address, /* IN/OUT */ vm_size_t *size, /* OUT */ vm_region_flavor_t flavor, /* IN */ vm_region_info_t info, /* OUT */ mach_msg_type_number_t *count, /* IN/OUT */ mach_port_t *object_name) /* OUT */ { vm_map_offset_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_offset_t)*address; map_size = (vm_map_size_t)*size; /* legacy conversion */ if (VM_REGION_BASIC_INFO == flavor) { flavor = VM_REGION_BASIC_INFO_64; } kr = vm_map_region(map, &map_addr, &map_size, flavor, info, count, object_name); *address = CAST_DOWN(vm_offset_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; } return kr; } kern_return_t vm_region( vm_map_t map, vm_address_t *address, /* IN/OUT */ vm_size_t *size, /* OUT */ vm_region_flavor_t flavor, /* IN */ vm_region_info_t info, /* OUT */ mach_msg_type_number_t *count, /* IN/OUT */ mach_port_t *object_name) /* OUT */ { vm_map_address_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region(map, &map_addr, &map_size, flavor, info, count, object_name); *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; } return kr; } /* * vm_region_recurse: A form of vm_region which follows the * submaps in a target map * */ kern_return_t mach_vm_region_recurse( vm_map_t map, mach_vm_address_t *address, mach_vm_size_t *size, uint32_t *depth, vm_region_recurse_info_t info, mach_msg_type_number_t *infoCnt) { vm_map_address_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region_recurse_64( map, &map_addr, &map_size, depth, (vm_region_submap_info_64_t)info, infoCnt); *address = map_addr; *size = map_size; return kr; } /* * vm_region_recurse: A form of vm_region which follows the * submaps in a target map * */ kern_return_t vm_region_recurse_64( vm_map_t map, vm_address_t *address, vm_size_t *size, uint32_t *depth, vm_region_recurse_info_64_t info, mach_msg_type_number_t *infoCnt) { vm_map_address_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region_recurse_64( map, &map_addr, &map_size, depth, (vm_region_submap_info_64_t)info, infoCnt); *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; } return kr; } kern_return_t vm_region_recurse( vm_map_t map, vm_offset_t *address, /* IN/OUT */ vm_size_t *size, /* OUT */ natural_t *depth, /* IN/OUT */ vm_region_recurse_info_t info32, /* IN/OUT */ mach_msg_type_number_t *infoCnt) /* IN/OUT */ { vm_region_submap_info_data_64_t info64; vm_region_submap_info_t info; vm_map_address_t map_addr; vm_map_size_t map_size; kern_return_t kr; if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) { return KERN_INVALID_ARGUMENT; } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; info = (vm_region_submap_info_t)info32; *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64; kr = vm_map_region_recurse_64(map, &map_addr, &map_size, depth, &info64, infoCnt); info->protection = info64.protection; info->max_protection = info64.max_protection; info->inheritance = info64.inheritance; info->offset = (uint32_t)info64.offset; /* trouble-maker */ info->user_tag = info64.user_tag; info->pages_resident = info64.pages_resident; info->pages_shared_now_private = info64.pages_shared_now_private; info->pages_swapped_out = info64.pages_swapped_out; info->pages_dirtied = info64.pages_dirtied; info->ref_count = info64.ref_count; info->shadow_depth = info64.shadow_depth; info->external_pager = info64.external_pager; info->share_mode = info64.share_mode; info->is_submap = info64.is_submap; info->behavior = info64.behavior; info->object_id = info64.object_id; info->user_wired_count = info64.user_wired_count; *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); *infoCnt = VM_REGION_SUBMAP_INFO_COUNT; if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; } return kr; } kern_return_t mach_vm_purgable_control( vm_map_t map, mach_vm_offset_t address, vm_purgable_t control, int *state) { if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { /* not allowed from user-space */ return KERN_INVALID_ARGUMENT; } return vm_map_purgable_control(map, vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)), control, state); } kern_return_t mach_vm_purgable_control_external( mach_port_t target_tport, mach_vm_offset_t address, vm_purgable_t control, int *state) { vm_map_t map; kern_return_t kr; if (control == VM_PURGABLE_GET_STATE) { map = convert_port_to_map_read(target_tport); } else { map = convert_port_to_map(target_tport); } kr = mach_vm_purgable_control(map, address, control, state); vm_map_deallocate(map); return kr; } kern_return_t vm_purgable_control( vm_map_t map, vm_offset_t address, vm_purgable_t control, int *state) { if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { /* not allowed from user-space */ return KERN_INVALID_ARGUMENT; } return vm_map_purgable_control(map, vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)), control, state); } kern_return_t vm_purgable_control_external( mach_port_t target_tport, vm_offset_t address, vm_purgable_t control, int *state) { vm_map_t map; kern_return_t kr; if (control == VM_PURGABLE_GET_STATE) { map = convert_port_to_map_read(target_tport); } else { map = convert_port_to_map(target_tport); } kr = vm_purgable_control(map, address, control, state); vm_map_deallocate(map); return kr; } kern_return_t mach_vm_page_query( vm_map_t map, mach_vm_offset_t offset, int *disposition, int *ref_count) { if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } return vm_map_page_query_internal( map, vm_map_trunc_page(offset, PAGE_MASK), disposition, ref_count); } kern_return_t vm_map_page_query( vm_map_t map, vm_offset_t offset, int *disposition, int *ref_count) { if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; } return vm_map_page_query_internal( map, vm_map_trunc_page(offset, PAGE_MASK), disposition, ref_count); } kern_return_t mach_vm_page_range_query( vm_map_t map, mach_vm_offset_t address, mach_vm_size_t size, mach_vm_address_t dispositions_addr, mach_vm_size_t *dispositions_count) { kern_return_t kr = KERN_SUCCESS; int num_pages = 0, i = 0; mach_vm_size_t curr_sz = 0, copy_sz = 0; mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0; mach_msg_type_number_t count = 0; void *info = NULL; void *local_disp = NULL; vm_map_size_t info_size = 0, local_disp_size = 0; mach_vm_offset_t start = 0, end = 0; int effective_page_shift, effective_page_size, effective_page_mask; if (map == VM_MAP_NULL || dispositions_count == NULL) { return KERN_INVALID_ARGUMENT; } effective_page_shift = vm_self_region_page_shift_safely(map); if (effective_page_shift == -1) { return KERN_INVALID_ARGUMENT; } effective_page_size = (1 << effective_page_shift); effective_page_mask = effective_page_size - 1; if (os_mul_overflow(*dispositions_count, sizeof(int), &disp_buf_req_size)) { return KERN_INVALID_ARGUMENT; } start = vm_map_trunc_page(address, effective_page_mask); end = vm_map_round_page(address + size, effective_page_mask); if (end < start) { return KERN_INVALID_ARGUMENT; } if ((end - start) < size) { /* * Aligned size is less than unaligned size. */ return KERN_INVALID_ARGUMENT; } if (disp_buf_req_size == 0 || (end == start)) { return KERN_SUCCESS; } /* * For large requests, we will go through them * MAX_PAGE_RANGE_QUERY chunk at a time. */ curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY); num_pages = (int) (curr_sz >> effective_page_shift); info_size = num_pages * sizeof(vm_page_info_basic_data_t); info = kalloc_data(info_size, Z_WAITOK); local_disp_size = num_pages * sizeof(int); local_disp = kalloc_data(local_disp_size, Z_WAITOK); if (info == NULL || local_disp == NULL) { kr = KERN_RESOURCE_SHORTAGE; goto out; } while (size) { count = VM_PAGE_INFO_BASIC_COUNT; kr = vm_map_page_range_info_internal( map, start, vm_map_round_page(start + curr_sz, effective_page_mask), effective_page_shift, VM_PAGE_INFO_BASIC, (vm_page_info_t) info, &count); assert(kr == KERN_SUCCESS); for (i = 0; i < num_pages; i++) { ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition; } copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */); kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz); start += curr_sz; disp_buf_req_size -= copy_sz; disp_buf_total_size += copy_sz; if (kr != 0) { break; } if ((disp_buf_req_size == 0) || (curr_sz >= size)) { /* * We might have inspected the full range OR * more than it esp. if the user passed in * non-page aligned start/size and/or if we * descended into a submap. We are done here. */ size = 0; } else { dispositions_addr += copy_sz; size -= curr_sz; curr_sz = MIN(vm_map_round_page(size, effective_page_mask), MAX_PAGE_RANGE_QUERY); num_pages = (int)(curr_sz >> effective_page_shift); } } *dispositions_count = disp_buf_total_size / sizeof(int); out: kfree_data(local_disp, local_disp_size); kfree_data(info, info_size); return kr; } kern_return_t mach_vm_page_info( vm_map_t map, mach_vm_address_t address, vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count) { kern_return_t kr; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } kr = vm_map_page_info(map, address, flavor, info, count); return kr; } /* * task_wire * * Set or clear the map's wiring_required flag. This flag, if set, * will cause all future virtual memory allocation to allocate * user wired memory. Unwiring pages wired down as a result of * this routine is done with the vm_wire interface. */ kern_return_t task_wire( vm_map_t map, boolean_t must_wire __unused) { if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } return KERN_NOT_SUPPORTED; } kern_return_t vm_map_exec_lockdown( vm_map_t map) { if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; } vm_map_lock(map); map->map_disallow_new_exec = TRUE; vm_map_unlock(map); return KERN_SUCCESS; } #if XNU_PLATFORM_MacOSX /* * Now a kernel-private interface (for BootCache * use only). Need a cleaner way to create an * empty vm_map() and return a handle to it. */ kern_return_t vm_region_object_create( vm_map_t target_map, vm_size_t size, ipc_port_t *object_handle) { vm_named_entry_t user_entry; vm_map_t new_map; user_entry = mach_memory_entry_allocate(object_handle); /* Create a named object based on a submap of specified size */ new_map = vm_map_create_options(PMAP_NULL, VM_MAP_MIN_ADDRESS, vm_map_round_page(size, VM_MAP_PAGE_MASK(target_map)), VM_MAP_CREATE_PAGEABLE); vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map)); user_entry->backing.map = new_map; user_entry->internal = TRUE; user_entry->is_sub_map = TRUE; user_entry->offset = 0; user_entry->protection = VM_PROT_ALL; user_entry->size = size; return KERN_SUCCESS; } #endif /* XNU_PLATFORM_MacOSX */ kern_return_t mach_vm_deferred_reclamation_buffer_init( task_t task, mach_vm_offset_t *address, mach_vm_size_t size) { #if CONFIG_DEFERRED_RECLAIM return vm_deferred_reclamation_buffer_init_internal(task, address, size); #else (void) task; (void) address; (void) size; (void) indices; return KERN_NOT_SUPPORTED; #endif /* CONFIG_DEFERRED_RECLAIM */ } kern_return_t mach_vm_deferred_reclamation_buffer_synchronize( task_t task, mach_vm_size_t num_entries_to_reclaim) { #if CONFIG_DEFERRED_RECLAIM return vm_deferred_reclamation_buffer_synchronize_internal(task, num_entries_to_reclaim); #else (void) task; (void) num_entries_to_reclaim; return KERN_NOT_SUPPORTED; #endif /* CONFIG_DEFERRED_RECLAIM */ } kern_return_t mach_vm_deferred_reclamation_buffer_update_reclaimable_bytes(task_t task, mach_vm_size_t reclaimable_bytes) { #if CONFIG_DEFERRED_RECLAIM return vm_deferred_reclamation_buffer_update_reclaimable_bytes_internal(task, reclaimable_bytes); #else (void) task; (void) reclaimable_bytes; return KERN_NOT_SUPPORTED; #endif /* CONFIG_DEFERRED_RECLAIM */ } #if CONFIG_MAP_RANGES extern void qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *)); static int vm_map_user_range_cmp(const void *e1, const void *e2) { const struct vm_map_user_range *r1 = e1; const struct vm_map_user_range *r2 = e2; if (r1->vmur_min_address != r2->vmur_min_address) { return r1->vmur_min_address < r2->vmur_min_address ? -1 : 1; } return 0; } static int mach_vm_range_recipe_v1_cmp(const void *e1, const void *e2) { const mach_vm_range_recipe_v1_t *r1 = e1; const mach_vm_range_recipe_v1_t *r2 = e2; if (r1->range.min_address != r2->range.min_address) { return r1->range.min_address < r2->range.min_address ? -1 : 1; } return 0; } /*! * @function mach_vm_range_create_v1() * * @brief * Handle the backend for mach_vm_range_create() for the * MACH_VM_RANGE_FLAVOR_V1 flavor. * * @description * This call allows to create "ranges" in the map of a task * that have special semantics/policies around placement of * new allocations (in the vm_map_locate_space() sense). * * @returns * - KERN_SUCCESS on success * - KERN_INVALID_ARGUMENT for incorrect arguments * - KERN_NO_SPACE if the maximum amount of ranges would be exceeded * - KERN_MEMORY_PRESENT if any of the requested ranges * overlaps with existing ranges or allocations in the map. */ static kern_return_t mach_vm_range_create_v1( vm_map_t map, mach_vm_range_recipe_v1_t *recipe, uint32_t new_count) { const vm_offset_t mask = VM_MAP_PAGE_MASK(map); vm_map_user_range_t table; kern_return_t kr = KERN_SUCCESS; uint16_t count; struct mach_vm_range void1 = { .min_address = map->default_range.max_address, .max_address = map->data_range.min_address, }; struct mach_vm_range void2 = { .min_address = map->data_range.max_address, #if XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT .max_address = MACH_VM_JUMBO_ADDRESS, #else /* !XNU_TARGET_OS_IOS || !EXTENDED_USER_VA_SUPPORT */ .max_address = vm_map_max(map), #endif /* XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT */ }; qsort(recipe, new_count, sizeof(mach_vm_range_recipe_v1_t), mach_vm_range_recipe_v1_cmp); /* * Step 1: Validate that the recipes have no intersections. */ for (size_t i = 0; i < new_count; i++) { mach_vm_range_t r = &recipe[i].range; mach_vm_size_t s; if (recipe[i].flags) { return KERN_INVALID_ARGUMENT; } static_assert(UMEM_RANGE_ID_FIXED == MACH_VM_RANGE_FIXED); switch (recipe[i].range_tag) { case MACH_VM_RANGE_FIXED: break; default: return KERN_INVALID_ARGUMENT; } if (!VM_MAP_PAGE_ALIGNED(r->min_address, mask) || !VM_MAP_PAGE_ALIGNED(r->max_address, mask) || r->min_address >= r->max_address) { return KERN_INVALID_ARGUMENT; } s = mach_vm_range_size(r); if (!mach_vm_range_contains(&void1, r->min_address, s) && !mach_vm_range_contains(&void2, r->min_address, s)) { return KERN_INVALID_ARGUMENT; } if (i > 0 && recipe[i - 1].range.max_address > recipe[i].range.min_address) { return KERN_INVALID_ARGUMENT; } } vm_map_lock(map); table = map->extra_ranges; count = map->extra_ranges_count; if (count + new_count > VM_MAP_EXTRA_RANGES_MAX) { kr = KERN_NO_SPACE; goto out_unlock; } /* * Step 2: Check that there is no intersection with existing ranges. */ for (size_t i = 0, j = 0; i < new_count && j < count;) { mach_vm_range_t r1 = &recipe[i].range; vm_map_user_range_t r2 = &table[j]; if (r1->max_address <= r2->vmur_min_address) { i++; } else if (r2->vmur_max_address <= r1->min_address) { j++; } else { kr = KERN_MEMORY_PRESENT; goto out_unlock; } } /* * Step 4: commit the new ranges. */ static_assert(VM_MAP_EXTRA_RANGES_MAX * sizeof(struct vm_map_user_range) <= KALLOC_SAFE_ALLOC_SIZE); table = krealloc_data(table, count * sizeof(struct vm_map_user_range), (count + new_count) * sizeof(struct vm_map_user_range), Z_ZERO | Z_WAITOK | Z_NOFAIL); for (size_t i = 0; i < new_count; i++) { static_assert(MACH_VM_MAX_ADDRESS < (1ull << 56)); table[count + i] = (struct vm_map_user_range){ .vmur_min_address = recipe[i].range.min_address, .vmur_max_address = recipe[i].range.max_address, .vmur_range_id = (vm_map_range_id_t)recipe[i].range_tag, }; } qsort(table, count + new_count, sizeof(struct vm_map_user_range), vm_map_user_range_cmp); map->extra_ranges_count += new_count; map->extra_ranges = table; out_unlock: vm_map_unlock(map); if (kr == KERN_SUCCESS) { for (size_t i = 0; i < new_count; i++) { vm_map_kernel_flags_t vmk_flags = { .vmf_fixed = true, .vmf_overwrite = true, .vmkf_overwrite_immutable = true, .vm_tag = recipe[i].vm_tag, }; __assert_only kern_return_t kr2; kr2 = vm_map_enter(map, &recipe[i].range.min_address, mach_vm_range_size(&recipe[i].range), 0, vmk_flags, VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_ALL, VM_INHERIT_DEFAULT); assert(kr2 == KERN_SUCCESS); } } return kr; } kern_return_t mach_vm_range_create( vm_map_t map, mach_vm_range_flavor_t flavor, mach_vm_range_recipes_raw_t recipe, natural_t size) { if (map != current_map()) { return KERN_INVALID_ARGUMENT; } if (!map->uses_user_ranges) { return KERN_NOT_SUPPORTED; } if (size == 0) { return KERN_SUCCESS; } if (flavor == MACH_VM_RANGE_FLAVOR_V1) { mach_vm_range_recipe_v1_t *array; if (size % sizeof(mach_vm_range_recipe_v1_t)) { return KERN_INVALID_ARGUMENT; } size /= sizeof(mach_vm_range_recipe_v1_t); if (size > VM_MAP_EXTRA_RANGES_MAX) { return KERN_NO_SPACE; } array = (mach_vm_range_recipe_v1_t *)recipe; return mach_vm_range_create_v1(map, array, size); } return KERN_INVALID_ARGUMENT; } #else /* !CONFIG_MAP_RANGES */ kern_return_t mach_vm_range_create( vm_map_t map, mach_vm_range_flavor_t flavor, mach_vm_range_recipes_raw_t recipe, natural_t size) { #pragma unused(map, flavor, recipe, size) return KERN_NOT_SUPPORTED; } #endif /* !CONFIG_MAP_RANGES */ /* * These symbols are looked up at runtime by vmware, VirtualBox, * despite not being exported in the symbol sets. */ #if defined(__x86_64__) extern typeof(mach_vm_remap_external) mach_vm_remap; extern typeof(mach_vm_map_external) mach_vm_map; extern typeof(vm_map_external) vm_map; kern_return_t mach_vm_map( vm_map_t target_map, mach_vm_offset_ut *address, mach_vm_size_ut initial_size, mach_vm_offset_ut mask, int flags, ipc_port_t port, memory_object_offset_ut offset, boolean_t copy, vm_prot_ut cur_protection, vm_prot_ut max_protection, vm_inherit_ut inheritance) { return mach_vm_map_external(target_map, address, initial_size, mask, flags, port, offset, copy, cur_protection, max_protection, inheritance); } kern_return_t mach_vm_remap( vm_map_t target_map, mach_vm_offset_ut *address, mach_vm_size_ut size, mach_vm_offset_ut mask, int flags, vm_map_t src_map, mach_vm_offset_ut memory_address, boolean_t copy, vm_prot_ut *cur_protection, /* OUT */ vm_prot_ut *max_protection, /* OUT */ vm_inherit_ut inheritance) { return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address, copy, cur_protection, max_protection, inheritance); } kern_return_t vm_map( vm_map_t target_map, vm_offset_ut *address, vm_size_ut size, vm_offset_ut mask, int flags, ipc_port_t port, vm_offset_ut offset, boolean_t copy, vm_prot_ut cur_protection, vm_prot_ut max_protection, vm_inherit_ut inheritance) { return mach_vm_map(target_map, address, size, mask, flags, port, offset, copy, cur_protection, max_protection, inheritance); } #endif /* __x86_64__ */