1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /**
29 * This header file stores the types, and prototypes used strictly by the pmap
30 * itself. The public pmap API exported to the rest of the kernel should be
31 * located in osfmk/arm64/sptm/pmap/pmap.h.
32 *
33 * This file will automatically include all of the other internal arm/pmap/
34 * headers so .c files will only need to include this one header.
35 */
36 #pragma once
37
38 #include <stdint.h>
39
40 #include <kern/debug.h>
41 #include <kern/locks.h>
42 #include <mach/vm_types.h>
43 #include <mach_assert.h>
44
45 #include <arm/cpu_data.h>
46 #include <arm64/proc_reg.h>
47 #include <arm64/sptm/sptm.h>
48
49 /**
50 * arm64/sptm/pmap/pmap.h and the other /arm/pmap/ internal header files are safe to be
51 * included in this file since they shouldn't rely on any of the internal pmap
52 * header files (so no circular dependencies). Implementation files will only
53 * need to include this one header to get all of the relevant pmap types.
54 */
55 #include <arm64/sptm/pmap/pmap.h>
56 #include <arm64/sptm/pmap/pmap_data.h>
57 #include <arm64/sptm/pmap/pmap_pt_geometry.h>
58
59 #define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \
60 extern __return_type __function_name##_internal __function_args
61
62 /**
63 * Global variables exported to the rest of the internal pmap implementation.
64 */
65 extern lck_grp_t pmap_lck_grp;
66 extern pmap_paddr_t avail_start;
67 extern pmap_paddr_t avail_end;
68 extern uint32_t pmap_max_asids;
69
70 /**
71 * SPTM TODO: The following flag is set up based on the presence and
72 * configuration of the 'sptm-stability-hacks' boot-arg; this
73 * is used in certain codepaths that do not properly function
74 * today in SPTM systems to make the system more stable and fully
75 * able to boot to user space.
76 */
77 extern bool sptm_stability_hacks;
78
79 /**
80 * Functions exported to the rest of the internal pmap implementation.
81 */
82
83 extern void pmap_remove_range_options(
84 pmap_t, vm_map_address_t, vm_map_address_t, int);
85
86 extern void pmap_tte_deallocate(
87 pmap_t, vm_offset_t, tt_entry_t *, unsigned int);
88
89 #if defined(PVH_FLAG_EXEC)
90 extern void pmap_set_ptov_ap(unsigned int, unsigned int, boolean_t);
91 #endif /* defined(PVH_FLAG_EXEC) */
92
93 extern pmap_t current_pmap(void);
94 extern void pmap_tt_ledger_credit(pmap_t, vm_size_t);
95 extern void pmap_tt_ledger_debit(pmap_t, vm_size_t);
96
97 extern void write_pte(pt_entry_t *, pt_entry_t);
98
99 /**
100 * The qsort function is used by various parts of the pmap but doesn't contain
101 * its own header file with prototype so it must be manually extern'd.
102 *
103 * The `cmpfunc_t` type is a pointer to a function that should return the
104 * following:
105 *
106 * return < 0 for a < b
107 * 0 for a == b
108 * > 0 for a > b
109 */
110 typedef int (*cmpfunc_t)(const void *a, const void *b);
111 extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
112
113 /**
114 * Inline and macro functions exported for usage by other pmap modules.
115 *
116 * In an effort to not cause any performance regressions while breaking up the
117 * pmap, I'm keeping all functions originally marked as "static inline", as
118 * inline and moving them into header files to be shared across the pmap
119 * modules. In reality, many of these functions probably don't need to be inline
120 * and can be moved back into a .c file.
121 *
122 * TODO: rdar://70538514 (PMAP Cleanup: re-evaluate whether inline functions should actually be inline)
123 */
124
125 /* Helper macro for rounding an address up to a correctly aligned value. */
126 #define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1))
127
128 /**
129 * pmap_data.h must be included before this point so that pmap_lock_mode_t is
130 * defined before the rest of the locking code.
131 */
132
133 /**
134 * Initialize a pmap object's reader/writer lock.
135 *
136 * @param pmap The pmap whose lock to initialize.
137 */
138 static inline void
pmap_lock_init(pmap_t pmap)139 pmap_lock_init(pmap_t pmap)
140 {
141 lck_rw_init(&pmap->rwlock, &pmap_lck_grp, 0);
142 }
143
144 /**
145 * Destroy a pmap object's reader/writer lock.
146 *
147 * @param pmap The pmap whose lock to destroy.
148 */
149 static inline void
pmap_lock_destroy(pmap_t pmap)150 pmap_lock_destroy(pmap_t pmap)
151 {
152 lck_rw_destroy(&pmap->rwlock, &pmap_lck_grp);
153 }
154
155 /**
156 * Initialize a pmap object's TXM reader/writer lock.
157 *
158 * @param pmap The pmap whose TXM lock to initialize.
159 */
160 static inline void
pmap_txmlock_init(pmap_t pmap)161 pmap_txmlock_init(pmap_t pmap)
162 {
163 lck_rw_init(&pmap->txm_lck, &pmap_lck_grp, 0);
164 }
165
166 /**
167 * Destroy a pmap object's TXM reader/writer lock.
168 *
169 * @param pmap The pmap whose TXM lock to destroy.
170 */
171 static inline void
pmap_txmlock_destroy(pmap_t pmap)172 pmap_txmlock_destroy(pmap_t pmap)
173 {
174 lck_rw_destroy(&pmap->txm_lck, &pmap_lck_grp);
175 }
176
177 /**
178 * Assert that the pmap lock is held in the given mode.
179 *
180 * @note See pmap_lock() below for an explanation of the special handling
181 * we do for kernel_pmap.
182 *
183 * @param pmap The pmap whose lock to assert is being held.
184 * @param mode The mode the lock should be held in.
185 */
186 static inline void
pmap_assert_locked(__unused pmap_t pmap,__unused pmap_lock_mode_t mode)187 pmap_assert_locked(__unused pmap_t pmap, __unused pmap_lock_mode_t mode)
188 {
189 #if MACH_ASSERT
190 if (__improbable(sptm_stability_hacks)) {
191 mode = PMAP_LOCK_EXCLUSIVE;
192 }
193
194 switch (mode) {
195 case PMAP_LOCK_SHARED:
196 if (pmap != kernel_pmap) {
197 LCK_RW_ASSERT(&pmap->rwlock, LCK_RW_ASSERT_SHARED);
198 }
199 break;
200 case PMAP_LOCK_EXCLUSIVE:
201 LCK_RW_ASSERT(&pmap->rwlock, LCK_RW_ASSERT_EXCLUSIVE);
202 break;
203 case PMAP_LOCK_HELD:
204 if (pmap != kernel_pmap) {
205 LCK_RW_ASSERT(&pmap->rwlock, LCK_RW_ASSERT_HELD);
206 }
207 break;
208 default:
209 panic("%s: Unknown pmap_lock_mode. pmap=%p, mode=%d", __FUNCTION__, pmap, mode);
210 }
211 #endif
212 }
213
214 /**
215 * Assert that the pmap lock is held in any mode.
216 *
217 * @param pmap The pmap whose lock should be held.
218 */
219 __unused static inline void
pmap_assert_locked_any(__unused pmap_t pmap)220 pmap_assert_locked_any(__unused pmap_t pmap)
221 {
222 if (pmap != kernel_pmap) {
223 LCK_RW_ASSERT(&pmap->rwlock, LCK_RW_ASSERT_HELD);
224 }
225 }
226
227 /**
228 * Acquire a pmap object's reader/writer lock as either shared (read-only) or
229 * exclusive (read/write).
230 *
231 * @note If this function is called to request shared acquisition of the kernel pmap
232 * lock, the lock will not be acquired as a performance optimization. See the
233 * the explanation in the function body for why this is safe to do.
234 *
235 * @param pmap The pmap whose lock to acquire.
236 * @param mode Whether to grab the lock as shared (read-only) or exclusive (read/write).
237 */
238 static inline void
pmap_lock(pmap_t pmap,pmap_lock_mode_t mode)239 pmap_lock(pmap_t pmap, pmap_lock_mode_t mode)
240 {
241 if (__improbable(sptm_stability_hacks)) {
242 mode = PMAP_LOCK_EXCLUSIVE;
243 }
244
245 switch (mode) {
246 case PMAP_LOCK_SHARED:
247 /**
248 * There are three cases in which we hold the pmap lock exclusive:
249 * 1) Removal of a leaf-level page table during pmap_remove(),
250 * to prevent concurrent mapping into the to-be-deleted table.
251 * 2) Nesting/unnesting of a region of one pmap into another, to
252 * both concurrent nesting and concurrent mapping into the nested
253 * region.
254 * 3) Installing a new page table during pmap_expand(), to prevent
255 * another thread from concurrently expanding the same pmap at
256 * the same location.
257 * Of the above, the kernel pmap only participates in 3) (nesting
258 * and table removal are only done for user pmaps). Because the
259 * exclusive lock in case 3) above is only meant to synchronize
260 * against other instances of case 3), we can effectively elide
261 * shared holders of the kernel pmap because there is no case in
262 * which shared<>exclusive locking of the kernel pmap matters.
263 */
264 if (pmap != kernel_pmap) {
265 lck_rw_lock_shared(&pmap->rwlock);
266 }
267 break;
268 case PMAP_LOCK_EXCLUSIVE:
269 lck_rw_lock_exclusive(&pmap->rwlock);
270 break;
271 default:
272 panic("%s: Unknown pmap_lock_mode. pmap=%p, mode=%d", __func__, pmap, mode);
273 }
274 }
275
276 /**
277 * Attempt to acquire the pmap lock in the specified mode. If the lock couldn't
278 * be acquired, then return immediately instead of spinning.
279 *
280 * @param pmap The pmap whose lock to attempt to acquire.
281 * @param mode Whether to grab the lock as shared (read-only) or exclusive (read/write).
282 *
283 * @return True if the lock was acquired, false otherwise.
284 */
285 static inline bool
pmap_try_lock(pmap_t pmap,pmap_lock_mode_t mode)286 pmap_try_lock(pmap_t pmap, pmap_lock_mode_t mode)
287 {
288 bool ret = false;
289
290 if (__improbable(sptm_stability_hacks)) {
291 mode = PMAP_LOCK_EXCLUSIVE;
292 }
293
294 switch (mode) {
295 case PMAP_LOCK_SHARED:
296 if (pmap != kernel_pmap) {
297 ret = lck_rw_try_lock_shared(&pmap->rwlock);
298 } else {
299 ret = true;
300 }
301 break;
302 case PMAP_LOCK_EXCLUSIVE:
303 ret = lck_rw_try_lock_exclusive(&pmap->rwlock);
304 break;
305 default:
306 panic("%s: Unknown pmap_lock_mode. pmap=%p, mode=%d", __func__, pmap, mode);
307 }
308
309 return ret;
310 }
311
312 /**
313 * Attempts to promote an already acquired pmap lock from shared to exclusive.
314 *
315 * @param pmap The pmap whose lock should be promoted from shared to exclusive.
316 *
317 * @return True if successfully promoted, otherwise false upon failure in
318 * which case the shared lock is dropped.
319 */
320 static inline bool
pmap_lock_shared_to_exclusive(pmap_t pmap)321 pmap_lock_shared_to_exclusive(pmap_t pmap)
322 {
323 pmap_assert_locked(pmap, PMAP_LOCK_SHARED);
324
325 if ((pmap == kernel_pmap) || __improbable(sptm_stability_hacks)) {
326 return true;
327 }
328
329 return lck_rw_lock_shared_to_exclusive(&pmap->rwlock);
330 }
331
332 /**
333 * Release a pmap object's reader/writer lock.
334 *
335 * @param pmap The pmap whose lock to release.
336 * @param mode Which mode the lock should be in at time of release.
337 */
338 static inline void
pmap_unlock(pmap_t pmap,pmap_lock_mode_t mode)339 pmap_unlock(pmap_t pmap, pmap_lock_mode_t mode)
340 {
341 if (__improbable(sptm_stability_hacks)) {
342 mode = PMAP_LOCK_EXCLUSIVE;
343 }
344
345 switch (mode) {
346 case PMAP_LOCK_SHARED:
347 if (pmap != kernel_pmap) {
348 lck_rw_unlock_shared(&pmap->rwlock);
349 }
350 break;
351 case PMAP_LOCK_EXCLUSIVE:
352 lck_rw_unlock_exclusive(&pmap->rwlock);
353 break;
354 default:
355 panic("%s: Unknown pmap_lock_mode. pmap=%p, mode=%d", __func__, pmap, mode);
356 }
357 }
358