1 /*
2 * Copyright (c) 2003-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Kernel stack management routines.
30 */
31
32 #include <mach/mach_host.h>
33 #include <mach/mach_types.h>
34 #include <mach/processor_set.h>
35
36 #include <kern/kern_types.h>
37 #include <kern/lock_group.h>
38 #include <kern/mach_param.h>
39 #include <kern/percpu.h>
40 #include <kern/processor.h>
41 #include <kern/thread.h>
42 #include <kern/zalloc.h>
43 #include <kern/kalloc.h>
44 #include <kern/ledger.h>
45
46 #include <vm/vm_map.h>
47 #include <vm/vm_kern.h>
48
49 #include <mach_debug.h>
50 #include <san/kasan.h>
51
52 /*
53 * We allocate stacks from generic kernel VM.
54 *
55 * The stack_free_list can only be accessed at splsched,
56 * because stack_alloc_try/thread_invoke operate at splsched.
57 */
58
59 static SIMPLE_LOCK_DECLARE(stack_lock_data, 0);
60 #define stack_lock() simple_lock(&stack_lock_data, LCK_GRP_NULL)
61 #define stack_unlock() simple_unlock(&stack_lock_data)
62
63 #define STACK_CACHE_SIZE 2
64
65 static vm_offset_t stack_free_list;
66
67 static unsigned int stack_free_count, stack_free_hiwat; /* free list count */
68 static unsigned int stack_hiwat;
69 unsigned int stack_total; /* current total count */
70 unsigned long long stack_allocs; /* total count of allocations */
71
72 static unsigned int stack_free_target;
73 static int stack_free_delta;
74
75 static unsigned int stack_new_count; /* total new stack allocations */
76
77 static SECURITY_READ_ONLY_LATE(vm_offset_t) stack_addr_mask;
78 SECURITY_READ_ONLY_LATE(vm_offset_t) kernel_stack_size;
79 SECURITY_READ_ONLY_LATE(vm_offset_t) kernel_stack_mask;
80 vm_offset_t kernel_stack_depth_max;
81
82 struct stack_cache {
83 vm_offset_t free;
84 unsigned int count;
85 };
86 static struct stack_cache PERCPU_DATA(stack_cache);
87
88 /*
89 * The next field is at the base of the stack,
90 * so the low end is left unsullied.
91 */
92 #define stack_next(stack) \
93 (*((vm_offset_t *)((stack) + kernel_stack_size) - 1))
94
95 static inline int
log2(vm_offset_t size)96 log2(vm_offset_t size)
97 {
98 int result;
99 for (result = 0; size > 0; result++) {
100 size >>= 1;
101 }
102 return result;
103 }
104
105 static inline vm_offset_t
roundup_pow2(vm_offset_t size)106 roundup_pow2(vm_offset_t size)
107 {
108 return 1UL << (log2(size - 1) + 1);
109 }
110
111 static vm_offset_t stack_alloc_internal(void);
112 static void stack_free_stack(vm_offset_t);
113
114 static void
stack_init(void)115 stack_init(void)
116 {
117 uint32_t kernel_stack_pages = atop(KERNEL_STACK_SIZE);
118
119 kernel_stack_size = KERNEL_STACK_SIZE;
120 kernel_stack_mask = -KERNEL_STACK_SIZE;
121
122 if (PE_parse_boot_argn("kernel_stack_pages",
123 &kernel_stack_pages,
124 sizeof(kernel_stack_pages))) {
125 kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
126 }
127
128 if (kernel_stack_size < round_page(kernel_stack_size)) {
129 panic("stack_init: stack size %p not a multiple of page size %d",
130 (void *) kernel_stack_size, PAGE_SIZE);
131 }
132
133 stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
134 kernel_stack_mask = ~stack_addr_mask;
135 }
136 STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, stack_init);
137
138 /*
139 * stack_alloc:
140 *
141 * Allocate a stack for a thread, may
142 * block.
143 */
144
145 static vm_offset_t
stack_alloc_internal(void)146 stack_alloc_internal(void)
147 {
148 vm_offset_t stack = 0;
149 spl_t s;
150 kma_flags_t flags = KMA_NOFAIL | KMA_GUARD_FIRST | KMA_GUARD_LAST |
151 KMA_KSTACK | KMA_KOBJECT | KMA_ZERO;
152
153 s = splsched();
154 stack_lock();
155 stack_allocs++;
156 stack = stack_free_list;
157 if (stack != 0) {
158 stack_free_list = stack_next(stack);
159 stack_free_count--;
160 } else {
161 if (++stack_total > stack_hiwat) {
162 stack_hiwat = stack_total;
163 }
164 stack_new_count++;
165 }
166 stack_free_delta--;
167 stack_unlock();
168 splx(s);
169
170 if (stack == 0) {
171 /*
172 * Request guard pages on either side of the stack. Ask
173 * kernel_memory_allocate() for two extra pages to account
174 * for these.
175 */
176
177 kernel_memory_allocate(kernel_map, &stack,
178 kernel_stack_size + ptoa(2), stack_addr_mask,
179 flags, VM_KERN_MEMORY_STACK);
180
181 /*
182 * The stack address that comes back is the address of the lower
183 * guard page. Skip past it to get the actual stack base address.
184 */
185
186 stack += PAGE_SIZE;
187 }
188 return stack;
189 }
190
191 void
stack_alloc(thread_t thread)192 stack_alloc(
193 thread_t thread)
194 {
195 assert(thread->kernel_stack == 0);
196 machine_stack_attach(thread, stack_alloc_internal());
197 }
198
199 void
stack_handoff(thread_t from,thread_t to)200 stack_handoff(thread_t from, thread_t to)
201 {
202 assert(from == current_thread());
203 machine_stack_handoff(from, to);
204 }
205
206 /*
207 * stack_free:
208 *
209 * Detach and free the stack for a thread.
210 */
211 void
stack_free(thread_t thread)212 stack_free(
213 thread_t thread)
214 {
215 vm_offset_t stack = machine_stack_detach(thread);
216
217 assert(stack);
218 if (stack != thread->reserved_stack) {
219 stack_free_stack(stack);
220 }
221 }
222
223 void
stack_free_reserved(thread_t thread)224 stack_free_reserved(
225 thread_t thread)
226 {
227 if (thread->reserved_stack != thread->kernel_stack) {
228 stack_free_stack(thread->reserved_stack);
229 }
230 }
231
232 static void
stack_free_stack(vm_offset_t stack)233 stack_free_stack(
234 vm_offset_t stack)
235 {
236 struct stack_cache *cache;
237 spl_t s;
238
239 #if KASAN_DEBUG
240 /* Sanity check - stack should be unpoisoned by now */
241 assert(kasan_check_shadow(stack, kernel_stack_size, 0));
242 #endif
243
244 s = splsched();
245 cache = PERCPU_GET(stack_cache);
246 if (cache->count < STACK_CACHE_SIZE) {
247 stack_next(stack) = cache->free;
248 cache->free = stack;
249 cache->count++;
250 } else {
251 stack_lock();
252 stack_next(stack) = stack_free_list;
253 stack_free_list = stack;
254 if (++stack_free_count > stack_free_hiwat) {
255 stack_free_hiwat = stack_free_count;
256 }
257 stack_free_delta++;
258 stack_unlock();
259 }
260 splx(s);
261 }
262
263 /*
264 * stack_alloc_try:
265 *
266 * Non-blocking attempt to allocate a
267 * stack for a thread.
268 *
269 * Returns TRUE on success.
270 *
271 * Called at splsched.
272 */
273 boolean_t
stack_alloc_try(thread_t thread)274 stack_alloc_try(
275 thread_t thread)
276 {
277 struct stack_cache *cache;
278 vm_offset_t stack;
279
280 cache = PERCPU_GET(stack_cache);
281 stack = cache->free;
282 if (stack != 0) {
283 cache->free = stack_next(stack);
284 cache->count--;
285 } else {
286 if (stack_free_list != 0) {
287 stack_lock();
288 stack = stack_free_list;
289 if (stack != 0) {
290 stack_free_list = stack_next(stack);
291 stack_free_count--;
292 stack_free_delta--;
293 }
294 stack_unlock();
295 }
296 }
297
298 if (stack != 0 || (stack = thread->reserved_stack) != 0) {
299 machine_stack_attach(thread, stack);
300 return TRUE;
301 }
302
303 return FALSE;
304 }
305
306 static unsigned int stack_collect_tick, last_stack_tick;
307
308 /*
309 * stack_collect:
310 *
311 * Free excess kernel stacks, may
312 * block.
313 */
314 void
stack_collect(void)315 stack_collect(void)
316 {
317 if (stack_collect_tick != last_stack_tick) {
318 unsigned int target;
319 vm_offset_t stack;
320 spl_t s;
321
322 s = splsched();
323 stack_lock();
324
325 target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
326 target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
327
328 while (stack_free_count > target) {
329 stack = stack_free_list;
330 stack_free_list = stack_next(stack);
331 stack_free_count--; stack_total--;
332 stack_unlock();
333 splx(s);
334
335 /*
336 * Get the stack base address, then decrement by one page
337 * to account for the lower guard page. Add two extra pages
338 * to the size to account for the guard pages on both ends
339 * that were originally requested when the stack was allocated
340 * back in stack_alloc().
341 */
342
343 stack = (vm_offset_t)vm_map_trunc_page(
344 stack,
345 VM_MAP_PAGE_MASK(kernel_map));
346 stack -= PAGE_SIZE;
347 kmem_free(kernel_map, stack, kernel_stack_size + ptoa(2));
348 stack = 0;
349
350 s = splsched();
351 stack_lock();
352
353 target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
354 target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
355 }
356
357 last_stack_tick = stack_collect_tick;
358
359 stack_unlock();
360 splx(s);
361 }
362 }
363
364 /*
365 * compute_stack_target:
366 *
367 * Computes a new target free list count
368 * based on recent alloc / free activity.
369 *
370 * Limits stack collection to once per
371 * computation period.
372 */
373 void
compute_stack_target(__unused void * arg)374 compute_stack_target(
375 __unused void *arg)
376 {
377 spl_t s;
378
379 s = splsched();
380 stack_lock();
381
382 if (stack_free_target > 5) {
383 stack_free_target = (4 * stack_free_target) / 5;
384 } else if (stack_free_target > 0) {
385 stack_free_target--;
386 }
387
388 stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
389
390 stack_free_delta = 0;
391 stack_collect_tick++;
392
393 stack_unlock();
394 splx(s);
395 }
396
397 /* OBSOLETE */
398 void stack_privilege(
399 thread_t thread);
400
401 void
stack_privilege(__unused thread_t thread)402 stack_privilege(
403 __unused thread_t thread)
404 {
405 /* OBSOLETE */
406 }
407
408 /*
409 * Return info on stack usage for threads in a specific processor set
410 */
411 kern_return_t
processor_set_stack_usage(processor_set_t pset,unsigned int * totalp,vm_size_t * spacep,vm_size_t * residentp,vm_size_t * maxusagep,vm_offset_t * maxstackp)412 processor_set_stack_usage(
413 processor_set_t pset,
414 unsigned int *totalp,
415 vm_size_t *spacep,
416 vm_size_t *residentp,
417 vm_size_t *maxusagep,
418 vm_offset_t *maxstackp)
419 {
420 #if !MACH_DEBUG
421 return KERN_NOT_SUPPORTED;
422 #else
423 unsigned int total = 0;
424 thread_t thread;
425
426 if (pset == PROCESSOR_SET_NULL || pset != &pset0) {
427 return KERN_INVALID_ARGUMENT;
428 }
429
430 lck_mtx_lock(&tasks_threads_lock);
431
432 queue_iterate(&threads, thread, thread_t, threads) {
433 total += (thread->kernel_stack != 0);
434 }
435
436 lck_mtx_unlock(&tasks_threads_lock);
437
438 *totalp = total;
439 *residentp = *spacep = total * round_page(kernel_stack_size);
440 *maxusagep = 0;
441 *maxstackp = 0;
442 return KERN_SUCCESS;
443
444 #endif /* MACH_DEBUG */
445 }
446
447 vm_offset_t
min_valid_stack_address(void)448 min_valid_stack_address(void)
449 {
450 return (vm_offset_t)vm_map_min(kernel_map);
451 }
452
453 vm_offset_t
max_valid_stack_address(void)454 max_valid_stack_address(void)
455 {
456 return (vm_offset_t)vm_map_max(kernel_map);
457 }
458