1 /*
2 * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <stdint.h>
30 #include <stdbool.h>
31 #include <kern/assert.h>
32 #include <kern/zalloc.h>
33 #include <mach/mach_vm.h>
34 #include <mach/vm_param.h>
35 #include <libkern/libkern.h>
36 #include <libkern/OSAtomic.h>
37 #include <sys/queue.h>
38 #include <kern/thread.h>
39 #include <kern/debug.h>
40
41 #include "kasan.h"
42 #include "kasan_internal.h"
43
44 int __asan_option_detect_stack_use_after_return = 0;
45
46 #define FAKESTACK_HEADER_SZ 64
47 #define FAKESTACK_NUM_SZCLASS 7
48
49 #define FAKESTACK_UNUSED 0 /* waiting to be collected at next gc - forced by clang */
50 #define FAKESTACK_ALLOCATED 1
51 #define FAKESTACK_FREED 2
52
53 #if FAKESTACK
54
55 #define FAKESTACK_NORETURN
56
57 struct fakestack_header {
58 LIST_ENTRY(fakestack_header) list;
59 void *site; /* allocation site */
60 struct {
61 uint8_t flag;
62 vm_size_t realsz : 52;
63 vm_size_t sz_class : 4;
64 };
65 uint64_t __pad0;
66 };
67 _Static_assert(sizeof(struct fakestack_header) <= FAKESTACK_HEADER_SZ, "fakestack_header size mismatch");
68
69 static zone_t fakestack_zones[FAKESTACK_NUM_SZCLASS];
70 static char fakestack_names[FAKESTACK_NUM_SZCLASS][16];
71 static const unsigned long fakestack_min = 1 << 6;
72 static const unsigned long __unused fakestack_max = 1 << 16;
73
74 /*
75 * Enter a fakestack critical section in a reentrant-safe fashion. Returns true on
76 * success with the kasan lock held.
77 */
78 static bool
thread_enter_fakestack(boolean_t * flags)79 thread_enter_fakestack(boolean_t *flags)
80 {
81 thread_t cur = current_thread();
82 if (cur && kasan_lock_held(cur)) {
83 /* current thread is already in kasan - fail */
84 return false;
85 }
86 kasan_lock(flags);
87 return true;
88 }
89
90 static volatile long suspend_count;
91 static const long suspend_threshold = 20;
92
93 void
kasan_fakestack_suspend(void)94 kasan_fakestack_suspend(void)
95 {
96 if (OSIncrementAtomicLong(&suspend_count) == suspend_threshold) {
97 __asan_option_detect_stack_use_after_return = 0;
98 }
99 }
100
101 void
kasan_fakestack_resume(void)102 kasan_fakestack_resume(void)
103 {
104 long orig = OSDecrementAtomicLong(&suspend_count);
105 assert(orig >= 0);
106
107 if (fakestack_enabled && orig == suspend_threshold) {
108 __asan_option_detect_stack_use_after_return = 1;
109 }
110 }
111
112 static bool
ptr_is_on_stack(uptr ptr)113 ptr_is_on_stack(uptr ptr)
114 {
115 vm_offset_t base = dtrace_get_kernel_stack(current_thread());
116
117 if (ptr >= base && ptr < (base + kernel_stack_size)) {
118 return true;
119 } else {
120 return false;
121 }
122 }
123
124 /* free all unused fakestack entries */
125 void
kasan_fakestack_gc(thread_t thread)126 kasan_fakestack_gc(thread_t thread)
127 {
128 struct fakestack_header *cur, *tmp;
129 LIST_HEAD(, fakestack_header) tofree = LIST_HEAD_INITIALIZER(tofree);
130
131 boolean_t flags;
132 if (!thread_enter_fakestack(&flags)) {
133 panic("expected success entering fakestack");
134 }
135
136 /* move the unused objects off the per-thread list... */
137 struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head;
138 LIST_FOREACH_SAFE(cur, head, list, tmp) {
139 if (cur->flag == FAKESTACK_UNUSED) {
140 LIST_REMOVE(cur, list);
141 LIST_INSERT_HEAD(&tofree, cur, list);
142 cur->flag = FAKESTACK_FREED;
143 }
144 }
145
146 kasan_unlock(flags);
147
148 /* ... then actually free them */
149 LIST_FOREACH_SAFE(cur, &tofree, list, tmp) {
150 LIST_REMOVE(cur, list);
151
152 zone_t zone = fakestack_zones[cur->sz_class];
153 size_t sz = (fakestack_min << cur->sz_class) + FAKESTACK_HEADER_SZ;
154
155 void *ptr = (void *)cur;
156 kasan_free_internal(&ptr, &sz, KASAN_HEAP_FAKESTACK, &zone, cur->realsz, 0, FAKESTACK_QUARANTINE);
157 if (ptr) {
158 zfree(zone, ptr);
159 }
160 }
161 }
162
163 static uint8_t **
fakestack_flag_ptr(vm_offset_t ptr,vm_size_t sz)164 fakestack_flag_ptr(vm_offset_t ptr, vm_size_t sz)
165 {
166 uint8_t **x = (uint8_t **)ptr;
167 size_t idx = sz / 8;
168 return &x[idx - 1];
169 }
170
171 static uptr ALWAYS_INLINE
kasan_fakestack_alloc(int sz_class,size_t realsz)172 kasan_fakestack_alloc(int sz_class, size_t realsz)
173 {
174 if (!__asan_option_detect_stack_use_after_return) {
175 return 0;
176 }
177
178 if (sz_class >= FAKESTACK_NUM_SZCLASS) {
179 return 0;
180 }
181
182 uptr ret = 0;
183 size_t sz = fakestack_min << sz_class;
184 assert(realsz <= sz);
185 assert(sz <= fakestack_max);
186 zone_t zone = fakestack_zones[sz_class];
187
188 boolean_t flags;
189 if (!thread_enter_fakestack(&flags)) {
190 return 0;
191 }
192
193 ret = (uptr)zalloc_noblock(zone);
194
195 if (ret) {
196 size_t leftrz = 32 + FAKESTACK_HEADER_SZ;
197 size_t validsz = realsz - 32 - 16; /* remove redzones */
198 size_t rightrz = sz - validsz - 32; /* 16 bytes, plus whatever is left over */
199 struct fakestack_header *hdr = (struct fakestack_header *)ret;
200
201 kasan_poison(ret, validsz, leftrz, rightrz, ASAN_STACK_RZ);
202
203 hdr->site = __builtin_return_address(0);
204 hdr->realsz = realsz;
205 hdr->sz_class = sz_class;
206 hdr->flag = FAKESTACK_ALLOCATED;
207 ret += FAKESTACK_HEADER_SZ;
208
209 *fakestack_flag_ptr(ret, sz) = &hdr->flag; /* back ptr to the slot */
210 struct fakestack_header_list *head = &kasan_get_thread_data(current_thread())->fakestack_head;
211 LIST_INSERT_HEAD(head, hdr, list);
212 }
213
214 kasan_unlock(flags);
215 return ret;
216 }
217
218 static void NOINLINE
kasan_fakestack_free(int sz_class,uptr dst,size_t realsz)219 kasan_fakestack_free(int sz_class, uptr dst, size_t realsz)
220 {
221 if (ptr_is_on_stack(dst)) {
222 return;
223 }
224
225 assert(realsz <= (fakestack_min << sz_class));
226
227 vm_size_t sz = fakestack_min << sz_class;
228 zone_t zone = fakestack_zones[sz_class];
229 assert(zone);
230
231 /* TODO: check the magic? */
232
233 dst -= FAKESTACK_HEADER_SZ;
234 sz += FAKESTACK_HEADER_SZ;
235
236 struct fakestack_header *hdr = (struct fakestack_header *)dst;
237 assert(hdr->sz_class == sz_class);
238
239 boolean_t flags;
240 kasan_lock(&flags);
241
242 LIST_REMOVE(hdr, list);
243
244 kasan_free_internal((void **)&dst, &sz, KASAN_HEAP_FAKESTACK, &zone, realsz, 1, FAKESTACK_QUARANTINE);
245 if (dst) {
246 zfree(zone, dst);
247 }
248
249 kasan_unlock(flags);
250 }
251
252 void NOINLINE
kasan_fakestack_drop(thread_t thread)253 kasan_fakestack_drop(thread_t thread)
254 {
255 boolean_t flags;
256 if (!thread_enter_fakestack(&flags)) {
257 panic("expected success entering fakestack");
258 }
259
260 struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head;
261 struct fakestack_header *cur;
262 LIST_FOREACH(cur, head, list) {
263 if (cur->flag == FAKESTACK_ALLOCATED) {
264 cur->flag = FAKESTACK_UNUSED;
265 }
266 }
267
268 kasan_unlock(flags);
269 }
270
271 void NOINLINE
kasan_init_fakestack(void)272 kasan_init_fakestack(void)
273 {
274 /* allocate the fakestack zones */
275 for (int i = 0; i < FAKESTACK_NUM_SZCLASS; i++) {
276 unsigned long sz = (fakestack_min << i) + FAKESTACK_HEADER_SZ;
277 size_t maxsz = 256UL * 1024;
278
279 if (i <= 3) {
280 /* size classes 0..3 are much more common */
281 maxsz *= 4;
282 }
283
284 snprintf(fakestack_names[i], 16, "fakestack.%d", i);
285 fakestack_zones[i] = zone_create_ext(fakestack_names[i], sz,
286 ZC_NOCALLOUT | ZC_NOGC | ZC_NOCACHING |
287 ZC_KASAN_NOREDZONE | ZC_KASAN_NOQUARANTINE,
288 ZONE_ID_ANY, ^(zone_t z) {
289 zone_set_exhaustible(z, maxsz / sz);
290 });
291 zone_fill_initially(fakestack_zones[i], maxsz / sz);
292 }
293
294 /* globally enable */
295 if (fakestack_enabled) {
296 __asan_option_detect_stack_use_after_return = 1;
297 }
298 }
299
300 static int
sysctl_fakestack_enable(__unused struct sysctl_oid * oidp,__unused void * arg1,int __unused arg2,struct sysctl_req * req)301 sysctl_fakestack_enable(__unused struct sysctl_oid *oidp, __unused void *arg1, int __unused arg2, struct sysctl_req *req)
302 {
303 int ch, err, val;
304
305 err = sysctl_io_number(req, fakestack_enabled, sizeof(fakestack_enabled), &val, &ch);
306 if (err == 0 && ch) {
307 fakestack_enabled = !!val;
308 __asan_option_detect_stack_use_after_return = !!val;
309 }
310
311 return err;
312 }
313
314 SYSCTL_PROC(_kern_kasan, OID_AUTO, fakestack,
315 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
316 0, 0, sysctl_fakestack_enable, "I", "");
317
318 #else /* FAKESTACK */
319
320 #define FAKESTACK_NORETURN OS_NORETURN
321
322 void
kasan_init_fakestack(void)323 kasan_init_fakestack(void)
324 {
325 assert(__asan_option_detect_stack_use_after_return == 0);
326 }
327
328 void
kasan_unpoison_fakestack(thread_t __unused thread)329 kasan_unpoison_fakestack(thread_t __unused thread)
330 {
331 assert(__asan_option_detect_stack_use_after_return == 0);
332 }
333
334 void NOINLINE
kasan_fakestack_drop(thread_t __unused thread)335 kasan_fakestack_drop(thread_t __unused thread)
336 {
337 assert(__asan_option_detect_stack_use_after_return == 0);
338 }
339
340 void
kasan_fakestack_gc(thread_t __unused thread)341 kasan_fakestack_gc(thread_t __unused thread)
342 {
343 assert(__asan_option_detect_stack_use_after_return == 0);
344 }
345
346 void
kasan_fakestack_suspend(void)347 kasan_fakestack_suspend(void)
348 {
349 assert(__asan_option_detect_stack_use_after_return == 0);
350 }
351
352 void
kasan_fakestack_resume(void)353 kasan_fakestack_resume(void)
354 {
355 assert(__asan_option_detect_stack_use_after_return == 0);
356 }
357
358 static uptr
kasan_fakestack_alloc(int __unused sz_class,size_t __unused realsz)359 kasan_fakestack_alloc(int __unused sz_class, size_t __unused realsz)
360 {
361 assert(__asan_option_detect_stack_use_after_return == 0);
362 return 0;
363 }
364
365 static void OS_NORETURN
kasan_fakestack_free(int __unused sz_class,uptr __unused dst,size_t __unused realsz)366 kasan_fakestack_free(int __unused sz_class, uptr __unused dst, size_t __unused realsz)
367 {
368 assert(__asan_option_detect_stack_use_after_return == 0);
369 panic("fakestack_free called on non-FAKESTACK config");
370 }
371
372 #endif
373
374 void
kasan_init_thread(struct kasan_thread_data * td)375 kasan_init_thread(struct kasan_thread_data *td)
376 {
377 LIST_INIT(&td->fakestack_head);
378 }
379
380 #define FAKESTACK_DECLARE(szclass) \
381 uptr __asan_stack_malloc_##szclass(size_t sz) { return kasan_fakestack_alloc(szclass, sz); } \
382 void FAKESTACK_NORETURN __asan_stack_free_##szclass (uptr dst, size_t sz) { kasan_fakestack_free(szclass, dst, sz); }
383
384 FAKESTACK_DECLARE(0)
385 FAKESTACK_DECLARE(1)
386 FAKESTACK_DECLARE(2)
387 FAKESTACK_DECLARE(3)
388 FAKESTACK_DECLARE(4)
389 FAKESTACK_DECLARE(5)
390 FAKESTACK_DECLARE(6)
391 FAKESTACK_DECLARE(7)
392 FAKESTACK_DECLARE(8)
393 FAKESTACK_DECLARE(9)
394 FAKESTACK_DECLARE(10)
395