1 /*
2 * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <stdint.h>
30 #include <stdbool.h>
31 #include <kern/assert.h>
32 #include <kern/zalloc.h>
33 #include <mach/mach_vm.h>
34 #include <mach/vm_param.h>
35 #include <libkern/libkern.h>
36 #include <libkern/OSAtomic.h>
37 #include <sys/queue.h>
38 #include <kern/thread.h>
39 #include <kern/debug.h>
40
41 #include "kasan.h"
42 #include "kasan_internal.h"
43
44 int __asan_option_detect_stack_use_after_return = 0;
45 int fakestack_enabled = 0;
46
47 #define FAKESTACK_HEADER_SZ 64
48 #define FAKESTACK_NUM_SZCLASS 7
49
50 #define FAKESTACK_UNUSED 0 /* waiting to be collected at next gc - forced by clang */
51 #define FAKESTACK_ALLOCATED 1
52 #define FAKESTACK_FREED 2
53
54 #if FAKESTACK
55
56 #define FAKESTACK_NORETURN
57
58 struct fakestack_header {
59 LIST_ENTRY(fakestack_header) list;
60 void *site; /* allocation site */
61 struct {
62 uint8_t flag;
63 vm_size_t realsz : 52;
64 vm_size_t sz_class : 4;
65 };
66 uint64_t __pad0;
67 };
68 _Static_assert(sizeof(struct fakestack_header) <= FAKESTACK_HEADER_SZ, "fakestack_header size mismatch");
69
70 static zone_t fakestack_zones[FAKESTACK_NUM_SZCLASS];
71 static char fakestack_names[FAKESTACK_NUM_SZCLASS][16];
72 static const unsigned long fakestack_min = 1 << 6;
73 static const unsigned long __unused fakestack_max = 1 << 16;
74
75 /*
76 * Enter a fakestack critical section in a reentrant-safe fashion. Returns true on
77 * success with the kasan lock held.
78 */
79 static bool
thread_enter_fakestack(boolean_t * flags)80 thread_enter_fakestack(boolean_t *flags)
81 {
82 thread_t cur = current_thread();
83 if (cur && kasan_lock_held(cur)) {
84 /* current thread is already in kasan - fail */
85 return false;
86 }
87 kasan_lock(flags);
88 return true;
89 }
90
91 static volatile long suspend_count;
92 static const long suspend_threshold = 20;
93
94 void
kasan_fakestack_suspend(void)95 kasan_fakestack_suspend(void)
96 {
97 if (OSIncrementAtomicLong(&suspend_count) == suspend_threshold) {
98 __asan_option_detect_stack_use_after_return = 0;
99 }
100 }
101
102 void
kasan_fakestack_resume(void)103 kasan_fakestack_resume(void)
104 {
105 long orig = OSDecrementAtomicLong(&suspend_count);
106 assert(orig >= 0);
107
108 if (fakestack_enabled && orig == suspend_threshold) {
109 __asan_option_detect_stack_use_after_return = 1;
110 }
111 }
112
113 static bool
ptr_is_on_stack(uptr ptr)114 ptr_is_on_stack(uptr ptr)
115 {
116 vm_offset_t base = dtrace_get_kernel_stack(current_thread());
117
118 if (ptr >= base && ptr < (base + kernel_stack_size)) {
119 return true;
120 } else {
121 return false;
122 }
123 }
124
125 /* free all unused fakestack entries */
126 void
kasan_fakestack_gc(thread_t thread)127 kasan_fakestack_gc(thread_t thread)
128 {
129 struct fakestack_header *cur, *tmp;
130 LIST_HEAD(, fakestack_header) tofree = LIST_HEAD_INITIALIZER(tofree);
131
132 boolean_t flags;
133 if (!thread_enter_fakestack(&flags)) {
134 panic("expected success entering fakestack");
135 }
136
137 /* move the unused objects off the per-thread list... */
138 struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head;
139 LIST_FOREACH_SAFE(cur, head, list, tmp) {
140 if (cur->flag == FAKESTACK_UNUSED) {
141 LIST_REMOVE(cur, list);
142 LIST_INSERT_HEAD(&tofree, cur, list);
143 cur->flag = FAKESTACK_FREED;
144 }
145 }
146
147 kasan_unlock(flags);
148
149 /* ... then actually free them */
150 LIST_FOREACH_SAFE(cur, &tofree, list, tmp) {
151 LIST_REMOVE(cur, list);
152
153 zone_t zone = fakestack_zones[cur->sz_class];
154 size_t sz = (fakestack_min << cur->sz_class) + FAKESTACK_HEADER_SZ;
155
156 void *ptr = (void *)cur;
157 kasan_free_internal(&ptr, &sz, KASAN_HEAP_FAKESTACK, &zone, cur->realsz, 0, FAKESTACK_QUARANTINE);
158 if (ptr) {
159 zfree(zone, ptr);
160 }
161 }
162 }
163
164 static uint8_t **
fakestack_flag_ptr(vm_offset_t ptr,vm_size_t sz)165 fakestack_flag_ptr(vm_offset_t ptr, vm_size_t sz)
166 {
167 uint8_t **x = (uint8_t **)ptr;
168 size_t idx = sz / 8;
169 return &x[idx - 1];
170 }
171
172 static uptr ALWAYS_INLINE
kasan_fakestack_alloc(int sz_class,size_t realsz)173 kasan_fakestack_alloc(int sz_class, size_t realsz)
174 {
175 if (!__asan_option_detect_stack_use_after_return) {
176 return 0;
177 }
178
179 if (sz_class >= FAKESTACK_NUM_SZCLASS) {
180 return 0;
181 }
182
183 uptr ret = 0;
184 size_t sz = fakestack_min << sz_class;
185 assert(realsz <= sz);
186 assert(sz <= fakestack_max);
187 zone_t zone = fakestack_zones[sz_class];
188
189 boolean_t flags;
190 if (!thread_enter_fakestack(&flags)) {
191 return 0;
192 }
193
194 ret = (uptr)zalloc_noblock(zone);
195
196 if (ret) {
197 size_t leftrz = 32 + FAKESTACK_HEADER_SZ;
198 size_t validsz = realsz - 32 - 16; /* remove redzones */
199 size_t rightrz = sz - validsz - 32; /* 16 bytes, plus whatever is left over */
200 struct fakestack_header *hdr = (struct fakestack_header *)ret;
201
202 kasan_poison(ret, validsz, leftrz, rightrz, ASAN_STACK_RZ);
203
204 hdr->site = __builtin_return_address(0);
205 hdr->realsz = realsz;
206 hdr->sz_class = sz_class;
207 hdr->flag = FAKESTACK_ALLOCATED;
208 ret += FAKESTACK_HEADER_SZ;
209
210 *fakestack_flag_ptr(ret, sz) = &hdr->flag; /* back ptr to the slot */
211 struct fakestack_header_list *head = &kasan_get_thread_data(current_thread())->fakestack_head;
212 LIST_INSERT_HEAD(head, hdr, list);
213 }
214
215 kasan_unlock(flags);
216 return ret;
217 }
218
219 static void NOINLINE
kasan_fakestack_free(int sz_class,uptr dst,size_t realsz)220 kasan_fakestack_free(int sz_class, uptr dst, size_t realsz)
221 {
222 if (ptr_is_on_stack(dst)) {
223 return;
224 }
225
226 assert(realsz <= (fakestack_min << sz_class));
227
228 vm_size_t sz = fakestack_min << sz_class;
229 zone_t zone = fakestack_zones[sz_class];
230 assert(zone);
231
232 /* TODO: check the magic? */
233
234 dst -= FAKESTACK_HEADER_SZ;
235 sz += FAKESTACK_HEADER_SZ;
236
237 struct fakestack_header *hdr = (struct fakestack_header *)dst;
238 assert(hdr->sz_class == sz_class);
239
240 boolean_t flags;
241 kasan_lock(&flags);
242
243 LIST_REMOVE(hdr, list);
244
245 kasan_free_internal((void **)&dst, &sz, KASAN_HEAP_FAKESTACK, &zone, realsz, 1, FAKESTACK_QUARANTINE);
246 if (dst) {
247 zfree(zone, dst);
248 }
249
250 kasan_unlock(flags);
251 }
252
253 void NOINLINE
kasan_fakestack_drop(thread_t thread)254 kasan_fakestack_drop(thread_t thread)
255 {
256 boolean_t flags;
257 if (!thread_enter_fakestack(&flags)) {
258 panic("expected success entering fakestack");
259 }
260
261 struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head;
262 struct fakestack_header *cur;
263 LIST_FOREACH(cur, head, list) {
264 if (cur->flag == FAKESTACK_ALLOCATED) {
265 cur->flag = FAKESTACK_UNUSED;
266 }
267 }
268
269 kasan_unlock(flags);
270 }
271
272 void NOINLINE
kasan_init_fakestack(void)273 kasan_init_fakestack(void)
274 {
275 /* allocate the fakestack zones */
276 for (int i = 0; i < FAKESTACK_NUM_SZCLASS; i++) {
277 unsigned long sz = (fakestack_min << i) + FAKESTACK_HEADER_SZ;
278 size_t maxsz = 256UL * 1024;
279
280 if (i <= 3) {
281 /* size classes 0..3 are much more common */
282 maxsz *= 4;
283 }
284
285 snprintf(fakestack_names[i], 16, "fakestack.%d", i);
286 fakestack_zones[i] = zone_create_ext(fakestack_names[i], sz,
287 ZC_NOCALLOUT | ZC_NOGC | ZC_NOCACHING |
288 ZC_KASAN_NOREDZONE | ZC_KASAN_NOQUARANTINE,
289 ZONE_ID_ANY, ^(zone_t z) {
290 zone_set_exhaustible(z, maxsz / sz);
291 });
292 zone_fill_initially(fakestack_zones[i], maxsz / sz);
293 }
294
295 /* globally enable */
296 if (fakestack_enabled) {
297 __asan_option_detect_stack_use_after_return = 1;
298 }
299 }
300
301 static int
sysctl_fakestack_enable(__unused struct sysctl_oid * oidp,__unused void * arg1,int __unused arg2,struct sysctl_req * req)302 sysctl_fakestack_enable(__unused struct sysctl_oid *oidp, __unused void *arg1, int __unused arg2, struct sysctl_req *req)
303 {
304 int ch, err, val;
305
306 err = sysctl_io_number(req, fakestack_enabled, sizeof(fakestack_enabled), &val, &ch);
307 if (err == 0 && ch) {
308 fakestack_enabled = !!val;
309 __asan_option_detect_stack_use_after_return = !!val;
310 }
311
312 return err;
313 }
314
315 SYSCTL_PROC(_kern_kasan, OID_AUTO, fakestack,
316 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
317 0, 0, sysctl_fakestack_enable, "I", "");
318
319 #else /* FAKESTACK */
320
321 #define FAKESTACK_NORETURN OS_NORETURN
322
323 void
kasan_init_fakestack(void)324 kasan_init_fakestack(void)
325 {
326 assert(__asan_option_detect_stack_use_after_return == 0);
327 }
328
329 void
kasan_unpoison_fakestack(thread_t __unused thread)330 kasan_unpoison_fakestack(thread_t __unused thread)
331 {
332 assert(__asan_option_detect_stack_use_after_return == 0);
333 }
334
335 void NOINLINE
kasan_fakestack_drop(thread_t __unused thread)336 kasan_fakestack_drop(thread_t __unused thread)
337 {
338 assert(__asan_option_detect_stack_use_after_return == 0);
339 }
340
341 void
kasan_fakestack_gc(thread_t __unused thread)342 kasan_fakestack_gc(thread_t __unused thread)
343 {
344 assert(__asan_option_detect_stack_use_after_return == 0);
345 }
346
347 void
kasan_fakestack_suspend(void)348 kasan_fakestack_suspend(void)
349 {
350 assert(__asan_option_detect_stack_use_after_return == 0);
351 }
352
353 void
kasan_fakestack_resume(void)354 kasan_fakestack_resume(void)
355 {
356 assert(__asan_option_detect_stack_use_after_return == 0);
357 }
358
359 static uptr
kasan_fakestack_alloc(int __unused sz_class,size_t __unused realsz)360 kasan_fakestack_alloc(int __unused sz_class, size_t __unused realsz)
361 {
362 assert(__asan_option_detect_stack_use_after_return == 0);
363 return 0;
364 }
365
366 static void OS_NORETURN
kasan_fakestack_free(int __unused sz_class,uptr __unused dst,size_t __unused realsz)367 kasan_fakestack_free(int __unused sz_class, uptr __unused dst, size_t __unused realsz)
368 {
369 assert(__asan_option_detect_stack_use_after_return == 0);
370 panic("fakestack_free called on non-FAKESTACK config");
371 }
372
373 #endif
374
375 void
kasan_init_thread(struct kasan_thread_data * td)376 kasan_init_thread(struct kasan_thread_data *td)
377 {
378 LIST_INIT(&td->fakestack_head);
379 }
380
381 #define FAKESTACK_DECLARE(szclass) \
382 uptr __asan_stack_malloc_##szclass(size_t sz) { return kasan_fakestack_alloc(szclass, sz); } \
383 void FAKESTACK_NORETURN __asan_stack_free_##szclass (uptr dst, size_t sz) { kasan_fakestack_free(szclass, dst, sz); }
384
385 FAKESTACK_DECLARE(0)
386 FAKESTACK_DECLARE(1)
387 FAKESTACK_DECLARE(2)
388 FAKESTACK_DECLARE(3)
389 FAKESTACK_DECLARE(4)
390 FAKESTACK_DECLARE(5)
391 FAKESTACK_DECLARE(6)
392 FAKESTACK_DECLARE(7)
393 FAKESTACK_DECLARE(8)
394 FAKESTACK_DECLARE(9)
395 FAKESTACK_DECLARE(10)
396