1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1987, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)kern_malloc.c 8.4 (Berkeley) 5/20/95
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #include <kern/zalloc.h>
71 #include <kern/kalloc.h>
72 #include <sys/ubc.h> /* mach_to_bsd_errno */
73
74 #include <sys/malloc.h>
75 #include <sys/sysctl.h>
76
77 #include <libkern/libkern.h>
78
79 ZONE_VIEW_DEFINE(ZV_NAMEI, "vfs.namei", KHEAP_ID_DATA_BUFFERS, MAXPATHLEN);
80 KALLOC_HEAP_DEFINE(KERN_OS_MALLOC, "kern_os_malloc", KHEAP_ID_KEXT);
81
82 /*
83 * macOS Only deprecated interfaces, here only for legacy reasons.
84 * There is no internal variant of any of these symbols on purpose.
85 */
86 #if PLATFORM_MacOSX
87
88 #define OSMallocDeprecatedMsg(msg)
89 #include <libkern/OSMalloc.h>
90
91 void *
92 _MALLOC_external(size_t size, int type, int flags);
93 void *
_MALLOC_external(size_t size,int type,int flags)94 _MALLOC_external(size_t size, int type, int flags)
95 {
96 kalloc_heap_t heap = KHEAP_KEXT;
97 void *addr = NULL;
98
99 if (type == M_SONAME) {
100 #if !XNU_TARGET_OS_OSX
101 assert3u(size, <=, UINT8_MAX);
102 #endif /* XNU_TARGET_OS_OSX */
103 heap = KHEAP_SONAME;
104 }
105
106 if (size == 0) {
107 return NULL;
108 }
109
110 static_assert(sizeof(vm_size_t) == sizeof(size_t));
111 static_assert(M_WAITOK == Z_WAITOK);
112 static_assert(M_NOWAIT == Z_NOWAIT);
113 static_assert(M_ZERO == Z_ZERO);
114
115 flags = Z_VM_TAG_BT(flags & Z_KPI_MASK, VM_KERN_MEMORY_KALLOC);
116 addr = kalloc_ext(heap, size, flags, NULL).addr;
117 if (__probable(addr)) {
118 return addr;
119 }
120
121 if (flags & (M_NOWAIT | M_NULL)) {
122 return NULL;
123 }
124
125 /*
126 * We get here when the caller told us to block waiting for memory, but
127 * kalloc said there's no memory left to get. Generally, this means there's a
128 * leak or the caller asked for an impossibly large amount of memory. If the caller
129 * is expecting a NULL return code then it should explicitly set the flag M_NULL.
130 * If the caller isn't expecting a NULL return code, we just panic. This is less
131 * than ideal, but returning NULL when the caller isn't expecting it doesn't help
132 * since the majority of callers don't check the return value and will just
133 * dereference the pointer and trap anyway. We may as well get a more
134 * descriptive message out while we can.
135 */
136 panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
137 }
138
139 void
140 _FREE_external(void *addr, int type);
141 void
_FREE_external(void * addr,int type)142 _FREE_external(void *addr, int type)
143 {
144 /*
145 * hashinit and other functions allocate on behalf of kexts and do not have
146 * a matching hashdestroy, so we sadly have to allow this for now.
147 */
148 kalloc_heap_t heap = KHEAP_ANY;
149
150 if (type == M_SONAME) {
151 /*
152 * On macOS, some KEXT is known to use M_SONAME for M_TEMP allocation
153 */
154 #if !XNU_TARGET_OS_OSX
155 kheap_free_bounded(KHEAP_SONAME, addr, 1, UINT8_MAX);
156 #else
157 kheap_free_addr(heap, addr);
158 #endif /* XNU_TARGET_OS_OSX */
159 return;
160 }
161
162 kheap_free_addr(heap, addr);
163 }
164
165 void
166 _FREE_ZONE_external(void *elem, size_t size, int type);
167 void
_FREE_ZONE_external(void * elem,size_t size,int type __unused)168 _FREE_ZONE_external(void *elem, size_t size, int type __unused)
169 {
170 (kheap_free)(KHEAP_KEXT, elem, size);
171 }
172
173 char *
174 STRDUP_external(const char *string, int type);
175 char *
STRDUP_external(const char * string,int type __unused)176 STRDUP_external(const char *string, int type __unused)
177 {
178 size_t len;
179 char *copy;
180
181 len = strlen(string) + 1;
182 copy = kheap_alloc(KHEAP_KEXT, len, Z_WAITOK);
183 if (copy) {
184 memcpy(copy, string, len);
185 }
186 return copy;
187 }
188
189 static KALLOC_HEAP_DEFINE(OSMALLOC, "osmalloc", KHEAP_ID_KEXT);
190 static queue_head_t OSMalloc_tag_list = QUEUE_HEAD_INITIALIZER(OSMalloc_tag_list);
191 static LCK_GRP_DECLARE(OSMalloc_tag_lck_grp, "OSMalloc_tag");
192 static LCK_SPIN_DECLARE(OSMalloc_tag_lock, &OSMalloc_tag_lck_grp);
193
194 #define OSMalloc_tag_spin_lock() lck_spin_lock(&OSMalloc_tag_lock)
195 #define OSMalloc_tag_unlock() lck_spin_unlock(&OSMalloc_tag_lock)
196
197 extern typeof(OSMalloc_Tagalloc) OSMalloc_Tagalloc_external;
198 OSMallocTag
OSMalloc_Tagalloc_external(const char * str,uint32_t flags)199 OSMalloc_Tagalloc_external(const char *str, uint32_t flags)
200 {
201 OSMallocTag OSMTag;
202
203 OSMTag = kalloc_type(struct _OSMallocTag_, Z_WAITOK | Z_ZERO);
204
205 if (flags & OSMT_PAGEABLE) {
206 OSMTag->OSMT_attr = OSMT_ATTR_PAGEABLE;
207 }
208
209 OSMTag->OSMT_refcnt = 1;
210
211 strlcpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME);
212
213 OSMalloc_tag_spin_lock();
214 enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag);
215 OSMalloc_tag_unlock();
216 OSMTag->OSMT_state = OSMT_VALID;
217 return OSMTag;
218 }
219
220 static void
OSMalloc_Tagref(OSMallocTag tag)221 OSMalloc_Tagref(OSMallocTag tag)
222 {
223 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) {
224 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X",
225 tag->OSMT_name, tag->OSMT_state);
226 }
227
228 os_atomic_inc(&tag->OSMT_refcnt, relaxed);
229 }
230
231 static void
OSMalloc_Tagrele(OSMallocTag tag)232 OSMalloc_Tagrele(OSMallocTag tag)
233 {
234 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) {
235 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X",
236 tag->OSMT_name, tag->OSMT_state);
237 }
238
239 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) != 0) {
240 return;
241 }
242
243 if (os_atomic_cmpxchg(&tag->OSMT_state,
244 OSMT_VALID | OSMT_RELEASED, OSMT_VALID | OSMT_RELEASED, acq_rel)) {
245 OSMalloc_tag_spin_lock();
246 (void)remque((queue_entry_t)tag);
247 OSMalloc_tag_unlock();
248 kfree_type(struct _OSMallocTag_, tag);
249 } else {
250 panic("OSMalloc_Tagrele():'%s' has refcnt 0", tag->OSMT_name);
251 }
252 }
253
254 extern typeof(OSMalloc_Tagfree) OSMalloc_Tagfree_external;
255 void
OSMalloc_Tagfree_external(OSMallocTag tag)256 OSMalloc_Tagfree_external(OSMallocTag tag)
257 {
258 if (!os_atomic_cmpxchg(&tag->OSMT_state,
259 OSMT_VALID, OSMT_VALID | OSMT_RELEASED, acq_rel)) {
260 panic("OSMalloc_Tagfree():'%s' has bad state 0x%08X",
261 tag->OSMT_name, tag->OSMT_state);
262 }
263
264 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) == 0) {
265 OSMalloc_tag_spin_lock();
266 (void)remque((queue_entry_t)tag);
267 OSMalloc_tag_unlock();
268 kfree_type(struct _OSMallocTag_, tag);
269 }
270 }
271
272 extern typeof(OSMalloc) OSMalloc_external;
273 void *
OSMalloc_external(uint32_t size,OSMallocTag tag)274 OSMalloc_external(uint32_t size, OSMallocTag tag)
275 {
276 void *addr = NULL;
277 kern_return_t kr;
278
279 OSMalloc_Tagref(tag);
280 if ((tag->OSMT_attr & OSMT_PAGEABLE) && (size & ~PAGE_MASK)) {
281 if ((kr = kmem_alloc_pageable(kernel_map,
282 (vm_offset_t *)&addr, size, vm_tag_bt())) != KERN_SUCCESS) {
283 addr = NULL;
284 }
285 } else {
286 addr = kheap_alloc(OSMALLOC, size,
287 Z_VM_TAG_BT(Z_WAITOK, VM_KERN_MEMORY_KALLOC));
288 }
289
290 if (!addr) {
291 OSMalloc_Tagrele(tag);
292 }
293
294 return addr;
295 }
296
297 extern typeof(OSMalloc_noblock) OSMalloc_noblock_external;
298 void *
OSMalloc_noblock_external(uint32_t size,OSMallocTag tag)299 OSMalloc_noblock_external(uint32_t size, OSMallocTag tag)
300 {
301 void *addr = NULL;
302
303 if (tag->OSMT_attr & OSMT_PAGEABLE) {
304 return NULL;
305 }
306
307 OSMalloc_Tagref(tag);
308 addr = kheap_alloc(OSMALLOC, (vm_size_t)size,
309 Z_VM_TAG_BT(Z_NOWAIT, VM_KERN_MEMORY_KALLOC));
310 if (addr == NULL) {
311 OSMalloc_Tagrele(tag);
312 }
313
314 return addr;
315 }
316
317 extern typeof(OSFree) OSFree_external;
318 void
OSFree_external(void * addr,uint32_t size,OSMallocTag tag)319 OSFree_external(void *addr, uint32_t size, OSMallocTag tag)
320 {
321 if ((tag->OSMT_attr & OSMT_PAGEABLE)
322 && (size & ~PAGE_MASK)) {
323 kmem_free(kernel_map, (vm_offset_t)addr, size);
324 } else {
325 kheap_free(OSMALLOC, addr, size);
326 }
327
328 OSMalloc_Tagrele(tag);
329 }
330
331 #endif /* PLATFORM_MacOSX */
332 #if DEBUG || DEVELOPMENT
333
334 static int
335 sysctl_zone_map_jetsam_limit SYSCTL_HANDLER_ARGS
336 {
337 #pragma unused(oidp, arg1, arg2)
338 int oldval = 0, val = 0, error = 0;
339
340 oldval = zone_map_jetsam_limit;
341 error = sysctl_io_number(req, oldval, sizeof(int), &val, NULL);
342 if (error || !req->newptr) {
343 return error;
344 }
345
346 return mach_to_bsd_errno(zone_map_jetsam_set_limit(val));
347 }
348 SYSCTL_PROC(_kern, OID_AUTO, zone_map_jetsam_limit,
349 CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_zone_map_jetsam_limit, "I",
350 "Zone map jetsam limit");
351
352
353 extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
354
355 static int
356 sysctl_zone_map_size_and_capacity SYSCTL_HANDLER_ARGS
357 {
358 #pragma unused(oidp, arg1, arg2)
359 uint64_t zstats[2];
360 get_zone_map_size(&zstats[0], &zstats[1]);
361
362 return SYSCTL_OUT(req, &zstats, sizeof(zstats));
363 }
364
365 SYSCTL_PROC(_kern, OID_AUTO, zone_map_size_and_capacity,
366 CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, 0,
367 &sysctl_zone_map_size_and_capacity, "Q",
368 "Current size and capacity of the zone map");
369
370 #endif /* DEBUG || DEVELOPMENT */
371 #if CONFIG_ZLEAKS
372
373 SYSCTL_DECL(_kern_zleak);
374 SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak");
375
376 SYSCTL_INT(_kern_zleak, OID_AUTO, active, CTLFLAG_RD,
377 &zleak_active, 0, "zleak activity");
378
379 /*
380 * kern.zleak.max_zonemap_size
381 *
382 * Read the value of the maximum zonemap size in bytes; useful
383 * as the maximum size that zleak.global_threshold and
384 * zleak.zone_threshold should be set to.
385 */
386 SYSCTL_LONG(_kern_zleak, OID_AUTO, max_zonemap_size,
387 CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, &zleak_max_zonemap_size,
388 "zleak max zonemap size");
389
390
391 static int
392 sysctl_zleak_threshold SYSCTL_HANDLER_ARGS
393 {
394 #pragma unused(oidp, arg2)
395 int error;
396 uint64_t value = *(vm_size_t *)arg1;
397
398 error = sysctl_io_number(req, value, sizeof(value), &value, NULL);
399
400 if (error || !req->newptr) {
401 return error;
402 }
403
404 return mach_to_bsd_errno(zleak_update_threshold(arg1, value));
405 }
406
407 /*
408 * kern.zleak.global_threshold
409 *
410 * Set the global zleak threshold size (in bytes). If the zone map
411 * grows larger than this value, zleaks are automatically activated.
412 *
413 * The default value is set in zleak_init().
414 */
415 SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold,
416 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
417 &zleak_global_tracking_threshold, 0, sysctl_zleak_threshold, "Q",
418 "zleak global threshold");
419
420 /*
421 * kern.zleak.zone_threshold
422 *
423 * Set the per-zone threshold size (in bytes) above which any
424 * zone will automatically start zleak tracking.
425 *
426 * The default value is set in zleak_init().
427 *
428 * Setting this variable will have no effect until zleak tracking is
429 * activated (See above.)
430 */
431 SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold,
432 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
433 &zleak_per_zone_tracking_threshold, 0, sysctl_zleak_threshold, "Q",
434 "zleak per-zone threshold");
435
436 #endif /* CONFIG_ZLEAKS */
437
438 extern uint64_t get_zones_collectable_bytes(void);
439
440 static int
441 sysctl_zones_collectable_bytes SYSCTL_HANDLER_ARGS
442 {
443 #pragma unused(oidp, arg1, arg2)
444 uint64_t zones_free_mem = get_zones_collectable_bytes();
445
446 return SYSCTL_OUT(req, &zones_free_mem, sizeof(zones_free_mem));
447 }
448
449 SYSCTL_PROC(_kern, OID_AUTO, zones_collectable_bytes,
450 CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
451 0, 0, &sysctl_zones_collectable_bytes, "Q",
452 "Collectable memory in zones");
453