1 /*
2 * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1997 Apple Inc.
30 *
31 */
32 #include <libkern/c++/OSMetaClass.h>
33 #include <libkern/c++/OSKext.h>
34 #include <libkern/c++/OSLib.h>
35 #include <libkern/c++/OSSymbol.h>
36 #include <IOKit/IOKitDebug.h>
37
38 #include <sys/cdefs.h>
39 #if defined(HAS_APPLE_PAC)
40 #include <ptrauth.h>
41 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer))
42 #else /* defined(HAS_APPLE_PAC) */
43 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) (x))
44 #endif /* !defined(HAS_APPLE_PAC) */
45
46 __BEGIN_DECLS
47
48 #include <string.h>
49 #include <mach/mach_types.h>
50 #include <libkern/kernel_mach_header.h>
51 #include <libkern/prelink.h>
52 #include <stdarg.h>
53
54 #if KASAN
55 #include <san/kasan.h>
56 #endif
57
58
59 #if PRAGMA_MARK
60 #pragma mark Constants &c.
61 #endif /* PRAGMA_MARK */
62 OSKextLogSpec kOSRuntimeLogSpec =
63 kOSKextLogErrorLevel |
64 kOSKextLogLoadFlag |
65 kOSKextLogKextBookkeepingFlag;
66
67 #if PRAGMA_MARK
68 #pragma mark Logging Bootstrap
69 #endif /* PRAGMA_MARK */
70 /*********************************************************************
71 * kern_os Logging Bootstrap
72 *
73 * We can't call in to OSKext until the kernel's C++ environment is up
74 * and running, so let's mask those references with a check variable.
75 * We print unconditionally if C++ isn't up, but if that's the case
76 * we've generally hit a serious error in kernel init!
77 *********************************************************************/
78 static bool gKernelCPPInitialized = false;
79
80 #define OSRuntimeLog(kext, flags, format, args ...) \
81 do { \
82 if (gKernelCPPInitialized) { \
83 OSKextLog((kext), (flags), (format), ## args); \
84 } else { \
85 printf((format), ## args); \
86 } \
87 } while (0)
88
89 #if PRAGMA_MARK
90 #pragma mark Libkern Init
91 #endif /* PRAGMA_MARK */
92 /*********************************************************************
93 * Libkern Init
94 *********************************************************************/
95
96 #if __GNUC__ >= 3
97 void __dead2
__cxa_pure_virtual(void)98 __cxa_pure_virtual( void )
99 {
100 panic("%s", __FUNCTION__);
101 }
102 #else
103 void __dead2
__pure_virtual(void)104 __pure_virtual( void )
105 {
106 panic("%s", __FUNCTION__);
107 }
108 #endif
109
110 extern lck_grp_t * IOLockGroup;
111 extern kmod_info_t g_kernel_kmod_info;
112
113 enum {
114 kOSSectionNamesDefault = 0,
115 kOSSectionNamesBuiltinKext = 1,
116 kOSSectionNamesCount = 2,
117 };
118 enum {
119 kOSSectionNameInitializer = 0,
120 kOSSectionNameFinalizer = 1,
121 kOSSectionNameCount = 2
122 };
123
124 static const char *
125 gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = {
126 { SECT_MODINITFUNC, SECT_MODTERMFUNC },
127 { kBuiltinInitSection, kBuiltinTermSection }
128 };
129
130 void
OSlibkernInit(void)131 OSlibkernInit(void)
132 {
133 // This must be called before calling OSRuntimeInitializeCPP.
134 OSMetaClassBase::initialize();
135
136 g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header;
137
138 if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) {
139 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) {
140 panic("OSRuntime: C++ runtime failed to initialize.");
141 }
142
143 gKernelCPPInitialized = true;
144
145 return;
146 }
147
148 __END_DECLS
149
150 #if PRAGMA_MARK
151 #pragma mark C++ Runtime Load/Unload
152 #endif /* PRAGMA_MARK */
153 /*********************************************************************
154 * kern_os C++ Runtime Load/Unload
155 *********************************************************************/
156
157 typedef void (*structor_t)(void);
158
159 static bool
OSRuntimeCallStructorsInSection(OSKext * theKext,kmod_info_t * kmodInfo,void * metaHandle,kernel_segment_command_t * segment,const char * sectionName,uintptr_t textStart,uintptr_t textEnd)160 OSRuntimeCallStructorsInSection(
161 OSKext * theKext,
162 kmod_info_t * kmodInfo,
163 void * metaHandle,
164 kernel_segment_command_t * segment,
165 const char * sectionName,
166 uintptr_t textStart,
167 uintptr_t textEnd)
168 {
169 kernel_section_t * section;
170 bool result = TRUE;
171
172 for (section = firstsect(segment);
173 section != NULL;
174 section = nextsect(segment, section)) {
175 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) {
176 continue;
177 }
178 if (section->size == 0) {
179 continue;
180 }
181
182 structor_t * structors = (structor_t *)section->addr;
183 if (!structors) {
184 continue;
185 }
186
187 structor_t structor;
188 uintptr_t value;
189 unsigned long num_structors = section->size / sizeof(structor_t);
190 unsigned int hit_null_structor = 0;
191 unsigned long firstIndex = 0;
192
193 if (textStart) {
194 // bsearch for any in range
195 unsigned long baseIdx;
196 unsigned long lim;
197 firstIndex = num_structors;
198 for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) {
199 structor = structors[baseIdx + (lim >> 1)];
200 if (!structor) {
201 panic("%s: null structor", kmodInfo->name);
202 }
203 value = PTRAUTH_STRIP_STRUCTOR(structor);
204 if ((value >= textStart) && (value < textEnd)) {
205 firstIndex = (baseIdx + (lim >> 1));
206 // scan back for the first in range
207 for (; firstIndex; firstIndex--) {
208 structor = structors[firstIndex - 1];
209 value = PTRAUTH_STRIP_STRUCTOR(structor);
210 if ((value < textStart) || (value >= textEnd)) {
211 break;
212 }
213 }
214 break;
215 }
216 if (textStart > value) {
217 // move right
218 baseIdx += (lim >> 1) + 1;
219 lim--;
220 }
221 // else move left
222 }
223 baseIdx = (baseIdx + (lim >> 1));
224 }
225 for (;
226 (firstIndex < num_structors)
227 && (!metaHandle || OSMetaClass::checkModLoad(metaHandle));
228 firstIndex++) {
229 if ((structor = structors[firstIndex])) {
230 value = PTRAUTH_STRIP_STRUCTOR(structor);
231 if ((textStart && (value < textStart))
232 || (textEnd && (value >= textEnd))) {
233 break;
234 }
235 (*structor)();
236 } else if (!hit_null_structor) {
237 hit_null_structor = 1;
238 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
239 "Null structor in kext %s segment %s!",
240 kmodInfo->name, section->segname);
241 }
242 }
243 if (metaHandle) {
244 result = OSMetaClass::checkModLoad(metaHandle);
245 }
246 break;
247 } /* for (section...) */
248 return result;
249 }
250
251 /*********************************************************************
252 *********************************************************************/
253 kern_return_t
OSRuntimeFinalizeCPP(OSKext * theKext)254 OSRuntimeFinalizeCPP(
255 OSKext * theKext)
256 {
257 kern_return_t result = KMOD_RETURN_FAILURE;
258 void * metaHandle = NULL;// do not free
259 kernel_mach_header_t * header;
260 kernel_segment_command_t * segment;
261 kmod_info_t * kmodInfo;
262 const char ** sectionNames;
263 uintptr_t textStart;
264 uintptr_t textEnd;
265
266 textStart = 0;
267 textEnd = 0;
268 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
269 if (theKext) {
270 if (!theKext->isCPPInitialized()) {
271 result = KMOD_RETURN_SUCCESS;
272 goto finish;
273 }
274 kmodInfo = theKext->kmod_info;
275 if (!kmodInfo || !kmodInfo->address) {
276 result = kOSKextReturnInvalidArgument;
277 goto finish;
278 }
279 header = (kernel_mach_header_t *)kmodInfo->address;
280 if (theKext->flags.builtin) {
281 header = (kernel_mach_header_t *)g_kernel_kmod_info.address;
282 textStart = kmodInfo->address;
283 textEnd = textStart + kmodInfo->size;
284 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
285 }
286 } else {
287 kmodInfo = &g_kernel_kmod_info;
288 header = (kernel_mach_header_t *)kmodInfo->address;
289 }
290
291 /* OSKext checks for this condition now, but somebody might call
292 * this function directly (the symbol is exported....).
293 */
294 if (OSMetaClass::modHasInstance(kmodInfo->name)) {
295 // xxx - Don't log under errors? this is more of an info thing
296 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
297 "Can't tear down kext %s C++; classes have instances:",
298 kmodInfo->name);
299 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec);
300 result = kOSMetaClassHasInstances;
301 goto finish;
302 }
303
304 /* Tell the meta class system that we are starting to unload.
305 * metaHandle isn't actually needed on the finalize path,
306 * so we don't check it here, even though OSMetaClass::postModLoad() will
307 * return a failure (it only does actual work on the init path anyhow).
308 */
309 metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
310
311 OSSymbol::checkForPageUnload((void *)kmodInfo->address,
312 (void *)(kmodInfo->address + kmodInfo->size));
313
314 header = (kernel_mach_header_t *)kmodInfo->address;
315 segment = firstsegfromheader(header);
316
317 for (segment = firstsegfromheader(header);
318 segment != NULL;
319 segment = nextsegfromheader(header, segment)) {
320 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
321 sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
322 }
323
324 (void)OSMetaClass::postModLoad(metaHandle);
325
326 if (theKext) {
327 theKext->setCPPInitialized(false);
328 }
329 result = KMOD_RETURN_SUCCESS;
330 finish:
331 return result;
332 }
333
334 #if defined(HAS_APPLE_PAC)
335 #if !KASAN
336 /*
337 * Place this function in __KLD,__text on non-kasan builds so it gets unmapped
338 * after CTRR lockdown.
339 */
340 __attribute__((noinline, section("__KLD,__text")))
341 #endif
342 static void
OSRuntimeSignStructorsInSegment(kernel_segment_command_t * segment)343 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment)
344 {
345 kernel_section_t * section;
346 structor_t * structors;
347 volatile structor_t structor;
348 size_t idx, num_structors;
349
350 for (section = firstsect(segment);
351 section != NULL;
352 section = nextsect(segment, section)) {
353 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags))
354 && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) {
355 continue;
356 }
357 structors = (structor_t *)section->addr;
358 if (!structors) {
359 continue;
360 }
361 num_structors = section->size / sizeof(structor_t);
362 for (idx = 0; idx < num_structors; idx++) {
363 structor = structors[idx];
364 if (NULL == structor) {
365 continue;
366 }
367 structor = ptrauth_strip(structor, ptrauth_key_function_pointer);
368 structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void)));
369 structors[idx] = structor;
370 }
371 } /* for (section...) */
372 }
373 #endif
374
375 /*********************************************************************
376 *********************************************************************/
377 void
OSRuntimeSignStructors(kernel_mach_header_t * header __unused)378 OSRuntimeSignStructors(
379 kernel_mach_header_t * header __unused)
380 {
381 #if defined(HAS_APPLE_PAC)
382
383 kernel_segment_command_t * segment;
384
385 for (segment = firstsegfromheader(header);
386 segment != NULL;
387 segment = nextsegfromheader(header, segment)) {
388 OSRuntimeSignStructorsInSegment(segment);
389 } /* for (segment...) */
390 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */
391 }
392
393 /*********************************************************************
394 *********************************************************************/
395 void
OSRuntimeSignStructorsInFileset(kernel_mach_header_t * fileset_header __unused)396 OSRuntimeSignStructorsInFileset(
397 kernel_mach_header_t * fileset_header __unused)
398 {
399 #if defined(HAS_APPLE_PAC)
400 struct load_command *lc;
401
402 lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header));
403 for (uint32_t i = 0; i < fileset_header->ncmds; i++,
404 lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) {
405 if (lc->cmd == LC_FILESET_ENTRY) {
406 struct fileset_entry_command *fse;
407 kernel_mach_header_t *mh;
408
409 fse = (struct fileset_entry_command *)(uintptr_t)lc;
410 mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr);
411 OSRuntimeSignStructors(mh);
412 } else if (lc->cmd == LC_SEGMENT_64) {
413 /*
414 * Slide/adjust all LC_SEGMENT_64 commands in the fileset
415 * (and any sections in those segments)
416 */
417 kernel_segment_command_t *seg;
418 seg = (kernel_segment_command_t *)(uintptr_t)lc;
419 OSRuntimeSignStructorsInSegment(seg);
420 }
421 }
422
423 #endif /* defined(HAS_APPLE_PAC) */
424 }
425
426 /*********************************************************************
427 *********************************************************************/
428 kern_return_t
OSRuntimeInitializeCPP(OSKext * theKext)429 OSRuntimeInitializeCPP(
430 OSKext * theKext)
431 {
432 kern_return_t result = KMOD_RETURN_FAILURE;
433 kernel_mach_header_t * header = NULL;
434 void * metaHandle = NULL;// do not free
435 bool load_success = true;
436 kernel_segment_command_t * segment = NULL;// do not free
437 kernel_segment_command_t * failure_segment = NULL; // do not free
438 kmod_info_t * kmodInfo;
439 const char ** sectionNames;
440 uintptr_t textStart;
441 uintptr_t textEnd;
442
443 textStart = 0;
444 textEnd = 0;
445 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
446 if (theKext) {
447 if (theKext->isCPPInitialized()) {
448 result = KMOD_RETURN_SUCCESS;
449 goto finish;
450 }
451
452 kmodInfo = theKext->kmod_info;
453 if (!kmodInfo || !kmodInfo->address) {
454 result = kOSKextReturnInvalidArgument;
455 goto finish;
456 }
457 header = (kernel_mach_header_t *)kmodInfo->address;
458
459 if (theKext->flags.builtin) {
460 header = (kernel_mach_header_t *)g_kernel_kmod_info.address;
461 textStart = kmodInfo->address;
462 textEnd = textStart + kmodInfo->size;
463 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
464 }
465 } else {
466 kmodInfo = &g_kernel_kmod_info;
467 header = (kernel_mach_header_t *)kmodInfo->address;
468 }
469
470 /* Tell the meta class system that we are starting the load
471 */
472 metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
473 assert(metaHandle);
474 if (!metaHandle) {
475 goto finish;
476 }
477
478 /* NO GOTO PAST HERE. */
479
480 /* Scan the header for all constructor sections, in any
481 * segment, and invoke the constructors within those sections.
482 */
483 for (segment = firstsegfromheader(header);
484 segment != NULL && load_success;
485 segment = nextsegfromheader(header, segment)) {
486 /* Record the current segment in the event of a failure.
487 */
488 failure_segment = segment;
489 load_success = OSRuntimeCallStructorsInSection(
490 theKext, kmodInfo, metaHandle, segment,
491 sectionNames[kOSSectionNameInitializer],
492 textStart, textEnd);
493 } /* for (segment...) */
494
495 /* We failed so call all of the destructors. We must do this before
496 * calling OSMetaClass::postModLoad() as the OSMetaClass destructors
497 * will alter state (in the metaHandle) used by that function.
498 */
499 if (!load_success) {
500 /* Scan the header for all destructor sections, in any
501 * segment, and invoke the constructors within those sections.
502 */
503 for (segment = firstsegfromheader(header);
504 segment != failure_segment && segment != NULL;
505 segment = nextsegfromheader(header, segment)) {
506 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
507 sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
508 } /* for (segment...) */
509 }
510
511 /* Now, regardless of success so far, do the post-init registration
512 * and cleanup. If we had to call the unloadCPP function, static
513 * destructors have removed classes from the stalled list so no
514 * metaclasses will actually be registered.
515 */
516 result = OSMetaClass::postModLoad(metaHandle);
517
518 /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad()
519 * fails (typically due to a duplicate class), tear down all the C++
520 * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff,
521 * but may be necessary for other C++ code. We ignore the return value
522 * because it's only a fail when there are existing instances of libkern
523 * classes, and there had better not be any created on the C++ init path.
524 */
525 if (load_success && result != KMOD_RETURN_SUCCESS) {
526 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd);
527 }
528
529 if (theKext && load_success && result == KMOD_RETURN_SUCCESS) {
530 theKext->setCPPInitialized(true);
531 }
532 finish:
533 return result;
534 }
535
536 /*********************************************************************
537 * Unload a kernel segment.
538 *********************************************************************/
539
540 void
OSRuntimeUnloadCPPForSegment(kernel_segment_command_t * segment)541 OSRuntimeUnloadCPPForSegment(
542 kernel_segment_command_t * segment)
543 {
544 OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment,
545 gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0);
546 }
547
548 #if PRAGMA_MARK
549 #pragma mark C++ Allocators & Deallocators
550 #endif /* PRAGMA_MARK */
551 /*********************************************************************
552 * C++ Allocators & Deallocators
553 *********************************************************************/
554 __typed_allocators_ignore_push
555
556 void *
operator new(size_t size)557 operator new(size_t size)
558 {
559 assert(size);
560 return kheap_alloc(KERN_OS_MALLOC, size,
561 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN));
562 }
563
564 void
operator delete(void * addr)565 operator delete(void * addr)
566 #if __cplusplus >= 201103L
567 noexcept
568 #endif
569 {
570 kheap_free_addr(KERN_OS_MALLOC, addr);
571 return;
572 }
573
574 void *
operator new[](unsigned long size)575 operator new[](unsigned long size)
576 {
577 return kheap_alloc(KERN_OS_MALLOC, size,
578 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN));
579 }
580
581 void
operator delete[](void * ptr)582 operator delete[](void * ptr)
583 #if __cplusplus >= 201103L
584 noexcept
585 #endif
586 {
587 if (ptr) {
588 #if KASAN
589 /*
590 * Unpoison the C++ array cookie inserted (but not removed) by the
591 * compiler on new[].
592 */
593 kasan_unpoison_cxx_array_cookie(ptr);
594 #endif
595 kheap_free_addr(KERN_OS_MALLOC, ptr);
596 }
597 return;
598 }
599
600 #if __cplusplus >= 201103L
601
602 void
operator delete(void * addr,size_t sz)603 operator delete(void * addr, size_t sz) noexcept
604 {
605 kheap_free(KERN_OS_MALLOC, addr, sz);
606 }
607
608 void
operator delete[](void * addr,size_t sz)609 operator delete[](void * addr, size_t sz) noexcept
610 {
611 if (addr) {
612 kheap_free(KERN_OS_MALLOC, addr, sz);
613 }
614 }
615
616 __typed_allocators_ignore_pop
617
618 #endif /* __cplusplus >= 201103L */
619
620 /* PR-6481964 - The compiler is going to check for size overflows in calls to
621 * new[], and if there is an overflow, it will call __throw_length_error.
622 * This is an unrecoverable error by the C++ standard, so we must panic here.
623 *
624 * We have to put the function inside the std namespace because of how the
625 * compiler expects the name to be mangled.
626 */
627 namespace std {
628 void __dead2
__throw_length_error(const char * msg __unused)629 __throw_length_error(const char *msg __unused)
630 {
631 panic("Size of array created by new[] has overflowed");
632 }
633 };
634