1 /*
2 * Copyright (c) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * The main orchestrator for kernel (and co-processor) coredumps. Here's a very simplistic view of
31 * the flow:
32 *
33 * At kernel initialization time (kdp_core_init):
34 * ----------------------------------------------
35 *
36 * - kdp_core_init() takes care of allocating all necessary data structures and initializes the
37 * coredump output stages
38 *
39 * At coredump time (do_kern_dump):
40 * --------------------------------
41 *
42 * - Depending on the coredump variant, we chain the necessary output stages together in chain_output_stages()
43 * - [Disk only] We initialize the corefile header
44 * - [Disk only] We stream the stackshot out through the output stages and update the corefile header
45 * - We perform the kernel coredump, streaming it out through the output stages
46 * - [Disk only] We update the corefile header
47 * - [Disk only] We perform the co-processor coredumps (driven by kern_do_coredump), streaming each out
48 * through the output stages and updating the corefile header.
49 * - [Disk only] We save the coredump log to the corefile
50 */
51
52 #include <mach/kern_return.h>
53 #include <mach/vm_types.h>
54 #include <kdp/core_exclude.h>
55 #include <kdp/kdp_core.h>
56 #include <kdp/core_notes.h>
57
58 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
59
60 #include <mach/mach_types.h>
61 #include <mach/vm_attributes.h>
62 #include <mach/vm_param.h>
63 #include <mach/vm_map.h>
64 #include <vm/vm_protos.h>
65 #include <vm/vm_kern_xnu.h>
66 #include <vm/vm_map.h>
67 #include <machine/cpu_capabilities.h>
68 #include <libsa/types.h>
69 #include <libkern/kernel_mach_header.h>
70 #include <kern/locks.h>
71 #include <kdp/kdp_internal.h>
72 #include <kdp/output_stages/output_stages.h>
73 #include <kdp/processor_core.h>
74 #include <IOKit/IOTypes.h>
75 #include <IOKit/IOBSD.h>
76 #include <sys/errno.h>
77 #include <sys/msgbuf.h>
78 #include <san/kasan.h>
79 #include <kern/debug.h>
80 #include <pexpert/pexpert.h>
81 #include <os/atomic_private.h>
82
83 #if CONFIG_SPTM
84 #include <sptm/debug_header.h>
85 #endif
86
87 #if defined(__x86_64__)
88 #include <i386/pmap_internal.h>
89 #include <kdp/ml/i386/kdp_x86_common.h>
90 #include <kern/debug.h>
91 #endif /* defined(__x86_64__) */
92
93 #if CONFIG_SPTM
94 #include <arm64/sptm/sptm.h>
95 #endif /* CONFIG_SPTM */
96
97 kern_return_t kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context);
98 kern_return_t kdp_core_polled_io_polled_file_unavailable(void);
99
100 typedef int (*pmap_traverse_callback)(vm_map_offset_t start,
101 vm_map_offset_t end,
102 void *context);
103
104 static kern_return_t kern_dump_init(void *refcon, void *context);
105 static int kern_dump_save_summary(void *refcon, core_save_summary_cb callback, void *context);
106 static int kern_dump_save_seg_descriptions(void *refcon, core_save_segment_descriptions_cb callback, void *context);
107 static int kern_dump_save_thread_state(void *refcon, void *buf, core_save_thread_state_cb callback, void *context);
108 static int kern_dump_save_sw_vers_detail(void *refcon, core_save_sw_vers_detail_cb callback, void *context);
109 static int kern_dump_save_segment_data(void *refcon, core_save_segment_data_cb callback, void *context);
110 static kern_return_t kern_dump_save_note_summary(void *refcon, core_save_note_summary_cb callback, void *context);
111 static kern_return_t kern_dump_save_note_descriptions(void *refcon, core_save_note_descriptions_cb callback, void *context);
112 static kern_return_t kern_dump_save_note_data(void *refcon, core_save_note_data_cb callback, void *context);
113
114 static int
115 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
116 vm_map_offset_t end,
117 void *context);
118 static int
119 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
120 vm_map_offset_t end,
121 void *context);
122
123 static int
124 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
125 vm_map_offset_t end,
126 void *context);
127
128 static struct kdp_output_stage disk_output_stage = {};
129 static struct kdp_output_stage lz4_output_stage = {};
130 static struct kdp_output_stage zlib_output_stage = {};
131 static struct kdp_output_stage buffer_output_stage = {};
132 static struct kdp_output_stage net_output_stage = {};
133 static struct kdp_output_stage progress_notify_output_stage = {};
134 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
135 static struct kdp_output_stage aea_output_stage = {};
136 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
137 #if defined(__arm64__)
138 static struct kdp_output_stage shmem_output_stage = {};
139 static struct kdp_output_stage memory_backing_aware_buffer_output_stage = {};
140 #endif /* defined(__arm64__) */
141
142 extern uint32_t kdp_crashdump_pkt_size;
143
144 static boolean_t kern_dump_successful = FALSE;
145
146 static const size_t kdp_core_header_size = sizeof(struct mach_core_fileheader_v2) + (KERN_COREDUMP_MAX_CORES * sizeof(struct mach_core_details_v2));
147 static struct mach_core_fileheader_v2 *kdp_core_header = NULL;
148
149 static lck_grp_t *kdp_core_initialization_lock_group = NULL;
150 static lck_mtx_t *kdp_core_disk_stage_lock = NULL;
151 static bool kdp_core_is_initializing_disk_stage = false;
152
153 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
154 static const size_t PUBLIC_KEY_RESERVED_LENGTH = roundup(4096, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
155 static void *kdp_core_public_key = NULL;
156 static lck_mtx_t *kdp_core_encryption_stage_lock = NULL;
157 static bool kdp_core_is_initializing_encryption_stage = false;
158 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
159
160 static lck_mtx_t *kdp_core_lz4_stage_lock = NULL;
161 static bool kdp_core_is_initializing_lz4_stage = false;
162
163 /*
164 * These variables will be modified by the BSD layer if the root device is
165 * a RAMDisk.
166 */
167 uint64_t kdp_core_ramdisk_addr = 0;
168 uint64_t kdp_core_ramdisk_size = 0;
169
170 #define COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY (1 << 0)
171 #define COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT (1 << 1)
172
173 boolean_t
kdp_has_polled_corefile(void)174 kdp_has_polled_corefile(void)
175 {
176 return NULL != gIOPolledCoreFileVars;
177 }
178
179 kern_return_t
kdp_polled_corefile_error(void)180 kdp_polled_corefile_error(void)
181 {
182 return gIOPolledCoreFileOpenRet;
183 }
184
185 IOPolledCoreFileMode_t
kdp_polled_corefile_mode(void)186 kdp_polled_corefile_mode(void)
187 {
188 return gIOPolledCoreFileMode;
189 }
190
191 struct kdp_core_excluded_region {
192 struct kdp_core_excluded_region *next;
193 vm_offset_t addr;
194 vm_size_t size;
195 };
196
197 static LCK_GRP_DECLARE(excluded_regions_grp, "kdp-exclude-regions");
198 static LCK_MTX_DECLARE(excluded_regions_mtx, &excluded_regions_grp);
199 static struct kdp_core_excluded_region *excluded_regions;
200
201 void
kdp_core_exclude_region(vm_offset_t addr,vm_size_t size)202 kdp_core_exclude_region(vm_offset_t addr, vm_size_t size)
203 {
204 struct kdp_core_excluded_region *region;
205
206 if (addr >= addr + size) {
207 panic("%s: cannot exclude region starting at %p with size %zu (zero or overflowing size)",
208 __func__, (void*)addr, (size_t)size);
209 }
210 if (addr != round_page(addr) || size != round_page(size)) {
211 panic("%s: cannot exclude region starting at %p with size %zu (not page aligned)",
212 __func__, (void*)addr, (size_t)size);
213 }
214
215 region = kalloc_type(typeof(*region), Z_WAITOK | Z_NOFAIL);
216 region->addr = addr;
217 region->size = size;
218
219 lck_mtx_lock(&excluded_regions_mtx);
220 region->next = excluded_regions;
221 excluded_regions = region;
222 lck_mtx_unlock(&excluded_regions_mtx);
223 }
224
225 void
kdp_core_unexclude_region(vm_offset_t addr,vm_size_t size)226 kdp_core_unexclude_region(vm_offset_t addr, vm_size_t size)
227 {
228 struct kdp_core_excluded_region *region;
229 struct kdp_core_excluded_region **fixup = &excluded_regions;
230
231 lck_mtx_lock(&excluded_regions_mtx);
232 for (region = excluded_regions; region; region = region->next) {
233 if (region->addr == addr && region->size == size) {
234 *fixup = region->next;
235 break;
236 }
237 fixup = ®ion->next;
238 }
239 if (!region) {
240 panic("%s: cannot unexclude region starting at %p with size %zu (not currently excluded)",
241 __func__, (void*)addr, (size_t)size);
242 }
243 lck_mtx_unlock(&excluded_regions_mtx);
244
245 // We had exclusive access to the list when we removed the region, and it is no longer
246 // reachable from the list, so it is safe to free.
247 kfree_type(typeof(*region), region);
248 }
249
250 static bool
kernel_vaddr_in_excluded_region(vm_offset_t addr,uint64_t * vincr)251 kernel_vaddr_in_excluded_region(vm_offset_t addr, uint64_t *vincr)
252 {
253 struct kdp_core_excluded_region *region;
254
255 // We check this earlier before attempting to dump the kernel, but verify here.
256 assert(!kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx));
257
258 for (region = excluded_regions; region; region = region->next) {
259 if (region->addr <= addr && addr < (region->addr + region->size)) {
260 *vincr = region->size;
261 return true;
262 }
263 }
264
265 return false;
266 }
267
268 kern_return_t
kdp_core_output(void * kdp_core_out_state,uint64_t length,void * data)269 kdp_core_output(void *kdp_core_out_state, uint64_t length, void * data)
270 {
271 kern_return_t err = KERN_SUCCESS;
272 uint64_t percent;
273 struct kdp_core_out_state *vars = (struct kdp_core_out_state *)kdp_core_out_state;
274 struct kdp_output_stage *first_stage = STAILQ_FIRST(&vars->kcos_out_stage);
275
276 if (vars->kcos_error == KERN_SUCCESS) {
277 #if DEVELOPMENT || DEBUG
278 // panic testing: force the write to fail after X number of writes
279 if ((panic_test_case & PANIC_TEST_CASE_COREFILE_IO_ERR) && (--panic_test_action_count == 0)) {
280 panic_test_case &= ~PANIC_TEST_CASE_COREFILE_IO_ERR;
281 length = -1;
282 }
283 #endif
284
285 if ((err = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, length, data)) != KERN_SUCCESS) {
286 kern_coredump_log(NULL, "(kdp_core_output) outproc(KDP_DATA, NULL, 0x%llx, %p) returned 0x%x\n",
287 length, data, err);
288 vars->kcos_error = err;
289 }
290 if (!data && !length) {
291 kern_coredump_log(NULL, "100..");
292 } else {
293 vars->kcos_bytes_written += length;
294 percent = (vars->kcos_bytes_written * 100) / vars->kcos_totalbytes;
295 if ((percent - vars->kcos_lastpercent) >= 10) {
296 vars->kcos_lastpercent = percent;
297 kern_coredump_log(NULL, "%lld..\n", percent);
298 }
299 }
300 }
301 return err;
302 }
303
304 #if defined(__arm64__)
305 extern pmap_paddr_t avail_start, avail_end;
306 extern struct vm_object pmap_object_store;
307 #endif
308 extern vm_offset_t c_buffers;
309 extern vm_size_t c_buffers_size;
310
311 static bool
kernel_vaddr_in_coredump_stage(const struct kdp_output_stage * stage,uint64_t vaddr,uint64_t * vincr)312 kernel_vaddr_in_coredump_stage(const struct kdp_output_stage *stage, uint64_t vaddr, uint64_t *vincr)
313 {
314 uint64_t start_addr = (uint64_t)stage->kos_data;
315 uint64_t end_addr = start_addr + stage->kos_data_size;
316
317 if (!stage->kos_data) {
318 return false;
319 }
320
321 if (vaddr >= start_addr && vaddr < end_addr) {
322 *vincr = stage->kos_data_size - (vaddr - start_addr);
323 return true;
324 }
325
326 return false;
327 }
328
329 static bool
kernel_vaddr_in_coredump_stages(uint64_t vaddr,uint64_t * vincr)330 kernel_vaddr_in_coredump_stages(uint64_t vaddr, uint64_t *vincr)
331 {
332 if (kernel_vaddr_in_coredump_stage(&disk_output_stage, vaddr, vincr)) {
333 return true;
334 }
335
336 if (kernel_vaddr_in_coredump_stage(&lz4_output_stage, vaddr, vincr)) {
337 return true;
338 }
339
340 if (kernel_vaddr_in_coredump_stage(&zlib_output_stage, vaddr, vincr)) {
341 return true;
342 }
343
344 if (kernel_vaddr_in_coredump_stage(&buffer_output_stage, vaddr, vincr)) {
345 return true;
346 }
347
348 if (kernel_vaddr_in_coredump_stage(&net_output_stage, vaddr, vincr)) {
349 return true;
350 }
351
352 if (kernel_vaddr_in_coredump_stage(&progress_notify_output_stage, vaddr, vincr)) {
353 return true;
354 }
355
356 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
357 if (kernel_vaddr_in_coredump_stage(&aea_output_stage, vaddr, vincr)) {
358 return true;
359 }
360 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
361
362 #if defined(__arm64__)
363 if (kernel_vaddr_in_coredump_stage(&shmem_output_stage, vaddr, vincr)) {
364 return true;
365 }
366 #endif /* defined(__arm64__) */
367
368 #if defined(__arm64__)
369 if (kernel_vaddr_in_coredump_stage(&memory_backing_aware_buffer_output_stage, vaddr, vincr)) {
370 return true;
371 }
372 #endif /* defined(__arm64__) */
373
374 return false;
375 }
376
377 #if CONFIG_SPTM && HAS_MTE && (DEVELOPMENT || DEBUG)
378 static bool
is_tag_space_page_used_for_mte(ppnum_t tag_space_page)379 is_tag_space_page_used_for_mte(ppnum_t tag_space_page)
380 {
381 bool is_tag_space_page_used = false;
382 ppnum_t first_covered_ppn = map_tag_ppnum_to_first_covered_ppnum(tag_space_page);
383
384 for (int i = 0; i < MTE_PAGES_PER_TAG_PAGE; i++) {
385 vm_page_t covered_page = vm_page_find_canonical(first_covered_ppn + i);
386 if (covered_page && covered_page->vmp_using_mte) {
387 is_tag_space_page_used = true;
388 break;
389 }
390 }
391
392 return is_tag_space_page_used;
393 }
394
395 static void
dump_mte_tag_space_into_coredump(pmap_traverse_callback cb,void * ctx)396 dump_mte_tag_space_into_coredump(pmap_traverse_callback cb, void *ctx)
397 {
398 uintptr_t tag_space_page_vaddr = 0;
399 uintptr_t virt_range_start = 0;
400 uintptr_t virt_range_end = 0;
401
402 // For each tag storage page, check if exists at least one covered page that's actually tagged.
403 // If found, it'd mean the tag storage page is used and should be included in the core dump.
404 // By iterating over all tag space page numbers continously, we can guarentee that we'll store
405 // the pages in the coredump as grouped-together as possible.
406 for (int i = 0; i < mte_tag_storage_count; i++) {
407 ppnum_t tag_space_pnum = mte_tag_storage_start_pnum + i;
408 tag_space_page_vaddr = phystokv(ptoa(tag_space_pnum));
409
410 if (is_tag_space_page_used_for_mte(tag_space_pnum)) {
411 if (0 == virt_range_start) {
412 virt_range_start = tag_space_page_vaddr;
413 }
414
415 // Encountered a new range of pages, continue until range of range is reached.
416 virt_range_end = tag_space_page_vaddr + PAGE_SIZE;
417 continue;
418 }
419
420 // If we got here, current page is not used for mte, so we got to the end of a tag-space-pages range.
421 if (virt_range_start && virt_range_end) {
422 cb(virt_range_start, virt_range_end, ctx);
423 virt_range_start = 0;
424 virt_range_end = 0;
425 }
426 }
427
428 // If the last pages-range continues until the end of the tag space, we'd exit the for-loop without running into a page that cuts the range.
429 // This ensure that last range is also saved to the coredump.
430 if (virt_range_start && virt_range_end) {
431 cb(virt_range_start, virt_range_end, ctx);
432 }
433 }
434 #endif /* CONFIG_SPTM && HAS_MTE && (DEVELOPMENT || DEBUG) */
435
436 ppnum_t
kernel_pmap_present_mapping(uint64_t vaddr,uint64_t * pvincr,uintptr_t * pvphysaddr)437 kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphysaddr)
438 {
439 ppnum_t ppn = 0;
440 uint64_t vincr = PAGE_SIZE_64;
441
442 assert(!(vaddr & PAGE_MASK_64));
443
444 /* VA ranges to exclude */
445 if (vaddr == c_buffers) {
446 /* compressor data */
447 ppn = 0;
448 vincr = c_buffers_size;
449 } else if (kernel_vaddr_in_coredump_stages(vaddr, &vincr)) {
450 /* coredump output stage working memory */
451 ppn = 0;
452 } else if ((kdp_core_ramdisk_addr != 0) && (vaddr == kdp_core_ramdisk_addr)) {
453 ppn = 0;
454 vincr = kdp_core_ramdisk_size;
455 } else
456 #if defined(__arm64__)
457 if (vaddr == phystokv(avail_start)) {
458 /* physical memory map */
459 ppn = 0;
460 vincr = (avail_end - avail_start);
461 } else
462 #endif /* defined(__arm64__) */
463 {
464 ppn = (pvphysaddr != NULL ?
465 pmap_find_phys(kernel_pmap, vaddr) :
466 pmap_find_phys_nofault(kernel_pmap, vaddr));
467 }
468
469 *pvincr = round_page_64(vincr);
470
471 if (ppn && pvphysaddr) {
472 uint64_t phys = ptoa_64(ppn);
473 if (physmap_enclosed(phys)) {
474 *pvphysaddr = phystokv(phys);
475 } else {
476 ppn = 0;
477 }
478 }
479
480 return ppn;
481 }
482
483 static int
pmap_traverse_present_mappings(pmap_t __unused pmap,vm_map_offset_t start,vm_map_offset_t end,pmap_traverse_callback callback,void * context)484 pmap_traverse_present_mappings(pmap_t __unused pmap,
485 vm_map_offset_t start,
486 vm_map_offset_t end,
487 pmap_traverse_callback callback,
488 void *context)
489 {
490 IOReturn ret;
491 vm_map_offset_t vcurstart, vcur;
492 uint64_t vincr = 0;
493 vm_map_offset_t debug_start = trunc_page((vm_map_offset_t) debug_buf_base);
494 vm_map_offset_t debug_end = round_page((vm_map_offset_t) (debug_buf_base + debug_buf_size));
495 #if defined(XNU_TARGET_OS_BRIDGE)
496 vm_map_offset_t macos_panic_start = trunc_page((vm_map_offset_t) macos_panic_base);
497 vm_map_offset_t macos_panic_end = round_page((vm_map_offset_t) (macos_panic_base + macos_panic_size));
498 #endif
499
500 boolean_t lastvavalid;
501 #if defined(__arm64__)
502 vm_page_t m = VM_PAGE_NULL;
503 #endif
504
505 #if defined(__x86_64__)
506 assert(!is_ept_pmap(pmap));
507 #endif
508
509 /* Assumes pmap is locked, or being called from the kernel debugger */
510 if (start > end) {
511 return KERN_INVALID_ARGUMENT;
512 }
513
514 ret = KERN_SUCCESS;
515 lastvavalid = FALSE;
516 for (vcur = vcurstart = start; (ret == KERN_SUCCESS) && (vcur < end);) {
517 ppnum_t ppn = 0;
518
519 #if defined(__arm64__)
520 /* We're at the start of the physmap, so pull out the pagetable pages that
521 * are accessed through that region.*/
522 if (vcur == phystokv(avail_start) && vm_object_lock_try_shared(&pmap_object_store)) {
523 m = (vm_page_t)vm_page_queue_first(&pmap_object_store.memq);
524 }
525
526 if (m != VM_PAGE_NULL) {
527 vm_map_offset_t vprev = vcur;
528 ppn = (ppnum_t)atop(avail_end);
529 while (!vm_page_queue_end(&pmap_object_store.memq, (vm_page_queue_entry_t)m)) {
530 /* Ignore pages that come from the static region and have already been dumped.*/
531 if (VM_PAGE_GET_PHYS_PAGE(m) >= atop(avail_start)) {
532 ppn = VM_PAGE_GET_PHYS_PAGE(m);
533 break;
534 }
535 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
536 }
537 vincr = PAGE_SIZE_64;
538 if (ppn == atop(avail_end)) {
539 vm_object_unlock(&pmap_object_store);
540 m = VM_PAGE_NULL;
541 // avail_end is not a valid physical address,
542 // so phystokv(avail_end) may not produce the expected result.
543 vcur = phystokv(avail_start) + (avail_end - avail_start);
544 } else {
545 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
546 vcur = phystokv(ptoa(ppn));
547 }
548 if (vcur != vprev) {
549 ret = callback(vcurstart, vprev, context);
550 lastvavalid = FALSE;
551 }
552 }
553 if (m == VM_PAGE_NULL) {
554 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
555 }
556 #else /* defined(__arm64__) */
557 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
558 #endif
559 if (ppn != 0 && kernel_vaddr_in_excluded_region(vcur, &vincr)) {
560 /* excluded region */
561 ppn = 0;
562 }
563 if (ppn != 0) {
564 if (((vcur < debug_start) || (vcur >= debug_end))
565 && !(
566 pmap_valid_page(ppn)
567 || bootloader_valid_page(ppn)
568 )
569 #if defined(XNU_TARGET_OS_BRIDGE)
570 // include the macOS panic region if it's mapped
571 && ((vcur < macos_panic_start) || (vcur >= macos_panic_end))
572 #endif /* defined(XNU_TARGET_OS_BRIDGE) */
573 ) {
574 /* not something we want */
575 ppn = 0;
576 }
577 /* include the phys carveout only if explictly marked */
578 if (debug_is_in_phys_carveout(vcur) &&
579 !debug_can_coredump_phys_carveout()) {
580 ppn = 0;
581 }
582 }
583
584 if (ppn != 0) {
585 if (!lastvavalid) {
586 /* Start of a new virtual region */
587 vcurstart = vcur;
588 lastvavalid = TRUE;
589 }
590 } else {
591 if (lastvavalid) {
592 /* end of a virtual region */
593 ret = callback(vcurstart, vcur, context);
594 lastvavalid = FALSE;
595 }
596
597 #if defined(__x86_64__)
598 /* Try to skip by 2MB if possible */
599 if ((vcur & PDMASK) == 0) {
600 pd_entry_t *pde;
601 pde = pmap_pde(pmap, vcur);
602 if (0 == pde || ((*pde & INTEL_PTE_VALID) == 0)) {
603 /* Make sure we wouldn't overflow */
604 if (vcur < (end - NBPD)) {
605 vincr = NBPD;
606 }
607 }
608 }
609 #endif /* defined(__x86_64__) */
610 }
611 vcur += vincr;
612 }
613
614 if ((ret == KERN_SUCCESS) && lastvavalid) {
615 /* send previous run */
616 ret = callback(vcurstart, vcur, context);
617 }
618
619 #if KASAN
620 if (ret == KERN_SUCCESS) {
621 ret = kasan_traverse_mappings(callback, context);
622 }
623 #endif
624
625 #if CONFIG_SPTM && HAS_MTE && (DEVELOPMENT || DEBUG)
626 if (ret == KERN_SUCCESS) {
627 dump_mte_tag_space_into_coredump(callback, context);
628 }
629 #endif /* CONFIG_SPTM && HAS_MTE && (DEVELOPMENT || DEBUG) */
630
631 return ret;
632 }
633
634 struct kern_dump_preflight_context {
635 uint32_t region_count;
636 uint64_t dumpable_bytes;
637 };
638
639 int
kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)640 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
641 vm_map_offset_t end,
642 void *context)
643 {
644 struct kern_dump_preflight_context *kdc = (struct kern_dump_preflight_context *)context;
645 IOReturn ret = KERN_SUCCESS;
646
647 kdc->region_count++;
648 kdc->dumpable_bytes += (end - start);
649
650 return ret;
651 }
652
653
654 struct kern_dump_send_seg_desc_context {
655 core_save_segment_descriptions_cb callback;
656 void *context;
657 };
658
659 int
kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)660 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
661 vm_map_offset_t end,
662 void *context)
663 {
664 struct kern_dump_send_seg_desc_context *kds_context = (struct kern_dump_send_seg_desc_context *)context;
665 uint64_t seg_start = (uint64_t) start;
666 uint64_t seg_end = (uint64_t) end;
667
668 return kds_context->callback(seg_start, seg_end, kds_context->context);
669 }
670
671 struct kern_dump_send_segdata_context {
672 core_save_segment_data_cb callback;
673 void *context;
674 };
675
676 int
kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)677 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
678 vm_map_offset_t end,
679 void *context)
680 {
681 struct kern_dump_send_segdata_context *kds_context = (struct kern_dump_send_segdata_context *)context;
682
683 return kds_context->callback((void *)start, (uint64_t)(end - start), kds_context->context);
684 }
685
686 static kern_return_t
kern_dump_init(__unused void * refcon,void * context)687 kern_dump_init(__unused void *refcon, void *context)
688 {
689 /* TODO: consider doing mmu flush from an init function */
690
691 // If excluded regions list is locked, it is unsafe to dump the kernel.
692 if (kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx)) {
693 kern_coredump_log(context, "%s: skipping kernel because excluded regions list is locked\n",
694 __func__);
695 #if defined(__arm64__)
696 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
697 #else
698 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
699 #endif
700 paniclog_flush();
701 return KERN_NODE_DOWN;
702 }
703
704 return KERN_SUCCESS;
705 }
706
707 static int
kern_dump_save_summary(__unused void * refcon,core_save_summary_cb callback,void * context)708 kern_dump_save_summary(__unused void *refcon, core_save_summary_cb callback, void *context)
709 {
710 struct kern_dump_preflight_context kdc_preflight = { };
711 uint64_t thread_state_size = 0, thread_count = 0;
712 vm_map_offset_t vstart = kdp_core_start_addr();
713 kern_return_t ret;
714
715 ret = pmap_traverse_present_mappings(kernel_pmap,
716 vstart,
717 VM_MAX_KERNEL_ADDRESS,
718 kern_dump_pmap_traverse_preflight_callback,
719 &kdc_preflight);
720 if (ret != KERN_SUCCESS) {
721 kern_coredump_log(context, "save_summary: pmap traversal failed: %d\n", ret);
722 return ret;
723 }
724
725 kern_collectth_state_size(&thread_count, &thread_state_size);
726
727 ret = callback(kdc_preflight.region_count, kdc_preflight.dumpable_bytes,
728 thread_count, thread_state_size, 0, context);
729 return ret;
730 }
731
732 static int
kern_dump_save_seg_descriptions(__unused void * refcon,core_save_segment_descriptions_cb callback,void * context)733 kern_dump_save_seg_descriptions(__unused void *refcon, core_save_segment_descriptions_cb callback, void *context)
734 {
735 vm_map_offset_t vstart = kdp_core_start_addr();
736 kern_return_t ret;
737 struct kern_dump_send_seg_desc_context kds_context;
738
739 kds_context.callback = callback;
740 kds_context.context = context;
741
742 ret = pmap_traverse_present_mappings(kernel_pmap,
743 vstart,
744 VM_MAX_KERNEL_ADDRESS,
745 kern_dump_pmap_traverse_send_segdesc_callback,
746 &kds_context);
747 if (ret != KERN_SUCCESS) {
748 kern_coredump_log(context, "save_seg_desc: pmap traversal failed: %d\n", ret);
749 return ret;
750 }
751
752 return KERN_SUCCESS;
753 }
754
755 static int
kern_dump_save_thread_state(__unused void * refcon,void * buf,core_save_thread_state_cb callback,void * context)756 kern_dump_save_thread_state(__unused void *refcon, void *buf, core_save_thread_state_cb callback, void *context)
757 {
758 kern_return_t ret;
759 uint64_t thread_state_size = 0, thread_count = 0;
760
761 kern_collectth_state_size(&thread_count, &thread_state_size);
762
763 if (thread_state_size > 0) {
764 void * iter = NULL;
765 do {
766 kern_collectth_state(current_thread(), buf, thread_state_size, &iter);
767
768 ret = callback(buf, context);
769 if (ret != KERN_SUCCESS) {
770 return ret;
771 }
772 } while (iter);
773 }
774
775 return KERN_SUCCESS;
776 }
777
778
779 static int
kern_dump_save_sw_vers_detail(__unused void * refcon,core_save_sw_vers_detail_cb callback,void * context)780 kern_dump_save_sw_vers_detail(__unused void *refcon, core_save_sw_vers_detail_cb callback, void *context)
781 {
782 return callback(vm_kernel_stext, kernel_uuid, 0, context);
783 }
784
785 static int
kern_dump_save_segment_data(__unused void * refcon,core_save_segment_data_cb callback,void * context)786 kern_dump_save_segment_data(__unused void *refcon, core_save_segment_data_cb callback, void *context)
787 {
788 vm_map_offset_t vstart = kdp_core_start_addr();
789 kern_return_t ret;
790 struct kern_dump_send_segdata_context kds_context;
791
792 kds_context.callback = callback;
793 kds_context.context = context;
794
795 ret = pmap_traverse_present_mappings(kernel_pmap,
796 vstart,
797 VM_MAX_KERNEL_ADDRESS, kern_dump_pmap_traverse_send_segdata_callback, &kds_context);
798 if (ret != KERN_SUCCESS) {
799 kern_coredump_log(context, "save_seg_data: pmap traversal failed: %d\n", ret);
800 return ret;
801 }
802
803 return KERN_SUCCESS;
804 }
805
806 kern_return_t
kdp_reset_output_vars(void * kdp_core_out_state,uint64_t totalbytes,bool encrypt_core,bool * out_should_skip_coredump,const char * corename,kern_coredump_type_t coretype)807 kdp_reset_output_vars(void *kdp_core_out_state, uint64_t totalbytes, bool encrypt_core, bool *out_should_skip_coredump,
808 const char *corename, kern_coredump_type_t coretype)
809 {
810 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
811 struct kdp_output_stage *current_stage = NULL;
812
813 /* Re-initialize kdp_outstate */
814 outstate->kcos_totalbytes = totalbytes;
815 outstate->kcos_bytes_written = 0;
816 outstate->kcos_lastpercent = 0;
817 outstate->kcos_error = KERN_SUCCESS;
818
819 /* Reset the output stages */
820 STAILQ_FOREACH(current_stage, &outstate->kcos_out_stage, kos_next) {
821 kern_return_t res = current_stage->kos_funcs.kosf_reset(current_stage, corename, coretype);
822
823 /* Skip coredump if requested by an output stage. */
824 if (res == KERN_NODE_DOWN) {
825 *out_should_skip_coredump = true;
826 return KERN_SUCCESS;
827 }
828
829 if (res != KERN_SUCCESS) {
830 return res;
831 }
832 }
833
834 *out_should_skip_coredump = false;
835 if (encrypt_core) {
836 if (outstate->kcos_enforce_encryption && !outstate->kcos_encryption_stage) {
837 *out_should_skip_coredump = true;
838 #if defined(__arm64__)
839 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
840 #else
841 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
842 #endif
843 kern_coredump_log(NULL, "(kdp_reset_output_vars) Encryption requested, is unavailable, and enforcement is active. Skipping current core.\n");
844 }
845 } else if (outstate->kcos_encryption_stage) {
846 outstate->kcos_encryption_stage->kos_bypass = true;
847 }
848
849 return KERN_SUCCESS;
850 }
851
852 static kern_return_t
kern_dump_update_header(struct kdp_core_out_state * outstate)853 kern_dump_update_header(struct kdp_core_out_state *outstate)
854 {
855 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
856 uint64_t foffset;
857 kern_return_t ret;
858
859 /* Write the file header -- first seek to the beginning of the file */
860 foffset = 0;
861 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
862 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
863 sizeof(foffset), &foffset, foffset, ret);
864 return ret;
865 }
866
867 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header_size, kdp_core_header)) != KERN_SUCCESS) {
868 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
869 kdp_core_header_size, kdp_core_header, ret);
870 return ret;
871 }
872
873 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
874 kern_coredump_log(NULL, "(kern_dump_update_header) outproc data flush returned 0x%x\n", ret);
875 return ret;
876 }
877
878 #if defined(__arm64__)
879 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
880 kern_coredump_log(NULL, "(kern_dump_update_header) outproc explicit flush returned 0x%x\n", ret);
881 return ret;
882 }
883 #endif /* defined(__arm64__) */
884
885 return ret;
886 }
887
888 kern_return_t
kern_dump_record_file(void * kdp_core_out_state,const char * filename,uint64_t file_offset,uint64_t * out_file_length,uint64_t details_flags)889 kern_dump_record_file(void *kdp_core_out_state, const char *filename, uint64_t file_offset, uint64_t *out_file_length, uint64_t details_flags)
890 {
891 kern_return_t ret = KERN_SUCCESS;
892 uint64_t bytes_written = 0;
893 struct mach_core_details_v2 *core_details = NULL;
894 struct kdp_output_stage *last_stage;
895 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
896
897 assert(kdp_core_header->num_files < KERN_COREDUMP_MAX_CORES);
898 assert(out_file_length != NULL);
899 *out_file_length = 0;
900
901 last_stage = STAILQ_LAST(&outstate->kcos_out_stage, kdp_output_stage, kos_next);
902 bytes_written = last_stage->kos_bytes_written;
903
904 core_details = &(kdp_core_header->files[kdp_core_header->num_files]);
905 core_details->flags = details_flags;
906 core_details->offset = file_offset;
907 core_details->length = bytes_written;
908 strncpy((char *)&core_details->core_name, filename,
909 MACH_CORE_FILEHEADER_NAMELEN);
910 core_details->core_name[MACH_CORE_FILEHEADER_NAMELEN - 1] = '\0';
911
912 kdp_core_header->num_files++;
913
914 ret = kern_dump_update_header(outstate);
915 if (ret == KERN_SUCCESS) {
916 *out_file_length = bytes_written;
917 }
918
919 return ret;
920 }
921
922 kern_return_t
kern_dump_seek_to_next_file(void * kdp_core_out_state,uint64_t next_file_offset)923 kern_dump_seek_to_next_file(void *kdp_core_out_state, uint64_t next_file_offset)
924 {
925 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
926 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
927 kern_return_t ret;
928
929 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(next_file_offset), &next_file_offset)) != KERN_SUCCESS) {
930 kern_coredump_log(NULL, "(kern_dump_seek_to_next_file) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
931 sizeof(next_file_offset), &next_file_offset, next_file_offset, ret);
932 }
933
934 return ret;
935 }
936
937 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
938
939 static kern_return_t
kern_dump_write_public_key(struct kdp_core_out_state * outstate)940 kern_dump_write_public_key(struct kdp_core_out_state *outstate)
941 {
942 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
943 uint64_t foffset;
944 uint64_t remainder = PUBLIC_KEY_RESERVED_LENGTH - kdp_core_header->pub_key_length;
945 kern_return_t ret;
946
947 if (kdp_core_header->pub_key_offset == 0 || kdp_core_header->pub_key_length == 0) {
948 // Nothing to do
949 return KERN_SUCCESS;
950 }
951
952 /* Write the public key -- first seek to the appropriate offset */
953 foffset = kdp_core_header->pub_key_offset;
954 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
955 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
956 sizeof(foffset), &foffset, foffset, ret);
957 return ret;
958 }
959
960 // Write the public key
961 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
962 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
963 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
964 return ret;
965 }
966
967 // Fill out the remainder of the block with zeroes
968 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
969 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
970 remainder, ret);
971 return ret;
972 }
973
974 // Do it once more to write the "next" public key
975 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
976 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
977 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
978 return ret;
979 }
980
981 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
982 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
983 remainder, ret);
984 return ret;
985 }
986
987 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
988 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc data flush returned 0x%x\n", ret);
989 return ret;
990 }
991
992 #if defined(__arm64__)
993 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
994 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc explicit flush returned 0x%x\n", ret);
995 return ret;
996 }
997 #endif /* defined(__arm64__) */
998
999 return ret;
1000 }
1001
1002 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1003
1004 static kern_return_t
chain_output_stages(enum kern_dump_type kd_variant,struct kdp_core_out_state * outstate,uint64_t * details_flags)1005 chain_output_stages(enum kern_dump_type kd_variant, struct kdp_core_out_state *outstate, uint64_t *details_flags)
1006 {
1007 struct kdp_output_stage *current = NULL;
1008
1009 assert(details_flags);
1010 *details_flags = 0;
1011
1012 switch (kd_variant) {
1013 case KERN_DUMP_STACKSHOT_DISK:
1014 OS_FALLTHROUGH;
1015 case KERN_DUMP_DISK:
1016 #if defined(__arm64__)
1017 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &memory_backing_aware_buffer_output_stage, kos_next);
1018 #endif
1019 if (!kdp_corezip_disabled) {
1020 if (kdp_core_is_initializing_lz4_stage) {
1021 kern_coredump_log(NULL, "We were in the middle of initializing LZ4 stage. Cannot write a coredump to disk\n");
1022 return KERN_FAILURE;
1023 } else if (!lz4_output_stage.kos_initialized) {
1024 kern_coredump_log(NULL, "LZ4 stage is not yet initialized. Cannot write a coredump to disk\n");
1025 return KERN_FAILURE;
1026 }
1027 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &lz4_output_stage, kos_next);
1028 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_LZ4;
1029 }
1030 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
1031 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1032 if (kdp_core_is_initializing_encryption_stage) {
1033 kern_coredump_log(NULL, "We were in the middle of initializing encryption. Marking it as unavailable\n");
1034 } else if (aea_output_stage.kos_initialized) {
1035 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &aea_output_stage, kos_next);
1036 outstate->kcos_encryption_stage = &aea_output_stage;
1037 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_ENCRYPTED_AEA;
1038 }
1039 outstate->kcos_enforce_encryption = kern_dump_should_enforce_encryption();
1040 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1041 if (kdp_core_is_initializing_disk_stage) {
1042 kern_coredump_log(NULL, "We were in the middle of initializing the disk stage. Cannot write a coredump to disk\n");
1043 return KERN_FAILURE;
1044 } else if (disk_output_stage.kos_initialized == false) {
1045 kern_coredump_log(NULL, "Corefile is not yet initialized. Cannot write a coredump to disk\n");
1046 return KERN_FAILURE;
1047 }
1048 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &disk_output_stage, kos_next);
1049 break;
1050 case KERN_DUMP_NET:
1051 if (!kdp_corezip_disabled) {
1052 if (!zlib_output_stage.kos_initialized) {
1053 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to the network\n");
1054 return KERN_FAILURE;
1055 }
1056 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
1057 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
1058 }
1059 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
1060 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &buffer_output_stage, kos_next);
1061 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &net_output_stage, kos_next);
1062 break;
1063 #if defined(__arm64__)
1064 case KERN_DUMP_HW_SHMEM_DBG:
1065 if (!kdp_corezip_disabled) {
1066 if (!zlib_output_stage.kos_initialized) {
1067 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to shared memory\n");
1068 return KERN_FAILURE;
1069 }
1070 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
1071 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
1072 }
1073 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &shmem_output_stage, kos_next);
1074 break;
1075 #endif /* defined(__arm64__) */
1076 }
1077
1078 STAILQ_FOREACH(current, &outstate->kcos_out_stage, kos_next) {
1079 current->kos_outstate = outstate;
1080 }
1081
1082 return KERN_SUCCESS;
1083 }
1084
1085 #if defined(__arm64__)
1086
1087 static const char *panic_buf_filename = "panic_region";
1088
1089 static kern_return_t
dump_panic_buffer(struct kdp_core_out_state * outstate,char * panic_buf,size_t panic_len,uint64_t * foffset,uint64_t details_flags)1090 dump_panic_buffer(struct kdp_core_out_state *outstate, char *panic_buf, size_t panic_len,
1091 uint64_t *foffset, uint64_t details_flags)
1092 {
1093 kern_return_t ret = KERN_SUCCESS;
1094 bool should_skip = false;
1095
1096 kern_coredump_log(NULL, "\nBeginning dump of panic region of size 0x%zx\n", panic_len);
1097
1098 ret = kdp_reset_output_vars(outstate, panic_len, true, &should_skip,
1099 panic_buf_filename, RAW_COREDUMP);
1100 if (KERN_SUCCESS != ret) {
1101 return ret;
1102 }
1103
1104 if (should_skip) {
1105 kern_coredump_log(NULL, "Skipping panic region dump\n");
1106 return ret;
1107 }
1108
1109 uint64_t compressed_panic_region_len = 0;
1110 ret = kdp_core_output(outstate, panic_len, panic_buf);
1111 if (KERN_SUCCESS != ret) {
1112 kern_coredump_log(NULL, "Failed to write panic region to file, kdp_coreoutput(outstate, %zu, %p) returned 0x%x\n",
1113 panic_len, panic_buf, ret);
1114 return ret;
1115 }
1116
1117 ret = kdp_core_output(outstate, 0, NULL);
1118 if (KERN_SUCCESS != ret) {
1119 kern_coredump_log(NULL, "Failed to flush panic region data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", outstate, ret);
1120 return ret;
1121 }
1122
1123 ret = kern_dump_record_file(outstate, panic_buf_filename, *foffset, &compressed_panic_region_len,
1124 details_flags);
1125 if (KERN_SUCCESS != ret) {
1126 kern_coredump_log(NULL, "Failed to record panic region in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1127 return ret;
1128 }
1129
1130 kern_coredump_log(NULL, "Recorded panic region in corefile at offset 0x%llx, compressed to %llu bytes\n", *foffset, compressed_panic_region_len);
1131 *foffset = roundup((*foffset + compressed_panic_region_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1132
1133 ret = kern_dump_seek_to_next_file(outstate, *foffset);
1134 if (KERN_SUCCESS != ret) {
1135 kern_coredump_log(NULL, "Failed to seek to panic region file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", *foffset, ret);
1136 return ret;
1137 }
1138
1139 return ret;
1140 }
1141 #endif /* defined(__arm64__) */
1142
1143 static int
do_kern_dump(enum kern_dump_type kd_variant)1144 do_kern_dump(enum kern_dump_type kd_variant)
1145 {
1146 struct kdp_core_out_state outstate = { };
1147 struct kdp_output_stage *first_stage = NULL;
1148 char *coredump_log_start = NULL, *buf = NULL;
1149 size_t reserved_debug_logsize = 0, prior_debug_logsize = 0;
1150 uint64_t foffset = 0;
1151 kern_return_t ret = KERN_SUCCESS;
1152 boolean_t output_opened = FALSE, dump_succeeded = TRUE;
1153 uint64_t details_flags = 0;
1154
1155 /* Initialize output context */
1156
1157 bzero(&outstate, sizeof(outstate));
1158 STAILQ_INIT(&outstate.kcos_out_stage);
1159 ret = chain_output_stages(kd_variant, &outstate, &details_flags);
1160 if (KERN_SUCCESS != ret) {
1161 dump_succeeded = FALSE;
1162 goto exit;
1163 }
1164 first_stage = STAILQ_FIRST(&outstate.kcos_out_stage);
1165
1166 /*
1167 * Record the initial panic log buffer length so we can dump the coredump log
1168 * and panic log to disk
1169 */
1170 coredump_log_start = debug_buf_ptr;
1171 #if defined(__arm64__)
1172 assert(panic_info->eph_other_log_offset != 0);
1173 assert(panic_info->eph_panic_log_len != 0);
1174 /* Include any data from before the panic log as well */
1175 prior_debug_logsize = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1176 panic_info->eph_panic_log_len + panic_info->eph_other_log_len;
1177 #else /* defined(__arm64__) */
1178 if (panic_info->mph_panic_log_offset != 0) {
1179 prior_debug_logsize = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1180 panic_info->mph_panic_log_len + panic_info->mph_other_log_len;
1181 }
1182 #endif /* defined(__arm64__) */
1183
1184 assert(prior_debug_logsize <= debug_buf_size);
1185
1186 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1187 /* Open the file for output */
1188 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_WRQ, NULL, 0, NULL)) != KERN_SUCCESS) {
1189 kern_coredump_log(NULL, "outproc(KDP_WRQ, NULL, 0, NULL) returned 0x%x\n", ret);
1190 dump_succeeded = FALSE;
1191 goto exit;
1192 }
1193 }
1194 output_opened = true;
1195
1196 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1197 const size_t aligned_corefile_header_size = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1198 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1199 const size_t aligned_public_key_size = PUBLIC_KEY_RESERVED_LENGTH * 2;
1200 #else
1201 const size_t aligned_public_key_size = 0;
1202 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1203
1204 reserved_debug_logsize = prior_debug_logsize + KERN_COREDUMP_MAXDEBUGLOGSIZE;
1205
1206 /* Space for file header, public key, panic log, core log */
1207 foffset = roundup(aligned_corefile_header_size + aligned_public_key_size + reserved_debug_logsize, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1208 kdp_core_header->log_offset = aligned_corefile_header_size + aligned_public_key_size;
1209
1210 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1211 /* Write the public key */
1212 ret = kern_dump_write_public_key(&outstate);
1213 if (KERN_SUCCESS != ret) {
1214 kern_coredump_log(NULL, "(do_kern_dump write public key) returned 0x%x\n", ret);
1215 dump_succeeded = FALSE;
1216 goto exit;
1217 }
1218 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1219
1220 /* Seek the calculated offset (we'll scrollback later to flush the logs and header) */
1221 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1222 kern_coredump_log(NULL, "(do_kern_dump seek begin) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1223 sizeof(foffset), &foffset, foffset, ret);
1224 dump_succeeded = FALSE;
1225 goto exit;
1226 }
1227 }
1228
1229 #if defined(__arm64__)
1230 flush_mmu_tlb();
1231 #endif
1232
1233 kern_coredump_log(NULL, "%s", (kd_variant == KERN_DUMP_DISK) ? "Writing local cores...\n" :
1234 "Transmitting kernel state, please wait:\n");
1235
1236 #if defined (__arm64__)
1237 char *panic_buf = (char *)gPanicBase;
1238 size_t panic_len = (vm_offset_t)debug_buf_ptr - gPanicBase;
1239 if (kd_variant == KERN_DUMP_DISK && (panic_buf && panic_len)) {
1240 ret = dump_panic_buffer(&outstate, panic_buf, panic_len, &foffset, details_flags);
1241 if (KERN_SUCCESS != ret) {
1242 dump_succeeded = FALSE;
1243 }
1244 }
1245 #endif
1246
1247 #if defined(__x86_64__)
1248 if (((kd_variant == KERN_DUMP_STACKSHOT_DISK) || (kd_variant == KERN_DUMP_DISK)) && ((panic_stackshot_buf != 0) && (panic_stackshot_len != 0))) {
1249 bool should_skip = false;
1250 static const char *stackshot_filename = "panic_stackshot.kcdata";
1251
1252 kern_coredump_log(NULL, "\nBeginning dump of kernel stackshot\n");
1253
1254 ret = kdp_reset_output_vars(&outstate, panic_stackshot_len, true, &should_skip, stackshot_filename, RAW_COREDUMP);
1255
1256 if (ret != KERN_SUCCESS) {
1257 kern_coredump_log(NULL, "Failed to reset outstate for stackshot with len 0x%zx, returned 0x%x\n", panic_stackshot_len, ret);
1258 dump_succeeded = FALSE;
1259 } else if (!should_skip) {
1260 uint64_t compressed_stackshot_len = 0;
1261 if ((ret = kdp_core_output(&outstate, panic_stackshot_len, (void *)panic_stackshot_buf)) != KERN_SUCCESS) {
1262 kern_coredump_log(NULL, "Failed to write panic stackshot to file, kdp_coreoutput(outstate, %lu, %p) returned 0x%x\n",
1263 panic_stackshot_len, (void *) panic_stackshot_buf, ret);
1264 dump_succeeded = FALSE;
1265 } else if ((ret = kdp_core_output(&outstate, 0, NULL)) != KERN_SUCCESS) {
1266 kern_coredump_log(NULL, "Failed to flush stackshot data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", &outstate, ret);
1267 dump_succeeded = FALSE;
1268 } else if ((ret = kern_dump_record_file(&outstate, stackshot_filename, foffset, &compressed_stackshot_len, details_flags)) != KERN_SUCCESS) {
1269 kern_coredump_log(NULL, "Failed to record panic stackshot in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1270 dump_succeeded = FALSE;
1271 } else {
1272 kern_coredump_log(NULL, "Recorded panic stackshot in corefile at offset 0x%llx, compressed to %llu bytes\n", foffset, compressed_stackshot_len);
1273 foffset = roundup((foffset + compressed_stackshot_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1274 if ((ret = kern_dump_seek_to_next_file(&outstate, foffset)) != KERN_SUCCESS) {
1275 kern_coredump_log(NULL, "Failed to seek to stackshot file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", foffset, ret);
1276 dump_succeeded = FALSE;
1277 }
1278 }
1279 } else {
1280 kern_coredump_log(NULL, "Skipping stackshot dump\n");
1281 }
1282 }
1283 #endif
1284
1285 if (kd_variant == KERN_DUMP_DISK) {
1286 /*
1287 * Dump co-processors as well, foffset will be overwritten with the
1288 * offset of the next location in the file to be written to.
1289 */
1290 if (kern_do_coredump(&outstate, KCF_NONE, foffset, &foffset, details_flags) != 0) {
1291 dump_succeeded = FALSE;
1292 }
1293 #if defined (__arm64__)
1294 } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) {
1295 kern_coredump_log(NULL, "Writing all cores through shared memory debugger\n");
1296 if (kern_do_coredump(&outstate, KCF_ABORT_ON_FAILURE, foffset, &foffset, details_flags) != 0) {
1297 dump_succeeded = FALSE;
1298 }
1299 #endif /* __arm64__ */
1300 } else if (kd_variant != KERN_DUMP_STACKSHOT_DISK) {
1301 /* Only the kernel */
1302 if (kern_do_coredump(&outstate, KCF_KERNEL_ONLY, foffset, &foffset, details_flags) != 0) {
1303 dump_succeeded = FALSE;
1304 }
1305 }
1306
1307 if (kd_variant == KERN_DUMP_DISK) {
1308 assert(reserved_debug_logsize != 0);
1309 size_t remaining_debug_logspace = reserved_debug_logsize;
1310
1311 /* Write the debug log -- first seek to the end of the corefile header */
1312 foffset = kdp_core_header->log_offset;
1313 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1314 kern_coredump_log(NULL, "(do_kern_dump seek logfile) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1315 sizeof(foffset), &foffset, foffset, ret);
1316 dump_succeeded = FALSE;
1317 goto exit;
1318 }
1319
1320 /* First flush the data from just the paniclog */
1321 size_t initial_log_length = 0;
1322 #if defined(__arm64__)
1323 initial_log_length = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1324 panic_info->eph_panic_log_len;
1325 #else
1326 if (panic_info->mph_panic_log_offset != 0) {
1327 initial_log_length = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1328 panic_info->mph_panic_log_len;
1329 }
1330 #endif
1331
1332 buf = debug_buf_base;
1333 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, initial_log_length, buf)) != KERN_SUCCESS) {
1334 kern_coredump_log(NULL, "(do_kern_dump paniclog) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1335 initial_log_length, buf, ret);
1336 dump_succeeded = FALSE;
1337 goto exit;
1338 }
1339
1340 remaining_debug_logspace -= initial_log_length;
1341
1342 /* Next include any log data from after the stackshot (the beginning of the 'other' log). */
1343 #if defined(__arm64__)
1344 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->eph_other_log_offset);
1345 #else
1346 /*
1347 * There may be no paniclog if we're doing a coredump after a call to Debugger() on x86 if debugger_is_panic was
1348 * configured to FALSE based on the boot-args. In that case just start from where the debug buffer was when
1349 * we began taking a coredump.
1350 */
1351 if (panic_info->mph_other_log_offset != 0) {
1352 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->mph_other_log_offset);
1353 } else {
1354 buf = coredump_log_start;
1355 }
1356 #endif
1357 assert(debug_buf_ptr >= buf);
1358
1359 size_t other_log_length = debug_buf_ptr - buf;
1360 if (other_log_length > remaining_debug_logspace) {
1361 other_log_length = remaining_debug_logspace;
1362 }
1363
1364 /* Write the coredump log */
1365 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, other_log_length, buf)) != KERN_SUCCESS) {
1366 kern_coredump_log(NULL, "(do_kern_dump coredump log) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1367 other_log_length, buf, ret);
1368 dump_succeeded = FALSE;
1369 goto exit;
1370 }
1371
1372 kdp_core_header->log_length = initial_log_length + other_log_length;
1373 kern_dump_update_header(&outstate);
1374 }
1375
1376 exit:
1377 /* close / last packet */
1378 if (output_opened && (ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_EOF, NULL, 0, ((void *) 0))) != KERN_SUCCESS) {
1379 kern_coredump_log(NULL, "(do_kern_dump close) outproc(KDP_EOF, NULL, 0, 0) returned 0x%x\n", ret);
1380 dump_succeeded = FALSE;
1381 }
1382
1383 /* If applicable, update the panic header and flush it so we update the CRC */
1384 #if defined(__arm64__)
1385 panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1386 EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1387 paniclog_flush();
1388 #else
1389 if (panic_info->mph_panic_log_offset != 0) {
1390 panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1391 MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1392 paniclog_flush();
1393 }
1394 #endif
1395
1396 return dump_succeeded ? 0 : -1;
1397 }
1398
1399 boolean_t
dumped_kernel_core(void)1400 dumped_kernel_core(void)
1401 {
1402 return kern_dump_successful;
1403 }
1404
1405 int
kern_dump(enum kern_dump_type kd_variant)1406 kern_dump(enum kern_dump_type kd_variant)
1407 {
1408 static boolean_t local_dump_in_progress = FALSE, dumped_local = FALSE;
1409 int ret = -1;
1410 #if KASAN
1411 kasan_kdp_disable();
1412 #endif
1413 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1414 if (dumped_local) {
1415 return 0;
1416 }
1417 if (local_dump_in_progress) {
1418 return -1;
1419 }
1420 local_dump_in_progress = TRUE;
1421 ret = do_kern_dump(kd_variant);
1422 if (ret == 0) {
1423 dumped_local = TRUE;
1424 kern_dump_successful = TRUE;
1425 local_dump_in_progress = FALSE;
1426 }
1427
1428 return ret;
1429 #if defined(__arm64__)
1430 } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) {
1431 ret = do_kern_dump(kd_variant);
1432 if (ret == 0) {
1433 kern_dump_successful = TRUE;
1434 }
1435 return ret;
1436 #endif
1437 } else {
1438 ret = do_kern_dump(kd_variant);
1439 if (ret == 0) {
1440 kern_dump_successful = TRUE;
1441 }
1442 return ret;
1443 }
1444 }
1445
1446 static kern_return_t
kdp_core_init_output_stages(void)1447 kdp_core_init_output_stages(void)
1448 {
1449 kern_return_t ret = KERN_SUCCESS;
1450
1451 // We only zero-out the disk stage. It will be initialized
1452 // later on when the corefile is initialized
1453 bzero(&disk_output_stage, sizeof(disk_output_stage));
1454
1455 // We only zero-out the LZ4 stage. It will be initialized
1456 // later on when the kext is loaded.
1457 bzero(&lz4_output_stage, sizeof(lz4_output_stage));
1458 lz4_stage_monitor_availability();
1459
1460 // We only initialize the zlib output stage if we can reach the debugger.
1461 // This saves us from wasting some wired memory that will never be used
1462 // in other configurations.
1463 bzero(&zlib_output_stage, sizeof(zlib_output_stage));
1464 if (debug_boot_arg && (debug_boot_arg & DB_REBOOT_ALWAYS) == 0) {
1465 ret = zlib_stage_initialize(&zlib_output_stage);
1466 if (KERN_SUCCESS != ret) {
1467 return ret;
1468 }
1469 }
1470
1471 bzero(&buffer_output_stage, sizeof(buffer_output_stage));
1472 ret = buffer_stage_initialize(&buffer_output_stage, kdp_crashdump_pkt_size);
1473 if (KERN_SUCCESS != ret) {
1474 return ret;
1475 }
1476
1477 bzero(&net_output_stage, sizeof(net_output_stage));
1478 ret = net_stage_initialize(&net_output_stage);
1479 if (KERN_SUCCESS != ret) {
1480 return ret;
1481 }
1482
1483 bzero(&progress_notify_output_stage, sizeof(progress_notify_output_stage));
1484 ret = progress_notify_stage_initialize(&progress_notify_output_stage);
1485 if (KERN_SUCCESS != ret) {
1486 return ret;
1487 }
1488
1489 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1490 // We only zero-out the AEA stage. It will be initialized
1491 // later on, if it's supported and needed
1492 bzero(&aea_output_stage, sizeof(aea_output_stage));
1493 aea_stage_monitor_availability();
1494 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1495
1496 #if defined(__arm64__)
1497 bzero(&shmem_output_stage, sizeof(shmem_output_stage));
1498 if (PE_consistent_debug_enabled() && PE_i_can_has_debugger(NULL)) {
1499 ret = shmem_stage_initialize(&shmem_output_stage);
1500 if (KERN_SUCCESS != ret) {
1501 return ret;
1502 }
1503 }
1504 #endif /* defined(__arm64__) */
1505
1506 #if defined(__arm64__)
1507 bzero(&memory_backing_aware_buffer_output_stage, sizeof(memory_backing_aware_buffer_output_stage));
1508 ret = memory_backing_aware_buffer_stage_initialize(&memory_backing_aware_buffer_output_stage);
1509 if (KERN_SUCCESS != ret) {
1510 return ret;
1511 }
1512 #endif /* defined(__arm64__) */
1513
1514 return ret;
1515 }
1516
1517 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1518
1519 bool
kern_dump_should_enforce_encryption(void)1520 kern_dump_should_enforce_encryption(void)
1521 {
1522 static int enforce_encryption = -1;
1523
1524 // Only check once
1525 if (enforce_encryption == -1) {
1526 uint32_t coredump_encryption_flags = 0;
1527
1528 // When set, the boot-arg is the sole decider
1529 if (!kernel_debugging_restricted() &&
1530 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags))) {
1531 enforce_encryption = (coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT) != 0 ? 1 : 0;
1532 } else {
1533 enforce_encryption = 0;
1534 }
1535 }
1536
1537 return enforce_encryption != 0;
1538 }
1539
1540 static bool
kern_dump_is_encryption_available(void)1541 kern_dump_is_encryption_available(void)
1542 {
1543 // Default to feature enabled unless boot-arg says otherwise
1544 uint32_t coredump_encryption_flags = COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY;
1545
1546 if (!kernel_debugging_restricted()) {
1547 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags));
1548 }
1549
1550 if ((coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY) == 0) {
1551 return false;
1552 }
1553
1554 return aea_stage_is_available();
1555 }
1556
1557 /*
1558 * Initialize (or de-initialize) the encryption stage. This is done in a way such that if initializing the
1559 * encryption stage with a new key fails, then the existing encryption stage is left untouched. Once
1560 * the new stage is initialized, the old stage is uninitialized.
1561 *
1562 * This function is called whenever we have a new public key (whether from someone calling our sysctl, or because
1563 * we read it out of a corefile), or when encryption becomes available.
1564 *
1565 * Parameters:
1566 * - public_key: The public key to use when initializing the encryption stage. Can be NULL to indicate that
1567 * the encryption stage should be de-initialized.
1568 * - public_key_size: The size of the given public key.
1569 */
1570 static kern_return_t
kdp_core_init_encryption_stage(void * public_key,size_t public_key_size)1571 kdp_core_init_encryption_stage(void *public_key, size_t public_key_size)
1572 {
1573 kern_return_t ret = KERN_SUCCESS;
1574 struct kdp_output_stage new_encryption_stage = {};
1575 struct kdp_output_stage old_encryption_stage = {};
1576
1577 lck_mtx_assert(kdp_core_encryption_stage_lock, LCK_MTX_ASSERT_OWNED);
1578
1579 bzero(&new_encryption_stage, sizeof(new_encryption_stage));
1580
1581 if (public_key && kern_dump_is_encryption_available()) {
1582 ret = aea_stage_initialize(&new_encryption_stage, public_key, public_key_size);
1583 if (KERN_SUCCESS != ret) {
1584 printf("(kdp_core_init_encryption_stage) Failed to initialize the encryption stage. Error 0x%x\n", ret);
1585 return ret;
1586 }
1587 }
1588
1589 bcopy(&aea_output_stage, &old_encryption_stage, sizeof(aea_output_stage));
1590
1591 bcopy(&new_encryption_stage, &aea_output_stage, sizeof(new_encryption_stage));
1592
1593 if (old_encryption_stage.kos_initialized && old_encryption_stage.kos_funcs.kosf_free) {
1594 old_encryption_stage.kos_funcs.kosf_free(&old_encryption_stage);
1595 }
1596
1597 return KERN_SUCCESS;
1598 }
1599
1600 kern_return_t
kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data,void * access_context,void * recipient_context)1601 kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context)
1602 {
1603 kern_return_t ret = KERN_SUCCESS;
1604 struct kdp_core_encryption_key_descriptor *key_descriptor = (struct kdp_core_encryption_key_descriptor *) recipient_context;
1605 void *old_public_key = NULL;
1606 size_t old_public_key_size = 0;
1607
1608 if (!key_descriptor) {
1609 return kIOReturnBadArgument;
1610 }
1611
1612 lck_mtx_lock(kdp_core_encryption_stage_lock);
1613 kdp_core_is_initializing_encryption_stage = true;
1614
1615 do {
1616 // Do the risky part first, and bail out cleanly if it fails
1617 ret = kdp_core_init_encryption_stage(key_descriptor->kcekd_key, key_descriptor->kcekd_size);
1618 if (ret != KERN_SUCCESS) {
1619 printf("kdp_core_handle_new_encryption_key failed to re-initialize encryption stage. Error 0x%x\n", ret);
1620 break;
1621 }
1622
1623 // The rest of this function should technically never fail
1624
1625 old_public_key = kdp_core_public_key;
1626 old_public_key_size = kdp_core_header->pub_key_length;
1627
1628 kdp_core_public_key = key_descriptor->kcekd_key;
1629 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1630 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_EXISTING_COREFILE_KEY_FORMAT_MASK;
1631 if (key_descriptor->kcekd_key) {
1632 kdp_core_header->flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1633 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(key_descriptor->kcekd_format);
1634 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1635 kdp_core_header->pub_key_length = key_descriptor->kcekd_size;
1636 } else {
1637 kdp_core_header->pub_key_offset = 0;
1638 kdp_core_header->pub_key_length = 0;
1639 }
1640
1641 /*
1642 * Return the old key to the caller to free
1643 */
1644 key_descriptor->kcekd_key = old_public_key;
1645 key_descriptor->kcekd_size = (uint16_t)old_public_key_size;
1646
1647 // If this stuff fails, we have bigger problems
1648 struct mach_core_fileheader_v2 existing_header;
1649 bool used_existing_header = false;
1650 ret = access_data(access_context, FALSE, 0, sizeof(existing_header), &existing_header);
1651 if (ret != KERN_SUCCESS) {
1652 printf("kdp_core_handle_new_encryption_key failed to read the existing corefile header. Error 0x%x\n", ret);
1653 break;
1654 }
1655
1656 if (existing_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1657 && existing_header.version == 2
1658 && (existing_header.pub_key_length == 0
1659 || kdp_core_header->pub_key_length == 0
1660 || existing_header.pub_key_length == kdp_core_header->pub_key_length)) {
1661 used_existing_header = true;
1662 existing_header.flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1663
1664 if (kdp_core_public_key) {
1665 existing_header.flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1666
1667 if (existing_header.pub_key_offset == 0) {
1668 existing_header.pub_key_offset = kdp_core_header->pub_key_offset;
1669 existing_header.pub_key_length = kdp_core_header->pub_key_length;
1670 }
1671 }
1672
1673 ret = access_data(access_context, TRUE, 0, sizeof(existing_header), &existing_header);
1674 if (ret != KERN_SUCCESS) {
1675 printf("kdp_core_handle_new_encryption_key failed to update the existing corefile header. Error 0x%x\n", ret);
1676 break;
1677 }
1678 } else {
1679 ret = access_data(access_context, TRUE, 0, sizeof(struct mach_core_fileheader_v2), kdp_core_header);
1680 if (ret != KERN_SUCCESS) {
1681 printf("kdp_core_handle_new_encryption_key failed to write the corefile header. Error 0x%x\n", ret);
1682 break;
1683 }
1684 }
1685
1686 if (kdp_core_header->pub_key_length) {
1687 uint64_t offset = used_existing_header ? existing_header.pub_key_offset : kdp_core_header->pub_key_offset;
1688 ret = access_data(access_context, TRUE, offset + PUBLIC_KEY_RESERVED_LENGTH, kdp_core_header->pub_key_length, kdp_core_public_key);
1689 if (ret != KERN_SUCCESS) {
1690 printf("kdp_core_handle_new_encryption_key failed to write the next public key. Error 0x%x\n", ret);
1691 break;
1692 }
1693
1694 if (!used_existing_header) {
1695 // Everything that happens here is optional. It's not the end of the world if this stuff fails, so we don't return
1696 // any errors
1697 // Since we're writing out a completely new header, we make sure to zero-out the region that's reserved for the public key.
1698 // This allows us consumers of the corefile to know for sure that this corefile is not encrypted (yet). Once we actually
1699 // write out a corefile, we'll overwrite this region with the key that we ended up using at the time.
1700 // If we fail to zero-out this region, consumers would read garbage data and properly fail to interpret it as a public key,
1701 // which is why it is OK for us to fail here (it's hard to interpret garbage data as a valid key, and even then, they wouldn't
1702 // find a matching private key anyway)
1703 void *empty_key = NULL;
1704 kern_return_t temp_ret = KERN_SUCCESS;
1705
1706 empty_key = kalloc_data(PUBLIC_KEY_RESERVED_LENGTH,
1707 Z_WAITOK | Z_ZERO | Z_NOFAIL);
1708
1709 temp_ret = access_data(access_context, TRUE, offset, PUBLIC_KEY_RESERVED_LENGTH, empty_key);
1710 kfree_data(empty_key, PUBLIC_KEY_RESERVED_LENGTH);
1711
1712 if (temp_ret != KERN_SUCCESS) {
1713 printf("kdp_core_handle_new_encryption_key failed to zero-out the public key region. Error 0x%x\n", temp_ret);
1714 break;
1715 }
1716 }
1717 }
1718 } while (0);
1719
1720 kdp_core_is_initializing_encryption_stage = false;
1721 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1722
1723 return ret;
1724 }
1725
1726 kern_return_t
kdp_core_handle_encryption_available(void)1727 kdp_core_handle_encryption_available(void)
1728 {
1729 kern_return_t ret;
1730
1731 lck_mtx_lock(kdp_core_encryption_stage_lock);
1732 kdp_core_is_initializing_encryption_stage = true;
1733
1734 ret = kdp_core_init_encryption_stage(kdp_core_public_key, kdp_core_header->pub_key_length);
1735
1736 kdp_core_is_initializing_encryption_stage = false;
1737 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1738
1739 return ret;
1740 }
1741
1742 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1743
1744 kern_return_t
kdp_core_handle_lz4_available(void)1745 kdp_core_handle_lz4_available(void)
1746 {
1747 kern_return_t ret;
1748 lck_mtx_lock(kdp_core_lz4_stage_lock);
1749 kdp_core_is_initializing_lz4_stage = true;
1750
1751 ret = lz4_stage_initialize(&lz4_output_stage);
1752
1753 kdp_core_is_initializing_lz4_stage = false;
1754 lck_mtx_unlock(kdp_core_lz4_stage_lock);
1755
1756 return ret;
1757 }
1758
1759 kern_return_t
kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data,void * access_context,__unused void * recipient_context)1760 kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, __unused void *recipient_context)
1761 {
1762 kern_return_t ret = KERN_SUCCESS;
1763
1764 lck_mtx_lock(kdp_core_disk_stage_lock);
1765 kdp_core_is_initializing_disk_stage = true;
1766
1767 ret = disk_stage_initialize(&disk_output_stage);
1768
1769 kdp_core_is_initializing_disk_stage = false;
1770 lck_mtx_unlock(kdp_core_disk_stage_lock);
1771
1772 if (KERN_SUCCESS != ret) {
1773 return ret;
1774 }
1775
1776 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1777 // If someone has already provided a new public key,
1778 // there's no sense in reading the old one from the corefile.
1779 if (kdp_core_public_key != NULL) {
1780 return KERN_SUCCESS;
1781 }
1782
1783 // The kernel corefile is now available. Let's try to retrieve the public key from its
1784 // header (if available and supported).
1785
1786 // First let's read the corefile header itself
1787 struct mach_core_fileheader_v2 temp_header = {};
1788 ret = access_data(access_context, FALSE, 0, sizeof(temp_header), &temp_header);
1789 if (KERN_SUCCESS != ret) {
1790 printf("kdp_core_polled_io_polled_file_available failed to read corefile header. Error 0x%x\n", ret);
1791 return ret;
1792 }
1793
1794 // Check if the corefile header is initialized, and whether it's initialized to values that we support
1795 // (for backwards and forwards) compatibility, and check whether the header indicates that the corefile has
1796 // has a public key stashed inside of it.
1797 if (temp_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1798 && temp_header.version == 2
1799 && temp_header.pub_key_offset != 0
1800 && temp_header.pub_key_length != 0
1801 /* Future-proofing: make sure it's the key format that we support */
1802 && (temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK) == MACH_CORE_FILEHEADER_V2_FLAG_NEXT_COREFILE_KEY_FORMAT_NIST_P256
1803 /* Add some extra sanity checks. These are not necessary */
1804 && temp_header.pub_key_length <= 4096
1805 && temp_header.pub_key_offset < 65535) {
1806 // The corefile header is properly initialized, is supported, and contains a public key.
1807 // Let's adopt that public key for our encryption needs
1808 void *public_key = NULL;
1809
1810 public_key = kalloc_data(temp_header.pub_key_length,
1811 Z_ZERO | Z_WAITOK | Z_NOFAIL);
1812
1813 // Read the public key from the corefile. Note that the key we're trying to adopt is the "next" key, which is
1814 // PUBLIC_KEY_RESERVED_LENGTH bytes after the public key.
1815 ret = access_data(access_context, FALSE, temp_header.pub_key_offset + PUBLIC_KEY_RESERVED_LENGTH, temp_header.pub_key_length, public_key);
1816 if (KERN_SUCCESS != ret) {
1817 printf("kdp_core_polled_io_polled_file_available failed to read the public key. Error 0x%x\n", ret);
1818 kfree_data(public_key, temp_header.pub_key_length);
1819 return ret;
1820 }
1821
1822 lck_mtx_lock(kdp_core_encryption_stage_lock);
1823 kdp_core_is_initializing_encryption_stage = true;
1824
1825 ret = kdp_core_init_encryption_stage(public_key, temp_header.pub_key_length);
1826 if (KERN_SUCCESS == ret) {
1827 kdp_core_header->flags |= temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1828 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(temp_header.flags);
1829 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1830 kdp_core_header->pub_key_length = temp_header.pub_key_length;
1831 kdp_core_public_key = public_key;
1832 }
1833
1834 kdp_core_is_initializing_encryption_stage = false;
1835 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1836 }
1837 #else
1838 #pragma unused(access_data, access_context)
1839 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1840
1841 return ret;
1842 }
1843
1844 kern_return_t
kdp_core_polled_io_polled_file_unavailable(void)1845 kdp_core_polled_io_polled_file_unavailable(void)
1846 {
1847 lck_mtx_lock(kdp_core_disk_stage_lock);
1848 kdp_core_is_initializing_disk_stage = true;
1849
1850 if (disk_output_stage.kos_initialized && disk_output_stage.kos_funcs.kosf_free) {
1851 disk_output_stage.kos_funcs.kosf_free(&disk_output_stage);
1852 }
1853
1854 kdp_core_is_initializing_disk_stage = false;
1855 lck_mtx_unlock(kdp_core_disk_stage_lock);
1856
1857 return KERN_SUCCESS;
1858 }
1859
1860 void
kdp_core_init(void)1861 kdp_core_init(void)
1862 {
1863 kern_return_t kr;
1864 kern_coredump_callback_config core_config = { };
1865
1866 /* Initialize output stages */
1867 kr = kdp_core_init_output_stages();
1868 assert(KERN_SUCCESS == kr);
1869
1870 kmem_alloc(kernel_map, (vm_offset_t*)&kdp_core_header,
1871 kdp_core_header_size,
1872 KMA_NOFAIL | KMA_ZERO | KMA_PERMANENT | KMA_KOBJECT | KMA_DATA_SHARED,
1873 VM_KERN_MEMORY_DIAG);
1874
1875 kdp_core_header->signature = MACH_CORE_FILEHEADER_V2_SIGNATURE;
1876 kdp_core_header->version = 2;
1877
1878 kdp_core_initialization_lock_group = lck_grp_alloc_init("KDPCoreStageInit", LCK_GRP_ATTR_NULL);
1879 kdp_core_disk_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1880
1881 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1882 kdp_core_encryption_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1883
1884 (void) kern_dump_should_enforce_encryption();
1885 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1886
1887 kdp_core_lz4_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1888
1889 core_config.kcc_coredump_init = kern_dump_init;
1890 core_config.kcc_coredump_get_summary = kern_dump_save_summary;
1891 core_config.kcc_coredump_save_segment_descriptions = kern_dump_save_seg_descriptions;
1892 core_config.kcc_coredump_save_thread_state = kern_dump_save_thread_state;
1893 core_config.kcc_coredump_save_sw_vers_detail = kern_dump_save_sw_vers_detail;
1894 core_config.kcc_coredump_save_segment_data = kern_dump_save_segment_data;
1895 core_config.kcc_coredump_save_note_summary = kern_dump_save_note_summary;
1896 core_config.kcc_coredump_save_note_descriptions = kern_dump_save_note_descriptions;
1897 core_config.kcc_coredump_save_note_data = kern_dump_save_note_data;
1898
1899 kr = kern_register_xnu_coredump_helper(&core_config);
1900 assert(KERN_SUCCESS == kr);
1901 }
1902
1903 /*
1904 * Additional LC_NOTES added to the core.
1905 */
1906
1907 static kern_return_t
kern_dump_save_note_summary(void * refcon __unused,core_save_note_summary_cb callback,void * context)1908 kern_dump_save_note_summary(void *refcon __unused, core_save_note_summary_cb callback, void *context)
1909 {
1910 int count = 1;
1911 size_t size = sizeof(addrable_bits_note_t);
1912
1913 #ifdef CONFIG_SPTM
1914 /* Load binary spec note */
1915
1916 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1917
1918 if (debug_header != NULL &&
1919 debug_header->magic == DEBUG_HEADER_MAGIC_VAL &&
1920 debug_header->version == DEBUG_HEADER_CURRENT_VERSION) {
1921 /* Also add SPTM, TXM, and xnu kc load binary specs if present */
1922 count += debug_header->count;
1923 size += debug_header->count * sizeof(load_binary_spec_note_t);
1924 }
1925 #endif /* CONFIG_SPTM */
1926
1927 return callback(count, size, context);
1928 }
1929
1930 static kern_return_t
kern_dump_save_note_descriptions(void * refcon __unused,core_save_note_descriptions_cb callback,void * context)1931 kern_dump_save_note_descriptions(void *refcon __unused, core_save_note_descriptions_cb callback, void *context)
1932 {
1933 int max_ret = KERN_SUCCESS;
1934 int ret;
1935
1936 max_ret = ret = callback(ADDRABLE_BITS_DATA_OWNER, sizeof(addrable_bits_note_t), context);
1937
1938 #if CONFIG_SPTM
1939 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1940
1941 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1942 ret = callback(LOAD_BINARY_SPEC_DATA_OWNER, sizeof(load_binary_spec_note_t), context);
1943 max_ret = MAX(ret, max_ret);
1944 }
1945 #endif /* CONFIG_SPTM */
1946
1947 return max_ret;
1948 }
1949
1950 static kern_return_t
kern_dump_save_note_data(void * refcon __unused,core_save_note_data_cb callback,void * context)1951 kern_dump_save_note_data(void *refcon __unused, core_save_note_data_cb callback, void *context)
1952 {
1953 int max_ret = KERN_SUCCESS;
1954 int ret;
1955
1956 addrable_bits_note_t note = {
1957 .version = ADDRABLE_BITS_VER,
1958 .addressing_bits = pmap_kernel_va_bits(),
1959 .unused = 0
1960 };
1961
1962 max_ret = ret = callback(¬e, sizeof(addrable_bits_note_t), context);
1963
1964 #if CONFIG_SPTM
1965 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1966
1967 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1968 load_binary_spec_note_t load_binary_spec = {
1969 .version = LOAD_BINARY_SPEC_VERSION,
1970 .uuid = {0},
1971 .address = (uint64_t)debug_header->image[i],
1972 .slide = UINT64_MAX // unknown, load address specified
1973 };
1974
1975 char const *name;
1976 switch (i) {
1977 case DEBUG_HEADER_ENTRY_SPTM:
1978 name = "sptm";
1979 break;
1980 case DEBUG_HEADER_ENTRY_XNU:
1981 name = "xnu";
1982 break;
1983 case DEBUG_HEADER_ENTRY_TXM:
1984 name = "txm";
1985 break;
1986 default:
1987 name = "UNKNOWN";
1988 kern_coredump_log(context, "%s(): encountered unknown debug header entry %d, "
1989 "including anyway with name '%s'\n", __func__, i, name);
1990 }
1991
1992 strlcpy(load_binary_spec.name_cstring, name, LOAD_BINARY_NAME_BUF_SIZE);
1993
1994 ret = callback(&load_binary_spec, sizeof(load_binary_spec), context);
1995
1996 if (ret != KERN_SUCCESS) {
1997 kern_coredump_log(context, "%s(): failed to write load binary spec structure "
1998 "for binary #%d ('%s'): callback returned 0x%x\n",
1999 __func__, i, name, ret);
2000 max_ret = MAX(ret, max_ret);
2001 }
2002 }
2003 #endif /* CONFIG_SPTM */
2004
2005 return max_ret;
2006 }
2007
2008 #else
2009
2010 void
kdp_core_exclude_region(__unused vm_offset_t addr,__unused vm_size_t size)2011 kdp_core_exclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
2012 {
2013 }
2014
2015 void
kdp_core_unexclude_region(__unused vm_offset_t addr,__unused vm_size_t size)2016 kdp_core_unexclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
2017 {
2018 }
2019
2020 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
2021