1 /*
2 * Copyright (c) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * The main orchestrator for kernel (and co-processor) coredumps. Here's a very simplistic view of
31 * the flow:
32 *
33 * At kernel initialization time (kdp_core_init):
34 * ----------------------------------------------
35 *
36 * - kdp_core_init() takes care of allocating all necessary data structures and initializes the
37 * coredump output stages
38 *
39 * At coredump time (do_kern_dump):
40 * --------------------------------
41 *
42 * - Depending on the coredump variant, we chain the necessary output stages together in chain_output_stages()
43 * - [Disk only] We initialize the corefile header
44 * - [Disk only] We stream the stackshot out through the output stages and update the corefile header
45 * - We perform the kernel coredump, streaming it out through the output stages
46 * - [Disk only] We update the corefile header
47 * - [Disk only] We perform the co-processor coredumps (driven by kern_do_coredump), streaming each out
48 * through the output stages and updating the corefile header.
49 * - [Disk only] We save the coredump log to the corefile
50 */
51
52 #include <mach/kern_return.h>
53 #include <mach/vm_types.h>
54 #include <kdp/core_exclude.h>
55 #include <kdp/kdp_core.h>
56 #include <kdp/core_notes.h>
57
58 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
59
60 #include <mach/mach_types.h>
61 #include <mach/vm_attributes.h>
62 #include <mach/vm_param.h>
63 #include <mach/vm_map.h>
64 #include <vm/vm_protos.h>
65 #include <vm/vm_kern_xnu.h>
66 #include <vm/vm_map.h>
67 #include <machine/cpu_capabilities.h>
68 #include <libsa/types.h>
69 #include <libkern/kernel_mach_header.h>
70 #include <kern/locks.h>
71 #include <kdp/kdp_internal.h>
72 #include <kdp/output_stages/output_stages.h>
73 #include <kdp/processor_core.h>
74 #include <IOKit/IOTypes.h>
75 #include <IOKit/IOBSD.h>
76 #include <sys/errno.h>
77 #include <sys/msgbuf.h>
78 #include <san/kasan.h>
79 #include <kern/debug.h>
80 #include <pexpert/pexpert.h>
81 #include <os/atomic_private.h>
82
83 #if CONFIG_SPTM
84 #include <sptm/debug_header.h>
85 #endif
86
87 #if defined(__x86_64__)
88 #include <i386/pmap_internal.h>
89 #include <kdp/ml/i386/kdp_x86_common.h>
90 #include <kern/debug.h>
91 #endif /* defined(__x86_64__) */
92
93 #if CONFIG_SPTM
94 #include <arm64/sptm/sptm.h>
95 #endif /* CONFIG_SPTM */
96
97 kern_return_t kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context);
98 kern_return_t kdp_core_polled_io_polled_file_unavailable(void);
99
100 typedef int (*pmap_traverse_callback)(vm_map_offset_t start,
101 vm_map_offset_t end,
102 void *context);
103
104 static kern_return_t kern_dump_init(void *refcon, void *context);
105 static int kern_dump_save_summary(void *refcon, core_save_summary_cb callback, void *context);
106 static int kern_dump_save_seg_descriptions(void *refcon, core_save_segment_descriptions_cb callback, void *context);
107 static int kern_dump_save_thread_state(void *refcon, void *buf, core_save_thread_state_cb callback, void *context);
108 static int kern_dump_save_sw_vers_detail(void *refcon, core_save_sw_vers_detail_cb callback, void *context);
109 static int kern_dump_save_segment_data(void *refcon, core_save_segment_data_cb callback, void *context);
110 static kern_return_t kern_dump_save_note_summary(void *refcon, core_save_note_summary_cb callback, void *context);
111 static kern_return_t kern_dump_save_note_descriptions(void *refcon, core_save_note_descriptions_cb callback, void *context);
112 static kern_return_t kern_dump_save_note_data(void *refcon, core_save_note_data_cb callback, void *context);
113
114 static int
115 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
116 vm_map_offset_t end,
117 void *context);
118 static int
119 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
120 vm_map_offset_t end,
121 void *context);
122
123 static int
124 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
125 vm_map_offset_t end,
126 void *context);
127
128 static struct kdp_output_stage disk_output_stage = {};
129 static struct kdp_output_stage lz4_output_stage = {};
130 static struct kdp_output_stage zlib_output_stage = {};
131 static struct kdp_output_stage buffer_output_stage = {};
132 static struct kdp_output_stage net_output_stage = {};
133 static struct kdp_output_stage progress_notify_output_stage = {};
134 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
135 static struct kdp_output_stage aea_output_stage = {};
136 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
137 #if defined(__arm64__)
138 static struct kdp_output_stage shmem_output_stage = {};
139 static struct kdp_output_stage memory_backing_aware_buffer_output_stage = {};
140 #endif /* defined(__arm64__) */
141
142 extern uint32_t kdp_crashdump_pkt_size;
143
144 static boolean_t kern_dump_successful = FALSE;
145
146 static const size_t kdp_core_header_size = sizeof(struct mach_core_fileheader_v2) + (KERN_COREDUMP_MAX_CORES * sizeof(struct mach_core_details_v2));
147 static struct mach_core_fileheader_v2 *kdp_core_header = NULL;
148
149 static lck_grp_t *kdp_core_initialization_lock_group = NULL;
150 static lck_mtx_t *kdp_core_disk_stage_lock = NULL;
151 static bool kdp_core_is_initializing_disk_stage = false;
152
153 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
154 static const size_t PUBLIC_KEY_RESERVED_LENGTH = roundup(4096, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
155 static void *kdp_core_public_key = NULL;
156 static lck_mtx_t *kdp_core_encryption_stage_lock = NULL;
157 static bool kdp_core_is_initializing_encryption_stage = false;
158 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
159
160 static lck_mtx_t *kdp_core_lz4_stage_lock = NULL;
161 static bool kdp_core_is_initializing_lz4_stage = false;
162
163 /*
164 * These variables will be modified by the BSD layer if the root device is
165 * a RAMDisk.
166 */
167 uint64_t kdp_core_ramdisk_addr = 0;
168 uint64_t kdp_core_ramdisk_size = 0;
169
170 #define COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY (1 << 0)
171 #define COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT (1 << 1)
172
173 boolean_t
kdp_has_polled_corefile(void)174 kdp_has_polled_corefile(void)
175 {
176 return NULL != gIOPolledCoreFileVars;
177 }
178
179 kern_return_t
kdp_polled_corefile_error(void)180 kdp_polled_corefile_error(void)
181 {
182 return gIOPolledCoreFileOpenRet;
183 }
184
185 IOPolledCoreFileMode_t
kdp_polled_corefile_mode(void)186 kdp_polled_corefile_mode(void)
187 {
188 return gIOPolledCoreFileMode;
189 }
190
191 struct kdp_core_excluded_region {
192 struct kdp_core_excluded_region *next;
193 vm_offset_t addr;
194 vm_size_t size;
195 };
196
197 static LCK_GRP_DECLARE(excluded_regions_grp, "kdp-exclude-regions");
198 static LCK_MTX_DECLARE(excluded_regions_mtx, &excluded_regions_grp);
199 static struct kdp_core_excluded_region *excluded_regions;
200
201 void
kdp_core_exclude_region(vm_offset_t addr,vm_size_t size)202 kdp_core_exclude_region(vm_offset_t addr, vm_size_t size)
203 {
204 struct kdp_core_excluded_region *region;
205
206 if (addr >= addr + size) {
207 panic("%s: cannot exclude region starting at %p with size %zu (zero or overflowing size)",
208 __func__, (void*)addr, (size_t)size);
209 }
210 if (addr != round_page(addr) || size != round_page(size)) {
211 panic("%s: cannot exclude region starting at %p with size %zu (not page aligned)",
212 __func__, (void*)addr, (size_t)size);
213 }
214
215 region = kalloc_type(typeof(*region), Z_WAITOK | Z_NOFAIL);
216 region->addr = addr;
217 region->size = size;
218
219 lck_mtx_lock(&excluded_regions_mtx);
220 region->next = excluded_regions;
221 excluded_regions = region;
222 lck_mtx_unlock(&excluded_regions_mtx);
223 }
224
225 void
kdp_core_unexclude_region(vm_offset_t addr,vm_size_t size)226 kdp_core_unexclude_region(vm_offset_t addr, vm_size_t size)
227 {
228 struct kdp_core_excluded_region *region;
229 struct kdp_core_excluded_region **fixup = &excluded_regions;
230
231 lck_mtx_lock(&excluded_regions_mtx);
232 for (region = excluded_regions; region; region = region->next) {
233 if (region->addr == addr && region->size == size) {
234 *fixup = region->next;
235 break;
236 }
237 fixup = ®ion->next;
238 }
239 if (!region) {
240 panic("%s: cannot unexclude region starting at %p with size %zu (not currently excluded)",
241 __func__, (void*)addr, (size_t)size);
242 }
243 lck_mtx_unlock(&excluded_regions_mtx);
244
245 // We had exclusive access to the list when we removed the region, and it is no longer
246 // reachable from the list, so it is safe to free.
247 kfree_type(typeof(*region), region);
248 }
249
250 static bool
kernel_vaddr_in_excluded_region(vm_offset_t addr,uint64_t * vincr)251 kernel_vaddr_in_excluded_region(vm_offset_t addr, uint64_t *vincr)
252 {
253 struct kdp_core_excluded_region *region;
254
255 // We check this earlier before attempting to dump the kernel, but verify here.
256 assert(!kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx));
257
258 for (region = excluded_regions; region; region = region->next) {
259 if (region->addr <= addr && addr < (region->addr + region->size)) {
260 *vincr = region->size;
261 return true;
262 }
263 }
264
265 return false;
266 }
267
268 kern_return_t
kdp_core_output(void * kdp_core_out_state,uint64_t length,void * data)269 kdp_core_output(void *kdp_core_out_state, uint64_t length, void * data)
270 {
271 kern_return_t err = KERN_SUCCESS;
272 uint64_t percent;
273 struct kdp_core_out_state *vars = (struct kdp_core_out_state *)kdp_core_out_state;
274 struct kdp_output_stage *first_stage = STAILQ_FIRST(&vars->kcos_out_stage);
275
276 if (vars->kcos_error == KERN_SUCCESS) {
277 #if DEVELOPMENT || DEBUG
278 // panic testing: force the write to fail after X number of writes
279 if ((panic_test_case & PANIC_TEST_CASE_COREFILE_IO_ERR) && (--panic_test_action_count == 0)) {
280 panic_test_case &= ~PANIC_TEST_CASE_COREFILE_IO_ERR;
281 length = -1;
282 }
283 #endif
284
285 if ((err = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, length, data)) != KERN_SUCCESS) {
286 kern_coredump_log(NULL, "(kdp_core_output) outproc(KDP_DATA, NULL, 0x%llx, %p) returned 0x%x\n",
287 length, data, err);
288 vars->kcos_error = err;
289 }
290 if (!data && !length) {
291 kern_coredump_log(NULL, "100..");
292 } else {
293 vars->kcos_bytes_written += length;
294 percent = (vars->kcos_bytes_written * 100) / vars->kcos_totalbytes;
295 if ((percent - vars->kcos_lastpercent) >= 10) {
296 vars->kcos_lastpercent = percent;
297 kern_coredump_log(NULL, "%lld..\n", percent);
298 }
299 }
300 }
301 return err;
302 }
303
304 #if defined(__arm64__)
305 extern pmap_paddr_t avail_start, avail_end;
306 extern struct vm_object pmap_object_store;
307 #endif
308 extern vm_offset_t c_buffers;
309 extern vm_size_t c_buffers_size;
310
311 static bool
kernel_vaddr_in_coredump_stage(const struct kdp_output_stage * stage,uint64_t vaddr,uint64_t * vincr)312 kernel_vaddr_in_coredump_stage(const struct kdp_output_stage *stage, uint64_t vaddr, uint64_t *vincr)
313 {
314 uint64_t start_addr = (uint64_t)stage->kos_data;
315 uint64_t end_addr = start_addr + stage->kos_data_size;
316
317 if (!stage->kos_data) {
318 return false;
319 }
320
321 if (vaddr >= start_addr && vaddr < end_addr) {
322 *vincr = stage->kos_data_size - (vaddr - start_addr);
323 return true;
324 }
325
326 return false;
327 }
328
329 static bool
kernel_vaddr_in_coredump_stages(uint64_t vaddr,uint64_t * vincr)330 kernel_vaddr_in_coredump_stages(uint64_t vaddr, uint64_t *vincr)
331 {
332 if (kernel_vaddr_in_coredump_stage(&disk_output_stage, vaddr, vincr)) {
333 return true;
334 }
335
336 if (kernel_vaddr_in_coredump_stage(&lz4_output_stage, vaddr, vincr)) {
337 return true;
338 }
339
340 if (kernel_vaddr_in_coredump_stage(&zlib_output_stage, vaddr, vincr)) {
341 return true;
342 }
343
344 if (kernel_vaddr_in_coredump_stage(&buffer_output_stage, vaddr, vincr)) {
345 return true;
346 }
347
348 if (kernel_vaddr_in_coredump_stage(&net_output_stage, vaddr, vincr)) {
349 return true;
350 }
351
352 if (kernel_vaddr_in_coredump_stage(&progress_notify_output_stage, vaddr, vincr)) {
353 return true;
354 }
355
356 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
357 if (kernel_vaddr_in_coredump_stage(&aea_output_stage, vaddr, vincr)) {
358 return true;
359 }
360 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
361
362 #if defined(__arm64__)
363 if (kernel_vaddr_in_coredump_stage(&shmem_output_stage, vaddr, vincr)) {
364 return true;
365 }
366 #endif /* defined(__arm64__) */
367
368 #if defined(__arm64__)
369 if (kernel_vaddr_in_coredump_stage(&memory_backing_aware_buffer_output_stage, vaddr, vincr)) {
370 return true;
371 }
372 #endif /* defined(__arm64__) */
373
374 return false;
375 }
376
377
378 ppnum_t
kernel_pmap_present_mapping(uint64_t vaddr,uint64_t * pvincr,uintptr_t * pvphysaddr)379 kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphysaddr)
380 {
381 ppnum_t ppn = 0;
382 uint64_t vincr = PAGE_SIZE_64;
383
384 assert(!(vaddr & PAGE_MASK_64));
385
386 /* VA ranges to exclude */
387 if (vaddr == c_buffers) {
388 /* compressor data */
389 ppn = 0;
390 vincr = c_buffers_size;
391 } else if (kernel_vaddr_in_coredump_stages(vaddr, &vincr)) {
392 /* coredump output stage working memory */
393 ppn = 0;
394 } else if ((kdp_core_ramdisk_addr != 0) && (vaddr == kdp_core_ramdisk_addr)) {
395 ppn = 0;
396 vincr = kdp_core_ramdisk_size;
397 } else
398 #if defined(__arm64__)
399 if (vaddr == phystokv(avail_start)) {
400 /* physical memory map */
401 ppn = 0;
402 vincr = (avail_end - avail_start);
403 } else
404 #endif /* defined(__arm64__) */
405 {
406 ppn = (pvphysaddr != NULL ?
407 pmap_find_phys(kernel_pmap, vaddr) :
408 pmap_find_phys_nofault(kernel_pmap, vaddr));
409 }
410
411 *pvincr = round_page_64(vincr);
412
413 if (ppn && pvphysaddr) {
414 uint64_t phys = ptoa_64(ppn);
415 if (physmap_enclosed(phys)) {
416 *pvphysaddr = phystokv(phys);
417 } else {
418 ppn = 0;
419 }
420 }
421
422 return ppn;
423 }
424
425 static int
pmap_traverse_present_mappings(pmap_t __unused pmap,vm_map_offset_t start,vm_map_offset_t end,pmap_traverse_callback callback,void * context)426 pmap_traverse_present_mappings(pmap_t __unused pmap,
427 vm_map_offset_t start,
428 vm_map_offset_t end,
429 pmap_traverse_callback callback,
430 void *context)
431 {
432 IOReturn ret;
433 vm_map_offset_t vcurstart, vcur;
434 uint64_t vincr = 0;
435 vm_map_offset_t debug_start = trunc_page((vm_map_offset_t) debug_buf_base);
436 vm_map_offset_t debug_end = round_page((vm_map_offset_t) (debug_buf_base + debug_buf_size));
437 #if defined(XNU_TARGET_OS_BRIDGE)
438 vm_map_offset_t macos_panic_start = trunc_page((vm_map_offset_t) macos_panic_base);
439 vm_map_offset_t macos_panic_end = round_page((vm_map_offset_t) (macos_panic_base + macos_panic_size));
440 #endif
441
442 boolean_t lastvavalid;
443 #if defined(__arm64__)
444 vm_page_t m = VM_PAGE_NULL;
445 #endif
446
447 #if defined(__x86_64__)
448 assert(!is_ept_pmap(pmap));
449 #endif
450
451 /* Assumes pmap is locked, or being called from the kernel debugger */
452 if (start > end) {
453 return KERN_INVALID_ARGUMENT;
454 }
455
456 ret = KERN_SUCCESS;
457 lastvavalid = FALSE;
458 for (vcur = vcurstart = start; (ret == KERN_SUCCESS) && (vcur < end);) {
459 ppnum_t ppn = 0;
460
461 #if defined(__arm64__)
462 /* We're at the start of the physmap, so pull out the pagetable pages that
463 * are accessed through that region.*/
464 if (vcur == phystokv(avail_start) && vm_object_lock_try_shared(&pmap_object_store)) {
465 m = (vm_page_t)vm_page_queue_first(&pmap_object_store.memq);
466 }
467
468 if (m != VM_PAGE_NULL) {
469 vm_map_offset_t vprev = vcur;
470 ppn = (ppnum_t)atop(avail_end);
471 while (!vm_page_queue_end(&pmap_object_store.memq, (vm_page_queue_entry_t)m)) {
472 /* Ignore pages that come from the static region and have already been dumped.*/
473 if (VM_PAGE_GET_PHYS_PAGE(m) >= atop(avail_start)) {
474 ppn = VM_PAGE_GET_PHYS_PAGE(m);
475 break;
476 }
477 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
478 }
479 vincr = PAGE_SIZE_64;
480 if (ppn == atop(avail_end)) {
481 vm_object_unlock(&pmap_object_store);
482 m = VM_PAGE_NULL;
483 // avail_end is not a valid physical address,
484 // so phystokv(avail_end) may not produce the expected result.
485 vcur = phystokv(avail_start) + (avail_end - avail_start);
486 } else {
487 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
488 vcur = phystokv(ptoa(ppn));
489 }
490 if (vcur != vprev) {
491 ret = callback(vcurstart, vprev, context);
492 lastvavalid = FALSE;
493 }
494 }
495 if (m == VM_PAGE_NULL) {
496 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
497 }
498 #else /* defined(__arm64__) */
499 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
500 #endif
501 if (ppn != 0 && kernel_vaddr_in_excluded_region(vcur, &vincr)) {
502 /* excluded region */
503 ppn = 0;
504 }
505 if (ppn != 0) {
506 if (((vcur < debug_start) || (vcur >= debug_end))
507 && !(
508 pmap_valid_page(ppn)
509 || bootloader_valid_page(ppn)
510 )
511 #if defined(XNU_TARGET_OS_BRIDGE)
512 // include the macOS panic region if it's mapped
513 && ((vcur < macos_panic_start) || (vcur >= macos_panic_end))
514 #endif /* defined(XNU_TARGET_OS_BRIDGE) */
515 ) {
516 /* not something we want */
517 ppn = 0;
518 }
519 /* include the phys carveout only if explictly marked */
520 if (debug_is_in_phys_carveout(vcur) &&
521 !debug_can_coredump_phys_carveout()) {
522 ppn = 0;
523 }
524 }
525
526 if (ppn != 0) {
527 if (!lastvavalid) {
528 /* Start of a new virtual region */
529 vcurstart = vcur;
530 lastvavalid = TRUE;
531 }
532 } else {
533 if (lastvavalid) {
534 /* end of a virtual region */
535 ret = callback(vcurstart, vcur, context);
536 lastvavalid = FALSE;
537 }
538
539 #if defined(__x86_64__)
540 /* Try to skip by 2MB if possible */
541 if ((vcur & PDMASK) == 0) {
542 pd_entry_t *pde;
543 pde = pmap_pde(pmap, vcur);
544 if (0 == pde || ((*pde & INTEL_PTE_VALID) == 0)) {
545 /* Make sure we wouldn't overflow */
546 if (vcur < (end - NBPD)) {
547 vincr = NBPD;
548 }
549 }
550 }
551 #endif /* defined(__x86_64__) */
552 }
553 vcur += vincr;
554 }
555
556 if ((ret == KERN_SUCCESS) && lastvavalid) {
557 /* send previous run */
558 ret = callback(vcurstart, vcur, context);
559 }
560
561 #if KASAN
562 if (ret == KERN_SUCCESS) {
563 ret = kasan_traverse_mappings(callback, context);
564 }
565 #endif
566
567
568 return ret;
569 }
570
571 struct kern_dump_preflight_context {
572 uint32_t region_count;
573 uint64_t dumpable_bytes;
574 };
575
576 int
kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)577 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
578 vm_map_offset_t end,
579 void *context)
580 {
581 struct kern_dump_preflight_context *kdc = (struct kern_dump_preflight_context *)context;
582 IOReturn ret = KERN_SUCCESS;
583
584 kdc->region_count++;
585 kdc->dumpable_bytes += (end - start);
586
587 return ret;
588 }
589
590
591 struct kern_dump_send_seg_desc_context {
592 core_save_segment_descriptions_cb callback;
593 void *context;
594 };
595
596 int
kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)597 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
598 vm_map_offset_t end,
599 void *context)
600 {
601 struct kern_dump_send_seg_desc_context *kds_context = (struct kern_dump_send_seg_desc_context *)context;
602 uint64_t seg_start = (uint64_t) start;
603 uint64_t seg_end = (uint64_t) end;
604
605 return kds_context->callback(seg_start, seg_end, kds_context->context);
606 }
607
608 struct kern_dump_send_segdata_context {
609 core_save_segment_data_cb callback;
610 void *context;
611 };
612
613 int
kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)614 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
615 vm_map_offset_t end,
616 void *context)
617 {
618 struct kern_dump_send_segdata_context *kds_context = (struct kern_dump_send_segdata_context *)context;
619
620 return kds_context->callback((void *)start, (uint64_t)(end - start), kds_context->context);
621 }
622
623 static kern_return_t
kern_dump_init(__unused void * refcon,void * context)624 kern_dump_init(__unused void *refcon, void *context)
625 {
626 /* TODO: consider doing mmu flush from an init function */
627
628 // If excluded regions list is locked, it is unsafe to dump the kernel.
629 if (kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx)) {
630 kern_coredump_log(context, "%s: skipping kernel because excluded regions list is locked\n",
631 __func__);
632 #if defined(__arm64__)
633 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
634 #else
635 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
636 #endif
637 paniclog_flush();
638 return KERN_NODE_DOWN;
639 }
640
641 return KERN_SUCCESS;
642 }
643
644 static int
kern_dump_save_summary(__unused void * refcon,core_save_summary_cb callback,void * context)645 kern_dump_save_summary(__unused void *refcon, core_save_summary_cb callback, void *context)
646 {
647 struct kern_dump_preflight_context kdc_preflight = { };
648 uint64_t thread_state_size = 0, thread_count = 0;
649 vm_map_offset_t vstart = kdp_core_start_addr();
650 kern_return_t ret;
651
652 ret = pmap_traverse_present_mappings(kernel_pmap,
653 vstart,
654 VM_MAX_KERNEL_ADDRESS,
655 kern_dump_pmap_traverse_preflight_callback,
656 &kdc_preflight);
657 if (ret != KERN_SUCCESS) {
658 kern_coredump_log(context, "save_summary: pmap traversal failed: %d\n", ret);
659 return ret;
660 }
661
662 kern_collectth_state_size(&thread_count, &thread_state_size);
663
664 ret = callback(kdc_preflight.region_count, kdc_preflight.dumpable_bytes,
665 thread_count, thread_state_size, 0, context);
666 return ret;
667 }
668
669 static int
kern_dump_save_seg_descriptions(__unused void * refcon,core_save_segment_descriptions_cb callback,void * context)670 kern_dump_save_seg_descriptions(__unused void *refcon, core_save_segment_descriptions_cb callback, void *context)
671 {
672 vm_map_offset_t vstart = kdp_core_start_addr();
673 kern_return_t ret;
674 struct kern_dump_send_seg_desc_context kds_context;
675
676 kds_context.callback = callback;
677 kds_context.context = context;
678
679 ret = pmap_traverse_present_mappings(kernel_pmap,
680 vstart,
681 VM_MAX_KERNEL_ADDRESS,
682 kern_dump_pmap_traverse_send_segdesc_callback,
683 &kds_context);
684 if (ret != KERN_SUCCESS) {
685 kern_coredump_log(context, "save_seg_desc: pmap traversal failed: %d\n", ret);
686 return ret;
687 }
688
689 return KERN_SUCCESS;
690 }
691
692 static int
kern_dump_save_thread_state(__unused void * refcon,void * buf,core_save_thread_state_cb callback,void * context)693 kern_dump_save_thread_state(__unused void *refcon, void *buf, core_save_thread_state_cb callback, void *context)
694 {
695 kern_return_t ret;
696 uint64_t thread_state_size = 0, thread_count = 0;
697
698 kern_collectth_state_size(&thread_count, &thread_state_size);
699
700 if (thread_state_size > 0) {
701 void * iter = NULL;
702 do {
703 kern_collectth_state(current_thread(), buf, thread_state_size, &iter);
704
705 ret = callback(buf, context);
706 if (ret != KERN_SUCCESS) {
707 return ret;
708 }
709 } while (iter);
710 }
711
712 return KERN_SUCCESS;
713 }
714
715
716 static int
kern_dump_save_sw_vers_detail(__unused void * refcon,core_save_sw_vers_detail_cb callback,void * context)717 kern_dump_save_sw_vers_detail(__unused void *refcon, core_save_sw_vers_detail_cb callback, void *context)
718 {
719 return callback(vm_kernel_stext, kernel_uuid, 0, context);
720 }
721
722 static int
kern_dump_save_segment_data(__unused void * refcon,core_save_segment_data_cb callback,void * context)723 kern_dump_save_segment_data(__unused void *refcon, core_save_segment_data_cb callback, void *context)
724 {
725 vm_map_offset_t vstart = kdp_core_start_addr();
726 kern_return_t ret;
727 struct kern_dump_send_segdata_context kds_context;
728
729 kds_context.callback = callback;
730 kds_context.context = context;
731
732 ret = pmap_traverse_present_mappings(kernel_pmap,
733 vstart,
734 VM_MAX_KERNEL_ADDRESS, kern_dump_pmap_traverse_send_segdata_callback, &kds_context);
735 if (ret != KERN_SUCCESS) {
736 kern_coredump_log(context, "save_seg_data: pmap traversal failed: %d\n", ret);
737 return ret;
738 }
739
740 return KERN_SUCCESS;
741 }
742
743 kern_return_t
kdp_reset_output_vars(void * kdp_core_out_state,uint64_t totalbytes,bool encrypt_core,bool * out_should_skip_coredump)744 kdp_reset_output_vars(void *kdp_core_out_state, uint64_t totalbytes, bool encrypt_core, bool *out_should_skip_coredump)
745 {
746 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
747 struct kdp_output_stage *current_stage = NULL;
748
749 /* Re-initialize kdp_outstate */
750 outstate->kcos_totalbytes = totalbytes;
751 outstate->kcos_bytes_written = 0;
752 outstate->kcos_lastpercent = 0;
753 outstate->kcos_error = KERN_SUCCESS;
754
755 /* Reset the output stages */
756 STAILQ_FOREACH(current_stage, &outstate->kcos_out_stage, kos_next) {
757 current_stage->kos_funcs.kosf_reset(current_stage);
758 }
759
760 *out_should_skip_coredump = false;
761 if (encrypt_core) {
762 if (outstate->kcos_enforce_encryption && !outstate->kcos_encryption_stage) {
763 *out_should_skip_coredump = true;
764 #if defined(__arm64__)
765 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
766 #else
767 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
768 #endif
769 kern_coredump_log(NULL, "(kdp_reset_output_vars) Encryption requested, is unavailable, and enforcement is active. Skipping current core.\n");
770 }
771 } else if (outstate->kcos_encryption_stage) {
772 outstate->kcos_encryption_stage->kos_bypass = true;
773 }
774
775 return KERN_SUCCESS;
776 }
777
778 static kern_return_t
kern_dump_update_header(struct kdp_core_out_state * outstate)779 kern_dump_update_header(struct kdp_core_out_state *outstate)
780 {
781 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
782 uint64_t foffset;
783 kern_return_t ret;
784
785 /* Write the file header -- first seek to the beginning of the file */
786 foffset = 0;
787 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
788 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
789 sizeof(foffset), &foffset, foffset, ret);
790 return ret;
791 }
792
793 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header_size, kdp_core_header)) != KERN_SUCCESS) {
794 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
795 kdp_core_header_size, kdp_core_header, ret);
796 return ret;
797 }
798
799 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
800 kern_coredump_log(NULL, "(kern_dump_update_header) outproc data flush returned 0x%x\n", ret);
801 return ret;
802 }
803
804 #if defined(__arm64__)
805 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
806 kern_coredump_log(NULL, "(kern_dump_update_header) outproc explicit flush returned 0x%x\n", ret);
807 return ret;
808 }
809 #endif /* defined(__arm64__) */
810
811 return ret;
812 }
813
814 kern_return_t
kern_dump_record_file(void * kdp_core_out_state,const char * filename,uint64_t file_offset,uint64_t * out_file_length,uint64_t details_flags)815 kern_dump_record_file(void *kdp_core_out_state, const char *filename, uint64_t file_offset, uint64_t *out_file_length, uint64_t details_flags)
816 {
817 kern_return_t ret = KERN_SUCCESS;
818 uint64_t bytes_written = 0;
819 struct mach_core_details_v2 *core_details = NULL;
820 struct kdp_output_stage *last_stage;
821 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
822
823 assert(kdp_core_header->num_files < KERN_COREDUMP_MAX_CORES);
824 assert(out_file_length != NULL);
825 *out_file_length = 0;
826
827 last_stage = STAILQ_LAST(&outstate->kcos_out_stage, kdp_output_stage, kos_next);
828 bytes_written = last_stage->kos_bytes_written;
829
830 core_details = &(kdp_core_header->files[kdp_core_header->num_files]);
831 core_details->flags = details_flags;
832 core_details->offset = file_offset;
833 core_details->length = bytes_written;
834 strncpy((char *)&core_details->core_name, filename,
835 MACH_CORE_FILEHEADER_NAMELEN);
836 core_details->core_name[MACH_CORE_FILEHEADER_NAMELEN - 1] = '\0';
837
838 kdp_core_header->num_files++;
839
840 ret = kern_dump_update_header(outstate);
841 if (ret == KERN_SUCCESS) {
842 *out_file_length = bytes_written;
843 }
844
845 return ret;
846 }
847
848 kern_return_t
kern_dump_seek_to_next_file(void * kdp_core_out_state,uint64_t next_file_offset)849 kern_dump_seek_to_next_file(void *kdp_core_out_state, uint64_t next_file_offset)
850 {
851 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
852 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
853 kern_return_t ret;
854
855 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(next_file_offset), &next_file_offset)) != KERN_SUCCESS) {
856 kern_coredump_log(NULL, "(kern_dump_seek_to_next_file) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
857 sizeof(next_file_offset), &next_file_offset, next_file_offset, ret);
858 }
859
860 return ret;
861 }
862
863 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
864
865 static kern_return_t
kern_dump_write_public_key(struct kdp_core_out_state * outstate)866 kern_dump_write_public_key(struct kdp_core_out_state *outstate)
867 {
868 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
869 uint64_t foffset;
870 uint64_t remainder = PUBLIC_KEY_RESERVED_LENGTH - kdp_core_header->pub_key_length;
871 kern_return_t ret;
872
873 if (kdp_core_header->pub_key_offset == 0 || kdp_core_header->pub_key_length == 0) {
874 // Nothing to do
875 return KERN_SUCCESS;
876 }
877
878 /* Write the public key -- first seek to the appropriate offset */
879 foffset = kdp_core_header->pub_key_offset;
880 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
881 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
882 sizeof(foffset), &foffset, foffset, ret);
883 return ret;
884 }
885
886 // Write the public key
887 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
888 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
889 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
890 return ret;
891 }
892
893 // Fill out the remainder of the block with zeroes
894 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
895 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
896 remainder, ret);
897 return ret;
898 }
899
900 // Do it once more to write the "next" public key
901 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
902 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
903 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
904 return ret;
905 }
906
907 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
908 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
909 remainder, ret);
910 return ret;
911 }
912
913 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
914 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc data flush returned 0x%x\n", ret);
915 return ret;
916 }
917
918 #if defined(__arm64__)
919 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
920 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc explicit flush returned 0x%x\n", ret);
921 return ret;
922 }
923 #endif /* defined(__arm64__) */
924
925 return ret;
926 }
927
928 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
929
930 static kern_return_t
chain_output_stages(enum kern_dump_type kd_variant,struct kdp_core_out_state * outstate,uint64_t * details_flags)931 chain_output_stages(enum kern_dump_type kd_variant, struct kdp_core_out_state *outstate, uint64_t *details_flags)
932 {
933 struct kdp_output_stage *current = NULL;
934
935 assert(details_flags);
936 *details_flags = 0;
937
938 switch (kd_variant) {
939 case KERN_DUMP_STACKSHOT_DISK:
940 OS_FALLTHROUGH;
941 case KERN_DUMP_DISK:
942 #if defined(__arm64__)
943 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &memory_backing_aware_buffer_output_stage, kos_next);
944 #endif
945 if (!kdp_corezip_disabled) {
946 if (kdp_core_is_initializing_lz4_stage) {
947 kern_coredump_log(NULL, "We were in the middle of initializing LZ4 stage. Cannot write a coredump to disk\n");
948 return KERN_FAILURE;
949 } else if (!lz4_output_stage.kos_initialized) {
950 kern_coredump_log(NULL, "LZ4 stage is not yet initialized. Cannot write a coredump to disk\n");
951 return KERN_FAILURE;
952 }
953 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &lz4_output_stage, kos_next);
954 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_LZ4;
955 }
956 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
957 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
958 if (kdp_core_is_initializing_encryption_stage) {
959 kern_coredump_log(NULL, "We were in the middle of initializing encryption. Marking it as unavailable\n");
960 } else if (aea_output_stage.kos_initialized) {
961 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &aea_output_stage, kos_next);
962 outstate->kcos_encryption_stage = &aea_output_stage;
963 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_ENCRYPTED_AEA;
964 }
965 outstate->kcos_enforce_encryption = kern_dump_should_enforce_encryption();
966 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
967 if (kdp_core_is_initializing_disk_stage) {
968 kern_coredump_log(NULL, "We were in the middle of initializing the disk stage. Cannot write a coredump to disk\n");
969 return KERN_FAILURE;
970 } else if (disk_output_stage.kos_initialized == false) {
971 kern_coredump_log(NULL, "Corefile is not yet initialized. Cannot write a coredump to disk\n");
972 return KERN_FAILURE;
973 }
974 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &disk_output_stage, kos_next);
975 break;
976 case KERN_DUMP_NET:
977 if (!kdp_corezip_disabled) {
978 if (!zlib_output_stage.kos_initialized) {
979 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to the network\n");
980 return KERN_FAILURE;
981 }
982 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
983 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
984 }
985 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
986 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &buffer_output_stage, kos_next);
987 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &net_output_stage, kos_next);
988 break;
989 #if defined(__arm64__)
990 case KERN_DUMP_HW_SHMEM_DBG:
991 if (!kdp_corezip_disabled) {
992 if (!zlib_output_stage.kos_initialized) {
993 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to shared memory\n");
994 return KERN_FAILURE;
995 }
996 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
997 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
998 }
999 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &shmem_output_stage, kos_next);
1000 break;
1001 #endif /* defined(__arm64__) */
1002 }
1003
1004 STAILQ_FOREACH(current, &outstate->kcos_out_stage, kos_next) {
1005 current->kos_outstate = outstate;
1006 }
1007
1008 return KERN_SUCCESS;
1009 }
1010
1011 #if defined(__arm64__)
1012 static kern_return_t
dump_panic_buffer(struct kdp_core_out_state * outstate,char * panic_buf,size_t panic_len,uint64_t * foffset,uint64_t details_flags)1013 dump_panic_buffer(struct kdp_core_out_state *outstate, char *panic_buf, size_t panic_len,
1014 uint64_t *foffset, uint64_t details_flags)
1015 {
1016 kern_return_t ret = KERN_SUCCESS;
1017 bool should_skip = false;
1018
1019 kern_coredump_log(NULL, "\nBeginning dump of panic region of size 0x%zx\n", panic_len);
1020
1021 ret = kdp_reset_output_vars(outstate, panic_len, true, &should_skip);
1022 if (KERN_SUCCESS != ret) {
1023 return ret;
1024 }
1025
1026 if (should_skip) {
1027 kern_coredump_log(NULL, "Skipping panic region dump\n");
1028 return ret;
1029 }
1030
1031 uint64_t compressed_panic_region_len = 0;
1032 ret = kdp_core_output(outstate, panic_len, panic_buf);
1033 if (KERN_SUCCESS != ret) {
1034 kern_coredump_log(NULL, "Failed to write panic region to file, kdp_coreoutput(outstate, %zu, %p) returned 0x%x\n",
1035 panic_len, panic_buf, ret);
1036 return ret;
1037 }
1038
1039 ret = kdp_core_output(outstate, 0, NULL);
1040 if (KERN_SUCCESS != ret) {
1041 kern_coredump_log(NULL, "Failed to flush panic region data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", outstate, ret);
1042 return ret;
1043 }
1044
1045 ret = kern_dump_record_file(outstate, "panic_region", *foffset, &compressed_panic_region_len,
1046 details_flags);
1047 if (KERN_SUCCESS != ret) {
1048 kern_coredump_log(NULL, "Failed to record panic region in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1049 return ret;
1050 }
1051
1052 kern_coredump_log(NULL, "Recorded panic region in corefile at offset 0x%llx, compressed to %llu bytes\n", *foffset, compressed_panic_region_len);
1053 *foffset = roundup((*foffset + compressed_panic_region_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1054
1055 ret = kern_dump_seek_to_next_file(outstate, *foffset);
1056 if (KERN_SUCCESS != ret) {
1057 kern_coredump_log(NULL, "Failed to seek to panic region file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", *foffset, ret);
1058 return ret;
1059 }
1060
1061 return ret;
1062 }
1063 #endif /* defined(__arm64__) */
1064
1065 static int
do_kern_dump(enum kern_dump_type kd_variant)1066 do_kern_dump(enum kern_dump_type kd_variant)
1067 {
1068 struct kdp_core_out_state outstate = { };
1069 struct kdp_output_stage *first_stage = NULL;
1070 char *coredump_log_start = NULL, *buf = NULL;
1071 size_t reserved_debug_logsize = 0, prior_debug_logsize = 0;
1072 uint64_t foffset = 0;
1073 kern_return_t ret = KERN_SUCCESS;
1074 boolean_t output_opened = FALSE, dump_succeeded = TRUE;
1075 uint64_t details_flags = 0;
1076
1077 /* Initialize output context */
1078
1079 bzero(&outstate, sizeof(outstate));
1080 STAILQ_INIT(&outstate.kcos_out_stage);
1081 ret = chain_output_stages(kd_variant, &outstate, &details_flags);
1082 if (KERN_SUCCESS != ret) {
1083 dump_succeeded = FALSE;
1084 goto exit;
1085 }
1086 first_stage = STAILQ_FIRST(&outstate.kcos_out_stage);
1087
1088 /*
1089 * Record the initial panic log buffer length so we can dump the coredump log
1090 * and panic log to disk
1091 */
1092 coredump_log_start = debug_buf_ptr;
1093 #if defined(__arm64__)
1094 assert(panic_info->eph_other_log_offset != 0);
1095 assert(panic_info->eph_panic_log_len != 0);
1096 /* Include any data from before the panic log as well */
1097 prior_debug_logsize = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1098 panic_info->eph_panic_log_len + panic_info->eph_other_log_len;
1099 #else /* defined(__arm64__) */
1100 if (panic_info->mph_panic_log_offset != 0) {
1101 prior_debug_logsize = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1102 panic_info->mph_panic_log_len + panic_info->mph_other_log_len;
1103 }
1104 #endif /* defined(__arm64__) */
1105
1106 assert(prior_debug_logsize <= debug_buf_size);
1107
1108 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1109 /* Open the file for output */
1110 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_WRQ, NULL, 0, NULL)) != KERN_SUCCESS) {
1111 kern_coredump_log(NULL, "outproc(KDP_WRQ, NULL, 0, NULL) returned 0x%x\n", ret);
1112 dump_succeeded = FALSE;
1113 goto exit;
1114 }
1115 }
1116 output_opened = true;
1117
1118 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1119 const size_t aligned_corefile_header_size = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1120 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1121 const size_t aligned_public_key_size = PUBLIC_KEY_RESERVED_LENGTH * 2;
1122 #else
1123 const size_t aligned_public_key_size = 0;
1124 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1125
1126 reserved_debug_logsize = prior_debug_logsize + KERN_COREDUMP_MAXDEBUGLOGSIZE;
1127
1128 /* Space for file header, public key, panic log, core log */
1129 foffset = roundup(aligned_corefile_header_size + aligned_public_key_size + reserved_debug_logsize, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1130 kdp_core_header->log_offset = aligned_corefile_header_size + aligned_public_key_size;
1131
1132 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1133 /* Write the public key */
1134 ret = kern_dump_write_public_key(&outstate);
1135 if (KERN_SUCCESS != ret) {
1136 kern_coredump_log(NULL, "(do_kern_dump write public key) returned 0x%x\n", ret);
1137 dump_succeeded = FALSE;
1138 goto exit;
1139 }
1140 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1141
1142 /* Seek the calculated offset (we'll scrollback later to flush the logs and header) */
1143 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1144 kern_coredump_log(NULL, "(do_kern_dump seek begin) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1145 sizeof(foffset), &foffset, foffset, ret);
1146 dump_succeeded = FALSE;
1147 goto exit;
1148 }
1149 }
1150
1151 #if defined(__arm64__)
1152 flush_mmu_tlb();
1153 #endif
1154
1155 kern_coredump_log(NULL, "%s", (kd_variant == KERN_DUMP_DISK) ? "Writing local cores...\n" :
1156 "Transmitting kernel state, please wait:\n");
1157
1158 #if defined (__arm64__)
1159 char *panic_buf = (char *)gPanicBase;
1160 size_t panic_len = (vm_offset_t)debug_buf_ptr - gPanicBase;
1161 if (kd_variant == KERN_DUMP_DISK && (panic_buf && panic_len)) {
1162 ret = dump_panic_buffer(&outstate, panic_buf, panic_len, &foffset, details_flags);
1163 if (KERN_SUCCESS != ret) {
1164 dump_succeeded = FALSE;
1165 }
1166 }
1167 #endif
1168
1169 #if defined(__x86_64__)
1170 if (((kd_variant == KERN_DUMP_STACKSHOT_DISK) || (kd_variant == KERN_DUMP_DISK)) && ((panic_stackshot_buf != 0) && (panic_stackshot_len != 0))) {
1171 bool should_skip = false;
1172
1173 kern_coredump_log(NULL, "\nBeginning dump of kernel stackshot\n");
1174
1175 ret = kdp_reset_output_vars(&outstate, panic_stackshot_len, true, &should_skip);
1176
1177 if (ret != KERN_SUCCESS) {
1178 kern_coredump_log(NULL, "Failed to reset outstate for stackshot with len 0x%zx, returned 0x%x\n", panic_stackshot_len, ret);
1179 dump_succeeded = FALSE;
1180 } else if (!should_skip) {
1181 uint64_t compressed_stackshot_len = 0;
1182 if ((ret = kdp_core_output(&outstate, panic_stackshot_len, (void *)panic_stackshot_buf)) != KERN_SUCCESS) {
1183 kern_coredump_log(NULL, "Failed to write panic stackshot to file, kdp_coreoutput(outstate, %lu, %p) returned 0x%x\n",
1184 panic_stackshot_len, (void *) panic_stackshot_buf, ret);
1185 dump_succeeded = FALSE;
1186 } else if ((ret = kdp_core_output(&outstate, 0, NULL)) != KERN_SUCCESS) {
1187 kern_coredump_log(NULL, "Failed to flush stackshot data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", &outstate, ret);
1188 dump_succeeded = FALSE;
1189 } else if ((ret = kern_dump_record_file(&outstate, "panic_stackshot.kcdata", foffset, &compressed_stackshot_len, details_flags)) != KERN_SUCCESS) {
1190 kern_coredump_log(NULL, "Failed to record panic stackshot in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1191 dump_succeeded = FALSE;
1192 } else {
1193 kern_coredump_log(NULL, "Recorded panic stackshot in corefile at offset 0x%llx, compressed to %llu bytes\n", foffset, compressed_stackshot_len);
1194 foffset = roundup((foffset + compressed_stackshot_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1195 if ((ret = kern_dump_seek_to_next_file(&outstate, foffset)) != KERN_SUCCESS) {
1196 kern_coredump_log(NULL, "Failed to seek to stackshot file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", foffset, ret);
1197 dump_succeeded = FALSE;
1198 }
1199 }
1200 } else {
1201 kern_coredump_log(NULL, "Skipping stackshot dump\n");
1202 }
1203 }
1204 #endif
1205
1206 if (kd_variant == KERN_DUMP_DISK) {
1207 /*
1208 * Dump co-processors as well, foffset will be overwritten with the
1209 * offset of the next location in the file to be written to.
1210 */
1211 if (kern_do_coredump(&outstate, FALSE, foffset, &foffset, details_flags) != 0) {
1212 dump_succeeded = FALSE;
1213 }
1214 } else if (kd_variant != KERN_DUMP_STACKSHOT_DISK) {
1215 /* Only the kernel */
1216 if (kern_do_coredump(&outstate, TRUE, foffset, &foffset, details_flags) != 0) {
1217 dump_succeeded = FALSE;
1218 }
1219 }
1220
1221 if (kd_variant == KERN_DUMP_DISK) {
1222 assert(reserved_debug_logsize != 0);
1223 size_t remaining_debug_logspace = reserved_debug_logsize;
1224
1225 /* Write the debug log -- first seek to the end of the corefile header */
1226 foffset = kdp_core_header->log_offset;
1227 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1228 kern_coredump_log(NULL, "(do_kern_dump seek logfile) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1229 sizeof(foffset), &foffset, foffset, ret);
1230 dump_succeeded = FALSE;
1231 goto exit;
1232 }
1233
1234 /* First flush the data from just the paniclog */
1235 size_t initial_log_length = 0;
1236 #if defined(__arm64__)
1237 initial_log_length = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1238 panic_info->eph_panic_log_len;
1239 #else
1240 if (panic_info->mph_panic_log_offset != 0) {
1241 initial_log_length = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1242 panic_info->mph_panic_log_len;
1243 }
1244 #endif
1245
1246 buf = debug_buf_base;
1247 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, initial_log_length, buf)) != KERN_SUCCESS) {
1248 kern_coredump_log(NULL, "(do_kern_dump paniclog) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1249 initial_log_length, buf, ret);
1250 dump_succeeded = FALSE;
1251 goto exit;
1252 }
1253
1254 remaining_debug_logspace -= initial_log_length;
1255
1256 /* Next include any log data from after the stackshot (the beginning of the 'other' log). */
1257 #if defined(__arm64__)
1258 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->eph_other_log_offset);
1259 #else
1260 /*
1261 * There may be no paniclog if we're doing a coredump after a call to Debugger() on x86 if debugger_is_panic was
1262 * configured to FALSE based on the boot-args. In that case just start from where the debug buffer was when
1263 * we began taking a coredump.
1264 */
1265 if (panic_info->mph_other_log_offset != 0) {
1266 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->mph_other_log_offset);
1267 } else {
1268 buf = coredump_log_start;
1269 }
1270 #endif
1271 assert(debug_buf_ptr >= buf);
1272
1273 size_t other_log_length = debug_buf_ptr - buf;
1274 if (other_log_length > remaining_debug_logspace) {
1275 other_log_length = remaining_debug_logspace;
1276 }
1277
1278 /* Write the coredump log */
1279 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, other_log_length, buf)) != KERN_SUCCESS) {
1280 kern_coredump_log(NULL, "(do_kern_dump coredump log) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1281 other_log_length, buf, ret);
1282 dump_succeeded = FALSE;
1283 goto exit;
1284 }
1285
1286 kdp_core_header->log_length = initial_log_length + other_log_length;
1287 kern_dump_update_header(&outstate);
1288 }
1289
1290 exit:
1291 /* close / last packet */
1292 if (output_opened && (ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_EOF, NULL, 0, ((void *) 0))) != KERN_SUCCESS) {
1293 kern_coredump_log(NULL, "(do_kern_dump close) outproc(KDP_EOF, NULL, 0, 0) returned 0x%x\n", ret);
1294 dump_succeeded = FALSE;
1295 }
1296
1297 /* If applicable, update the panic header and flush it so we update the CRC */
1298 #if defined(__arm64__)
1299 panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1300 EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1301 paniclog_flush();
1302 #else
1303 if (panic_info->mph_panic_log_offset != 0) {
1304 panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1305 MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1306 paniclog_flush();
1307 }
1308 #endif
1309
1310 return dump_succeeded ? 0 : -1;
1311 }
1312
1313 boolean_t
dumped_kernel_core(void)1314 dumped_kernel_core(void)
1315 {
1316 return kern_dump_successful;
1317 }
1318
1319 int
kern_dump(enum kern_dump_type kd_variant)1320 kern_dump(enum kern_dump_type kd_variant)
1321 {
1322 static boolean_t local_dump_in_progress = FALSE, dumped_local = FALSE;
1323 int ret = -1;
1324 #if KASAN
1325 kasan_kdp_disable();
1326 #endif
1327 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1328 if (dumped_local) {
1329 return 0;
1330 }
1331 if (local_dump_in_progress) {
1332 return -1;
1333 }
1334 local_dump_in_progress = TRUE;
1335 ret = do_kern_dump(kd_variant);
1336 if (ret == 0) {
1337 dumped_local = TRUE;
1338 kern_dump_successful = TRUE;
1339 local_dump_in_progress = FALSE;
1340 }
1341
1342 return ret;
1343 #if defined(__arm64__)
1344 } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) {
1345 ret = do_kern_dump(kd_variant);
1346 if (ret == 0) {
1347 kern_dump_successful = TRUE;
1348 }
1349 return ret;
1350 #endif
1351 } else {
1352 ret = do_kern_dump(kd_variant);
1353 if (ret == 0) {
1354 kern_dump_successful = TRUE;
1355 }
1356 return ret;
1357 }
1358 }
1359
1360 static kern_return_t
kdp_core_init_output_stages(void)1361 kdp_core_init_output_stages(void)
1362 {
1363 kern_return_t ret = KERN_SUCCESS;
1364
1365 // We only zero-out the disk stage. It will be initialized
1366 // later on when the corefile is initialized
1367 bzero(&disk_output_stage, sizeof(disk_output_stage));
1368
1369 // We only zero-out the LZ4 stage. It will be initialized
1370 // later on when the kext is loaded.
1371 bzero(&lz4_output_stage, sizeof(lz4_output_stage));
1372 lz4_stage_monitor_availability();
1373
1374 // We only initialize the zlib output stage if we can reach the debugger.
1375 // This saves us from wasting some wired memory that will never be used
1376 // in other configurations.
1377 bzero(&zlib_output_stage, sizeof(zlib_output_stage));
1378 if (debug_boot_arg && (debug_boot_arg & DB_REBOOT_ALWAYS) == 0) {
1379 ret = zlib_stage_initialize(&zlib_output_stage);
1380 if (KERN_SUCCESS != ret) {
1381 return ret;
1382 }
1383 }
1384
1385 bzero(&buffer_output_stage, sizeof(buffer_output_stage));
1386 ret = buffer_stage_initialize(&buffer_output_stage, kdp_crashdump_pkt_size);
1387 if (KERN_SUCCESS != ret) {
1388 return ret;
1389 }
1390
1391 bzero(&net_output_stage, sizeof(net_output_stage));
1392 ret = net_stage_initialize(&net_output_stage);
1393 if (KERN_SUCCESS != ret) {
1394 return ret;
1395 }
1396
1397 bzero(&progress_notify_output_stage, sizeof(progress_notify_output_stage));
1398 ret = progress_notify_stage_initialize(&progress_notify_output_stage);
1399 if (KERN_SUCCESS != ret) {
1400 return ret;
1401 }
1402
1403 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1404 // We only zero-out the AEA stage. It will be initialized
1405 // later on, if it's supported and needed
1406 bzero(&aea_output_stage, sizeof(aea_output_stage));
1407 aea_stage_monitor_availability();
1408 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1409
1410 #if defined(__arm64__)
1411 bzero(&shmem_output_stage, sizeof(shmem_output_stage));
1412 if (PE_consistent_debug_enabled() && PE_i_can_has_debugger(NULL)) {
1413 ret = shmem_stage_initialize(&shmem_output_stage);
1414 if (KERN_SUCCESS != ret) {
1415 return ret;
1416 }
1417 }
1418 #endif /* defined(__arm64__) */
1419
1420 #if defined(__arm64__)
1421 bzero(&memory_backing_aware_buffer_output_stage, sizeof(memory_backing_aware_buffer_output_stage));
1422 ret = memory_backing_aware_buffer_stage_initialize(&memory_backing_aware_buffer_output_stage);
1423 if (KERN_SUCCESS != ret) {
1424 return ret;
1425 }
1426 #endif /* defined(__arm64__) */
1427
1428 return ret;
1429 }
1430
1431 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1432
1433 bool
kern_dump_should_enforce_encryption(void)1434 kern_dump_should_enforce_encryption(void)
1435 {
1436 static int enforce_encryption = -1;
1437
1438 // Only check once
1439 if (enforce_encryption == -1) {
1440 uint32_t coredump_encryption_flags = 0;
1441
1442 // When set, the boot-arg is the sole decider
1443 if (!kernel_debugging_restricted() &&
1444 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags))) {
1445 enforce_encryption = (coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT) != 0 ? 1 : 0;
1446 } else {
1447 enforce_encryption = 0;
1448 }
1449 }
1450
1451 return enforce_encryption != 0;
1452 }
1453
1454 static bool
kern_dump_is_encryption_available(void)1455 kern_dump_is_encryption_available(void)
1456 {
1457 // Default to feature enabled unless boot-arg says otherwise
1458 uint32_t coredump_encryption_flags = COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY;
1459
1460 if (!kernel_debugging_restricted()) {
1461 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags));
1462 }
1463
1464 if ((coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY) == 0) {
1465 return false;
1466 }
1467
1468 return aea_stage_is_available();
1469 }
1470
1471 /*
1472 * Initialize (or de-initialize) the encryption stage. This is done in a way such that if initializing the
1473 * encryption stage with a new key fails, then the existing encryption stage is left untouched. Once
1474 * the new stage is initialized, the old stage is uninitialized.
1475 *
1476 * This function is called whenever we have a new public key (whether from someone calling our sysctl, or because
1477 * we read it out of a corefile), or when encryption becomes available.
1478 *
1479 * Parameters:
1480 * - public_key: The public key to use when initializing the encryption stage. Can be NULL to indicate that
1481 * the encryption stage should be de-initialized.
1482 * - public_key_size: The size of the given public key.
1483 */
1484 static kern_return_t
kdp_core_init_encryption_stage(void * public_key,size_t public_key_size)1485 kdp_core_init_encryption_stage(void *public_key, size_t public_key_size)
1486 {
1487 kern_return_t ret = KERN_SUCCESS;
1488 struct kdp_output_stage new_encryption_stage = {};
1489 struct kdp_output_stage old_encryption_stage = {};
1490
1491 lck_mtx_assert(kdp_core_encryption_stage_lock, LCK_MTX_ASSERT_OWNED);
1492
1493 bzero(&new_encryption_stage, sizeof(new_encryption_stage));
1494
1495 if (public_key && kern_dump_is_encryption_available()) {
1496 ret = aea_stage_initialize(&new_encryption_stage, public_key, public_key_size);
1497 if (KERN_SUCCESS != ret) {
1498 printf("(kdp_core_init_encryption_stage) Failed to initialize the encryption stage. Error 0x%x\n", ret);
1499 return ret;
1500 }
1501 }
1502
1503 bcopy(&aea_output_stage, &old_encryption_stage, sizeof(aea_output_stage));
1504
1505 bcopy(&new_encryption_stage, &aea_output_stage, sizeof(new_encryption_stage));
1506
1507 if (old_encryption_stage.kos_initialized && old_encryption_stage.kos_funcs.kosf_free) {
1508 old_encryption_stage.kos_funcs.kosf_free(&old_encryption_stage);
1509 }
1510
1511 return KERN_SUCCESS;
1512 }
1513
1514 kern_return_t
kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data,void * access_context,void * recipient_context)1515 kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context)
1516 {
1517 kern_return_t ret = KERN_SUCCESS;
1518 struct kdp_core_encryption_key_descriptor *key_descriptor = (struct kdp_core_encryption_key_descriptor *) recipient_context;
1519 void *old_public_key = NULL;
1520 size_t old_public_key_size = 0;
1521
1522 if (!key_descriptor) {
1523 return kIOReturnBadArgument;
1524 }
1525
1526 lck_mtx_lock(kdp_core_encryption_stage_lock);
1527 kdp_core_is_initializing_encryption_stage = true;
1528
1529 do {
1530 // Do the risky part first, and bail out cleanly if it fails
1531 ret = kdp_core_init_encryption_stage(key_descriptor->kcekd_key, key_descriptor->kcekd_size);
1532 if (ret != KERN_SUCCESS) {
1533 printf("kdp_core_handle_new_encryption_key failed to re-initialize encryption stage. Error 0x%x\n", ret);
1534 break;
1535 }
1536
1537 // The rest of this function should technically never fail
1538
1539 old_public_key = kdp_core_public_key;
1540 old_public_key_size = kdp_core_header->pub_key_length;
1541
1542 kdp_core_public_key = key_descriptor->kcekd_key;
1543 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1544 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_EXISTING_COREFILE_KEY_FORMAT_MASK;
1545 if (key_descriptor->kcekd_key) {
1546 kdp_core_header->flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1547 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(key_descriptor->kcekd_format);
1548 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1549 kdp_core_header->pub_key_length = key_descriptor->kcekd_size;
1550 } else {
1551 kdp_core_header->pub_key_offset = 0;
1552 kdp_core_header->pub_key_length = 0;
1553 }
1554
1555 /*
1556 * Return the old key to the caller to free
1557 */
1558 key_descriptor->kcekd_key = old_public_key;
1559 key_descriptor->kcekd_size = (uint16_t)old_public_key_size;
1560
1561 // If this stuff fails, we have bigger problems
1562 struct mach_core_fileheader_v2 existing_header;
1563 bool used_existing_header = false;
1564 ret = access_data(access_context, FALSE, 0, sizeof(existing_header), &existing_header);
1565 if (ret != KERN_SUCCESS) {
1566 printf("kdp_core_handle_new_encryption_key failed to read the existing corefile header. Error 0x%x\n", ret);
1567 break;
1568 }
1569
1570 if (existing_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1571 && existing_header.version == 2
1572 && (existing_header.pub_key_length == 0
1573 || kdp_core_header->pub_key_length == 0
1574 || existing_header.pub_key_length == kdp_core_header->pub_key_length)) {
1575 used_existing_header = true;
1576 existing_header.flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1577
1578 if (kdp_core_public_key) {
1579 existing_header.flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1580
1581 if (existing_header.pub_key_offset == 0) {
1582 existing_header.pub_key_offset = kdp_core_header->pub_key_offset;
1583 existing_header.pub_key_length = kdp_core_header->pub_key_length;
1584 }
1585 }
1586
1587 ret = access_data(access_context, TRUE, 0, sizeof(existing_header), &existing_header);
1588 if (ret != KERN_SUCCESS) {
1589 printf("kdp_core_handle_new_encryption_key failed to update the existing corefile header. Error 0x%x\n", ret);
1590 break;
1591 }
1592 } else {
1593 ret = access_data(access_context, TRUE, 0, sizeof(struct mach_core_fileheader_v2), kdp_core_header);
1594 if (ret != KERN_SUCCESS) {
1595 printf("kdp_core_handle_new_encryption_key failed to write the corefile header. Error 0x%x\n", ret);
1596 break;
1597 }
1598 }
1599
1600 if (kdp_core_header->pub_key_length) {
1601 uint64_t offset = used_existing_header ? existing_header.pub_key_offset : kdp_core_header->pub_key_offset;
1602 ret = access_data(access_context, TRUE, offset + PUBLIC_KEY_RESERVED_LENGTH, kdp_core_header->pub_key_length, kdp_core_public_key);
1603 if (ret != KERN_SUCCESS) {
1604 printf("kdp_core_handle_new_encryption_key failed to write the next public key. Error 0x%x\n", ret);
1605 break;
1606 }
1607
1608 if (!used_existing_header) {
1609 // Everything that happens here is optional. It's not the end of the world if this stuff fails, so we don't return
1610 // any errors
1611 // Since we're writing out a completely new header, we make sure to zero-out the region that's reserved for the public key.
1612 // This allows us consumers of the corefile to know for sure that this corefile is not encrypted (yet). Once we actually
1613 // write out a corefile, we'll overwrite this region with the key that we ended up using at the time.
1614 // If we fail to zero-out this region, consumers would read garbage data and properly fail to interpret it as a public key,
1615 // which is why it is OK for us to fail here (it's hard to interpret garbage data as a valid key, and even then, they wouldn't
1616 // find a matching private key anyway)
1617 void *empty_key = NULL;
1618 kern_return_t temp_ret = KERN_SUCCESS;
1619
1620 empty_key = kalloc_data(PUBLIC_KEY_RESERVED_LENGTH,
1621 Z_WAITOK | Z_ZERO | Z_NOFAIL);
1622
1623 temp_ret = access_data(access_context, TRUE, offset, PUBLIC_KEY_RESERVED_LENGTH, empty_key);
1624 kfree_data(empty_key, PUBLIC_KEY_RESERVED_LENGTH);
1625
1626 if (temp_ret != KERN_SUCCESS) {
1627 printf("kdp_core_handle_new_encryption_key failed to zero-out the public key region. Error 0x%x\n", temp_ret);
1628 break;
1629 }
1630 }
1631 }
1632 } while (0);
1633
1634 kdp_core_is_initializing_encryption_stage = false;
1635 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1636
1637 return ret;
1638 }
1639
1640 kern_return_t
kdp_core_handle_encryption_available(void)1641 kdp_core_handle_encryption_available(void)
1642 {
1643 kern_return_t ret;
1644
1645 lck_mtx_lock(kdp_core_encryption_stage_lock);
1646 kdp_core_is_initializing_encryption_stage = true;
1647
1648 ret = kdp_core_init_encryption_stage(kdp_core_public_key, kdp_core_header->pub_key_length);
1649
1650 kdp_core_is_initializing_encryption_stage = false;
1651 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1652
1653 return ret;
1654 }
1655
1656 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1657
1658 kern_return_t
kdp_core_handle_lz4_available(void)1659 kdp_core_handle_lz4_available(void)
1660 {
1661 kern_return_t ret;
1662 lck_mtx_lock(kdp_core_lz4_stage_lock);
1663 kdp_core_is_initializing_lz4_stage = true;
1664
1665 ret = lz4_stage_initialize(&lz4_output_stage);
1666
1667 kdp_core_is_initializing_lz4_stage = false;
1668 lck_mtx_unlock(kdp_core_lz4_stage_lock);
1669
1670 return ret;
1671 }
1672
1673 kern_return_t
kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data,void * access_context,__unused void * recipient_context)1674 kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, __unused void *recipient_context)
1675 {
1676 kern_return_t ret = KERN_SUCCESS;
1677
1678 lck_mtx_lock(kdp_core_disk_stage_lock);
1679 kdp_core_is_initializing_disk_stage = true;
1680
1681 ret = disk_stage_initialize(&disk_output_stage);
1682
1683 kdp_core_is_initializing_disk_stage = false;
1684 lck_mtx_unlock(kdp_core_disk_stage_lock);
1685
1686 if (KERN_SUCCESS != ret) {
1687 return ret;
1688 }
1689
1690 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1691 // If someone has already provided a new public key,
1692 // there's no sense in reading the old one from the corefile.
1693 if (kdp_core_public_key != NULL) {
1694 return KERN_SUCCESS;
1695 }
1696
1697 // The kernel corefile is now available. Let's try to retrieve the public key from its
1698 // header (if available and supported).
1699
1700 // First let's read the corefile header itself
1701 struct mach_core_fileheader_v2 temp_header = {};
1702 ret = access_data(access_context, FALSE, 0, sizeof(temp_header), &temp_header);
1703 if (KERN_SUCCESS != ret) {
1704 printf("kdp_core_polled_io_polled_file_available failed to read corefile header. Error 0x%x\n", ret);
1705 return ret;
1706 }
1707
1708 // Check if the corefile header is initialized, and whether it's initialized to values that we support
1709 // (for backwards and forwards) compatibility, and check whether the header indicates that the corefile has
1710 // has a public key stashed inside of it.
1711 if (temp_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1712 && temp_header.version == 2
1713 && temp_header.pub_key_offset != 0
1714 && temp_header.pub_key_length != 0
1715 /* Future-proofing: make sure it's the key format that we support */
1716 && (temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK) == MACH_CORE_FILEHEADER_V2_FLAG_NEXT_COREFILE_KEY_FORMAT_NIST_P256
1717 /* Add some extra sanity checks. These are not necessary */
1718 && temp_header.pub_key_length <= 4096
1719 && temp_header.pub_key_offset < 65535) {
1720 // The corefile header is properly initialized, is supported, and contains a public key.
1721 // Let's adopt that public key for our encryption needs
1722 void *public_key = NULL;
1723
1724 public_key = kalloc_data(temp_header.pub_key_length,
1725 Z_ZERO | Z_WAITOK | Z_NOFAIL);
1726
1727 // Read the public key from the corefile. Note that the key we're trying to adopt is the "next" key, which is
1728 // PUBLIC_KEY_RESERVED_LENGTH bytes after the public key.
1729 ret = access_data(access_context, FALSE, temp_header.pub_key_offset + PUBLIC_KEY_RESERVED_LENGTH, temp_header.pub_key_length, public_key);
1730 if (KERN_SUCCESS != ret) {
1731 printf("kdp_core_polled_io_polled_file_available failed to read the public key. Error 0x%x\n", ret);
1732 kfree_data(public_key, temp_header.pub_key_length);
1733 return ret;
1734 }
1735
1736 lck_mtx_lock(kdp_core_encryption_stage_lock);
1737 kdp_core_is_initializing_encryption_stage = true;
1738
1739 ret = kdp_core_init_encryption_stage(public_key, temp_header.pub_key_length);
1740 if (KERN_SUCCESS == ret) {
1741 kdp_core_header->flags |= temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1742 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(temp_header.flags);
1743 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1744 kdp_core_header->pub_key_length = temp_header.pub_key_length;
1745 kdp_core_public_key = public_key;
1746 }
1747
1748 kdp_core_is_initializing_encryption_stage = false;
1749 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1750 }
1751 #else
1752 #pragma unused(access_data, access_context)
1753 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1754
1755 return ret;
1756 }
1757
1758 kern_return_t
kdp_core_polled_io_polled_file_unavailable(void)1759 kdp_core_polled_io_polled_file_unavailable(void)
1760 {
1761 lck_mtx_lock(kdp_core_disk_stage_lock);
1762 kdp_core_is_initializing_disk_stage = true;
1763
1764 if (disk_output_stage.kos_initialized && disk_output_stage.kos_funcs.kosf_free) {
1765 disk_output_stage.kos_funcs.kosf_free(&disk_output_stage);
1766 }
1767
1768 kdp_core_is_initializing_disk_stage = false;
1769 lck_mtx_unlock(kdp_core_disk_stage_lock);
1770
1771 return KERN_SUCCESS;
1772 }
1773
1774 void
kdp_core_init(void)1775 kdp_core_init(void)
1776 {
1777 kern_return_t kr;
1778 kern_coredump_callback_config core_config = { };
1779
1780 /* Initialize output stages */
1781 kr = kdp_core_init_output_stages();
1782 assert(KERN_SUCCESS == kr);
1783
1784 kmem_alloc(kernel_map, (vm_offset_t*)&kdp_core_header,
1785 kdp_core_header_size,
1786 KMA_NOFAIL | KMA_ZERO | KMA_PERMANENT | KMA_KOBJECT | KMA_DATA,
1787 VM_KERN_MEMORY_DIAG);
1788
1789 kdp_core_header->signature = MACH_CORE_FILEHEADER_V2_SIGNATURE;
1790 kdp_core_header->version = 2;
1791
1792 kdp_core_initialization_lock_group = lck_grp_alloc_init("KDPCoreStageInit", LCK_GRP_ATTR_NULL);
1793 kdp_core_disk_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1794
1795 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1796 kdp_core_encryption_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1797
1798 (void) kern_dump_should_enforce_encryption();
1799 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1800
1801 kdp_core_lz4_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1802
1803 core_config.kcc_coredump_init = kern_dump_init;
1804 core_config.kcc_coredump_get_summary = kern_dump_save_summary;
1805 core_config.kcc_coredump_save_segment_descriptions = kern_dump_save_seg_descriptions;
1806 core_config.kcc_coredump_save_thread_state = kern_dump_save_thread_state;
1807 core_config.kcc_coredump_save_sw_vers_detail = kern_dump_save_sw_vers_detail;
1808 core_config.kcc_coredump_save_segment_data = kern_dump_save_segment_data;
1809 core_config.kcc_coredump_save_note_summary = kern_dump_save_note_summary;
1810 core_config.kcc_coredump_save_note_descriptions = kern_dump_save_note_descriptions;
1811 core_config.kcc_coredump_save_note_data = kern_dump_save_note_data;
1812
1813 kr = kern_register_xnu_coredump_helper(&core_config);
1814 assert(KERN_SUCCESS == kr);
1815 }
1816
1817 /*
1818 * Additional LC_NOTES added to the core.
1819 */
1820
1821 static kern_return_t
kern_dump_save_note_summary(void * refcon __unused,core_save_note_summary_cb callback,void * context)1822 kern_dump_save_note_summary(void *refcon __unused, core_save_note_summary_cb callback, void *context)
1823 {
1824 int count = 1;
1825 size_t size = sizeof(addrable_bits_note_t);
1826
1827 #ifdef CONFIG_SPTM
1828 /* Load binary spec note */
1829
1830 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1831
1832 if (debug_header != NULL &&
1833 debug_header->magic == DEBUG_HEADER_MAGIC_VAL &&
1834 debug_header->version == DEBUG_HEADER_CURRENT_VERSION) {
1835 /* Also add SPTM, TXM, and xnu kc load binary specs if present */
1836 count += debug_header->count;
1837 size += debug_header->count * sizeof(load_binary_spec_note_t);
1838 }
1839 #endif /* CONFIG_SPTM */
1840
1841 return callback(count, size, context);
1842 }
1843
1844 static kern_return_t
kern_dump_save_note_descriptions(void * refcon __unused,core_save_note_descriptions_cb callback,void * context)1845 kern_dump_save_note_descriptions(void *refcon __unused, core_save_note_descriptions_cb callback, void *context)
1846 {
1847 int max_ret = KERN_SUCCESS;
1848 int ret;
1849
1850 max_ret = ret = callback(ADDRABLE_BITS_DATA_OWNER, sizeof(addrable_bits_note_t), context);
1851
1852 #if CONFIG_SPTM
1853 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1854
1855 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1856 ret = callback(LOAD_BINARY_SPEC_DATA_OWNER, sizeof(load_binary_spec_note_t), context);
1857 max_ret = MAX(ret, max_ret);
1858 }
1859 #endif /* CONFIG_SPTM */
1860
1861 return max_ret;
1862 }
1863
1864 static kern_return_t
kern_dump_save_note_data(void * refcon __unused,core_save_note_data_cb callback,void * context)1865 kern_dump_save_note_data(void *refcon __unused, core_save_note_data_cb callback, void *context)
1866 {
1867 int max_ret = KERN_SUCCESS;
1868 int ret;
1869
1870 addrable_bits_note_t note = {
1871 .version = ADDRABLE_BITS_VER,
1872 .addressing_bits = pmap_kernel_va_bits(),
1873 .unused = 0
1874 };
1875
1876 max_ret = ret = callback(¬e, sizeof(addrable_bits_note_t), context);
1877
1878 #if CONFIG_SPTM
1879 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1880
1881 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1882 load_binary_spec_note_t load_binary_spec = {
1883 .version = LOAD_BINARY_SPEC_VERSION,
1884 .uuid = {0},
1885 .address = (uint64_t)debug_header->image[i],
1886 .slide = UINT64_MAX // unknown, load address specified
1887 };
1888
1889 char const *name;
1890 switch (i) {
1891 case DEBUG_HEADER_ENTRY_SPTM:
1892 name = "sptm";
1893 break;
1894 case DEBUG_HEADER_ENTRY_XNU:
1895 name = "xnu";
1896 break;
1897 case DEBUG_HEADER_ENTRY_TXM:
1898 name = "txm";
1899 break;
1900 default:
1901 name = "UNKNOWN";
1902 kern_coredump_log(context, "%s(): encountered unknown debug header entry %d, "
1903 "including anyway with name '%s'\n", __func__, i, name);
1904 }
1905
1906 strlcpy(load_binary_spec.name_cstring, name, LOAD_BINARY_NAME_BUF_SIZE);
1907
1908 ret = callback(&load_binary_spec, sizeof(load_binary_spec), context);
1909
1910 if (ret != KERN_SUCCESS) {
1911 kern_coredump_log(context, "%s(): failed to write load binary spec structure "
1912 "for binary #%d ('%s'): callback returned 0x%x\n",
1913 __func__, i, name, ret);
1914 max_ret = MAX(ret, max_ret);
1915 }
1916 }
1917 #endif /* CONFIG_SPTM */
1918
1919 return max_ret;
1920 }
1921
1922 #else
1923
1924 void
kdp_core_exclude_region(__unused vm_offset_t addr,__unused vm_size_t size)1925 kdp_core_exclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
1926 {
1927 }
1928
1929 void
kdp_core_unexclude_region(__unused vm_offset_t addr,__unused vm_size_t size)1930 kdp_core_unexclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
1931 {
1932 }
1933
1934 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1935