1 /*
2 * Copyright (c) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * The main orchestrator for kernel (and co-processor) coredumps. Here's a very simplistic view of
31 * the flow:
32 *
33 * At kernel initialization time (kdp_core_init):
34 * ----------------------------------------------
35 *
36 * - kdp_core_init() takes care of allocating all necessary data structures and initializes the
37 * coredump output stages
38 *
39 * At coredump time (do_kern_dump):
40 * --------------------------------
41 *
42 * - Depending on the coredump variant, we chain the necessary output stages together in chain_output_stages()
43 * - [Disk only] We initialize the corefile header
44 * - [Disk only] We stream the stackshot out through the output stages and update the corefile header
45 * - We perform the kernel coredump, streaming it out through the output stages
46 * - [Disk only] We update the corefile header
47 * - [Disk only] We perform the co-processor coredumps (driven by kern_do_coredump), streaming each out
48 * through the output stages and updating the corefile header.
49 * - [Disk only] We save the coredump log to the corefile
50 */
51
52 #include <mach/kern_return.h>
53 #include <mach/vm_types.h>
54 #include <kdp/core_exclude.h>
55 #include <kdp/kdp_core.h>
56 #include <kdp/core_notes.h>
57
58 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
59
60 #include <mach/mach_types.h>
61 #include <mach/vm_attributes.h>
62 #include <mach/vm_param.h>
63 #include <mach/vm_map.h>
64 #include <vm/vm_protos.h>
65 #include <vm/vm_kern_xnu.h>
66 #include <vm/vm_map.h>
67 #include <machine/cpu_capabilities.h>
68 #include <libsa/types.h>
69 #include <libkern/kernel_mach_header.h>
70 #include <kern/locks.h>
71 #include <kdp/kdp_internal.h>
72 #include <kdp/output_stages/output_stages.h>
73 #include <kdp/processor_core.h>
74 #include <IOKit/IOTypes.h>
75 #include <IOKit/IOBSD.h>
76 #include <sys/errno.h>
77 #include <sys/msgbuf.h>
78 #include <san/kasan.h>
79 #include <kern/debug.h>
80 #include <pexpert/pexpert.h>
81 #include <os/atomic_private.h>
82
83 #if CONFIG_SPTM
84 #include <sptm/debug_header.h>
85 #endif
86
87 #if defined(__x86_64__)
88 #include <i386/pmap_internal.h>
89 #include <kdp/ml/i386/kdp_x86_common.h>
90 #include <kern/debug.h>
91 #endif /* defined(__x86_64__) */
92
93 #if CONFIG_SPTM
94 #include <arm64/sptm/sptm.h>
95 #endif /* CONFIG_SPTM */
96
97 kern_return_t kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context);
98 kern_return_t kdp_core_polled_io_polled_file_unavailable(void);
99
100 typedef int (*pmap_traverse_callback)(vm_map_offset_t start,
101 vm_map_offset_t end,
102 void *context);
103
104 static kern_return_t kern_dump_init(void *refcon, void *context);
105 static int kern_dump_save_summary(void *refcon, core_save_summary_cb callback, void *context);
106 static int kern_dump_save_seg_descriptions(void *refcon, core_save_segment_descriptions_cb callback, void *context);
107 static int kern_dump_save_thread_state(void *refcon, void *buf, core_save_thread_state_cb callback, void *context);
108 static int kern_dump_save_sw_vers_detail(void *refcon, core_save_sw_vers_detail_cb callback, void *context);
109 static int kern_dump_save_segment_data(void *refcon, core_save_segment_data_cb callback, void *context);
110 static kern_return_t kern_dump_save_note_summary(void *refcon, core_save_note_summary_cb callback, void *context);
111 static kern_return_t kern_dump_save_note_descriptions(void *refcon, core_save_note_descriptions_cb callback, void *context);
112 static kern_return_t kern_dump_save_note_data(void *refcon, core_save_note_data_cb callback, void *context);
113
114 static int
115 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
116 vm_map_offset_t end,
117 void *context);
118 static int
119 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
120 vm_map_offset_t end,
121 void *context);
122
123 static int
124 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
125 vm_map_offset_t end,
126 void *context);
127
128 static struct kdp_output_stage disk_output_stage = {};
129 static struct kdp_output_stage lz4_output_stage = {};
130 static struct kdp_output_stage zlib_output_stage = {};
131 static struct kdp_output_stage buffer_output_stage = {};
132 static struct kdp_output_stage net_output_stage = {};
133 static struct kdp_output_stage progress_notify_output_stage = {};
134 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
135 static struct kdp_output_stage aea_output_stage = {};
136 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
137 #if defined(__arm64__)
138 static struct kdp_output_stage shmem_output_stage = {};
139 static struct kdp_output_stage memory_backing_aware_buffer_output_stage = {};
140 #endif /* defined(__arm64__) */
141
142 extern uint32_t kdp_crashdump_pkt_size;
143
144 static boolean_t kern_dump_successful = FALSE;
145
146 static const size_t kdp_core_header_size = sizeof(struct mach_core_fileheader_v2) + (KERN_COREDUMP_MAX_CORES * sizeof(struct mach_core_details_v2));
147 static struct mach_core_fileheader_v2 *kdp_core_header = NULL;
148
149 static lck_grp_t *kdp_core_initialization_lock_group = NULL;
150 static lck_mtx_t *kdp_core_disk_stage_lock = NULL;
151 static bool kdp_core_is_initializing_disk_stage = false;
152
153 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
154 static const size_t PUBLIC_KEY_RESERVED_LENGTH = roundup(4096, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
155 static void *kdp_core_public_key = NULL;
156 static lck_mtx_t *kdp_core_encryption_stage_lock = NULL;
157 static bool kdp_core_is_initializing_encryption_stage = false;
158 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
159
160 static lck_mtx_t *kdp_core_lz4_stage_lock = NULL;
161 static bool kdp_core_is_initializing_lz4_stage = false;
162
163 /*
164 * These variables will be modified by the BSD layer if the root device is
165 * a RAMDisk.
166 */
167 uint64_t kdp_core_ramdisk_addr = 0;
168 uint64_t kdp_core_ramdisk_size = 0;
169
170 #define COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY (1 << 0)
171 #define COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT (1 << 1)
172
173 boolean_t
kdp_has_polled_corefile(void)174 kdp_has_polled_corefile(void)
175 {
176 return NULL != gIOPolledCoreFileVars;
177 }
178
179 kern_return_t
kdp_polled_corefile_error(void)180 kdp_polled_corefile_error(void)
181 {
182 return gIOPolledCoreFileOpenRet;
183 }
184
185 IOPolledCoreFileMode_t
kdp_polled_corefile_mode(void)186 kdp_polled_corefile_mode(void)
187 {
188 return gIOPolledCoreFileMode;
189 }
190
191 struct kdp_core_excluded_region {
192 struct kdp_core_excluded_region *next;
193 vm_offset_t addr;
194 vm_size_t size;
195 };
196
197 static LCK_GRP_DECLARE(excluded_regions_grp, "kdp-exclude-regions");
198 static LCK_MTX_DECLARE(excluded_regions_mtx, &excluded_regions_grp);
199 static struct kdp_core_excluded_region *excluded_regions;
200
201 void
kdp_core_exclude_region(vm_offset_t addr,vm_size_t size)202 kdp_core_exclude_region(vm_offset_t addr, vm_size_t size)
203 {
204 struct kdp_core_excluded_region *region;
205
206 if (addr >= addr + size) {
207 panic("%s: cannot exclude region starting at %p with size %zu (zero or overflowing size)",
208 __func__, (void*)addr, (size_t)size);
209 }
210 if (addr != round_page(addr) || size != round_page(size)) {
211 panic("%s: cannot exclude region starting at %p with size %zu (not page aligned)",
212 __func__, (void*)addr, (size_t)size);
213 }
214
215 region = kalloc_type(typeof(*region), Z_WAITOK | Z_NOFAIL);
216 region->addr = addr;
217 region->size = size;
218
219 lck_mtx_lock(&excluded_regions_mtx);
220 region->next = excluded_regions;
221 excluded_regions = region;
222 lck_mtx_unlock(&excluded_regions_mtx);
223 }
224
225 void
kdp_core_unexclude_region(vm_offset_t addr,vm_size_t size)226 kdp_core_unexclude_region(vm_offset_t addr, vm_size_t size)
227 {
228 struct kdp_core_excluded_region *region;
229 struct kdp_core_excluded_region **fixup = &excluded_regions;
230
231 lck_mtx_lock(&excluded_regions_mtx);
232 for (region = excluded_regions; region; region = region->next) {
233 if (region->addr == addr && region->size == size) {
234 *fixup = region->next;
235 break;
236 }
237 fixup = ®ion->next;
238 }
239 if (!region) {
240 panic("%s: cannot unexclude region starting at %p with size %zu (not currently excluded)",
241 __func__, (void*)addr, (size_t)size);
242 }
243 lck_mtx_unlock(&excluded_regions_mtx);
244
245 // We had exclusive access to the list when we removed the region, and it is no longer
246 // reachable from the list, so it is safe to free.
247 kfree_type(typeof(*region), region);
248 }
249
250 static bool
kernel_vaddr_in_excluded_region(vm_offset_t addr,uint64_t * vincr)251 kernel_vaddr_in_excluded_region(vm_offset_t addr, uint64_t *vincr)
252 {
253 struct kdp_core_excluded_region *region;
254
255 // We check this earlier before attempting to dump the kernel, but verify here.
256 assert(!kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx));
257
258 for (region = excluded_regions; region; region = region->next) {
259 if (region->addr <= addr && addr < (region->addr + region->size)) {
260 *vincr = region->size;
261 return true;
262 }
263 }
264
265 return false;
266 }
267
268 kern_return_t
kdp_core_output(void * kdp_core_out_state,uint64_t length,void * data)269 kdp_core_output(void *kdp_core_out_state, uint64_t length, void * data)
270 {
271 kern_return_t err = KERN_SUCCESS;
272 uint64_t percent;
273 struct kdp_core_out_state *vars = (struct kdp_core_out_state *)kdp_core_out_state;
274 struct kdp_output_stage *first_stage = STAILQ_FIRST(&vars->kcos_out_stage);
275
276 if (vars->kcos_error == KERN_SUCCESS) {
277 #if DEVELOPMENT || DEBUG
278 // panic testing: force the write to fail after X number of writes
279 if ((panic_test_case & PANIC_TEST_CASE_COREFILE_IO_ERR) && (--panic_test_action_count == 0)) {
280 panic_test_case &= ~PANIC_TEST_CASE_COREFILE_IO_ERR;
281 length = -1;
282 }
283 #endif
284
285 if ((err = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, length, data)) != KERN_SUCCESS) {
286 kern_coredump_log(NULL, "(kdp_core_output) outproc(KDP_DATA, NULL, 0x%llx, %p) returned 0x%x\n",
287 length, data, err);
288 vars->kcos_error = err;
289 }
290 if (!data && !length) {
291 kern_coredump_log(NULL, "100..");
292 } else {
293 vars->kcos_bytes_written += length;
294 percent = (vars->kcos_bytes_written * 100) / vars->kcos_totalbytes;
295 if ((percent - vars->kcos_lastpercent) >= 10) {
296 vars->kcos_lastpercent = percent;
297 kern_coredump_log(NULL, "%lld..\n", percent);
298 }
299 }
300 }
301 return err;
302 }
303
304 #if defined(__arm64__)
305 extern pmap_paddr_t avail_start, avail_end;
306 extern struct vm_object pmap_object_store;
307 #endif
308 extern vm_offset_t c_buffers;
309 extern vm_size_t c_buffers_size;
310
311 static bool
kernel_vaddr_in_coredump_stage(const struct kdp_output_stage * stage,uint64_t vaddr,uint64_t * vincr)312 kernel_vaddr_in_coredump_stage(const struct kdp_output_stage *stage, uint64_t vaddr, uint64_t *vincr)
313 {
314 uint64_t start_addr = (uint64_t)stage->kos_data;
315 uint64_t end_addr = start_addr + stage->kos_data_size;
316
317 if (!stage->kos_data) {
318 return false;
319 }
320
321 if (vaddr >= start_addr && vaddr < end_addr) {
322 *vincr = stage->kos_data_size - (vaddr - start_addr);
323 return true;
324 }
325
326 return false;
327 }
328
329 static bool
kernel_vaddr_in_coredump_stages(uint64_t vaddr,uint64_t * vincr)330 kernel_vaddr_in_coredump_stages(uint64_t vaddr, uint64_t *vincr)
331 {
332 if (kernel_vaddr_in_coredump_stage(&disk_output_stage, vaddr, vincr)) {
333 return true;
334 }
335
336 if (kernel_vaddr_in_coredump_stage(&lz4_output_stage, vaddr, vincr)) {
337 return true;
338 }
339
340 if (kernel_vaddr_in_coredump_stage(&zlib_output_stage, vaddr, vincr)) {
341 return true;
342 }
343
344 if (kernel_vaddr_in_coredump_stage(&buffer_output_stage, vaddr, vincr)) {
345 return true;
346 }
347
348 if (kernel_vaddr_in_coredump_stage(&net_output_stage, vaddr, vincr)) {
349 return true;
350 }
351
352 if (kernel_vaddr_in_coredump_stage(&progress_notify_output_stage, vaddr, vincr)) {
353 return true;
354 }
355
356 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
357 if (kernel_vaddr_in_coredump_stage(&aea_output_stage, vaddr, vincr)) {
358 return true;
359 }
360 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
361
362 #if defined(__arm64__)
363 if (kernel_vaddr_in_coredump_stage(&shmem_output_stage, vaddr, vincr)) {
364 return true;
365 }
366 #endif /* defined(__arm64__) */
367
368 #if defined(__arm64__)
369 if (kernel_vaddr_in_coredump_stage(&memory_backing_aware_buffer_output_stage, vaddr, vincr)) {
370 return true;
371 }
372 #endif /* defined(__arm64__) */
373
374 return false;
375 }
376
377
378 ppnum_t
kernel_pmap_present_mapping(uint64_t vaddr,uint64_t * pvincr,uintptr_t * pvphysaddr)379 kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphysaddr)
380 {
381 ppnum_t ppn = 0;
382 uint64_t vincr = PAGE_SIZE_64;
383
384 assert(!(vaddr & PAGE_MASK_64));
385
386 /* VA ranges to exclude */
387 if (vaddr == c_buffers) {
388 /* compressor data */
389 ppn = 0;
390 vincr = c_buffers_size;
391 } else if (kernel_vaddr_in_coredump_stages(vaddr, &vincr)) {
392 /* coredump output stage working memory */
393 ppn = 0;
394 } else if ((kdp_core_ramdisk_addr != 0) && (vaddr == kdp_core_ramdisk_addr)) {
395 ppn = 0;
396 vincr = kdp_core_ramdisk_size;
397 } else
398 #if defined(__arm64__)
399 if (vaddr == phystokv(avail_start)) {
400 /* physical memory map */
401 ppn = 0;
402 vincr = (avail_end - avail_start);
403 } else
404 #endif /* defined(__arm64__) */
405 {
406 ppn = (pvphysaddr != NULL ?
407 pmap_find_phys(kernel_pmap, vaddr) :
408 pmap_find_phys_nofault(kernel_pmap, vaddr));
409 }
410
411 *pvincr = round_page_64(vincr);
412
413 if (ppn && pvphysaddr) {
414 uint64_t phys = ptoa_64(ppn);
415 if (physmap_enclosed(phys)) {
416 *pvphysaddr = phystokv(phys);
417 } else {
418 ppn = 0;
419 }
420 }
421
422 return ppn;
423 }
424
425 static int
pmap_traverse_present_mappings(pmap_t __unused pmap,vm_map_offset_t start,vm_map_offset_t end,pmap_traverse_callback callback,void * context)426 pmap_traverse_present_mappings(pmap_t __unused pmap,
427 vm_map_offset_t start,
428 vm_map_offset_t end,
429 pmap_traverse_callback callback,
430 void *context)
431 {
432 IOReturn ret;
433 vm_map_offset_t vcurstart, vcur;
434 uint64_t vincr = 0;
435 vm_map_offset_t debug_start = trunc_page((vm_map_offset_t) debug_buf_base);
436 vm_map_offset_t debug_end = round_page((vm_map_offset_t) (debug_buf_base + debug_buf_size));
437 #if defined(XNU_TARGET_OS_BRIDGE)
438 vm_map_offset_t macos_panic_start = trunc_page((vm_map_offset_t) macos_panic_base);
439 vm_map_offset_t macos_panic_end = round_page((vm_map_offset_t) (macos_panic_base + macos_panic_size));
440 #endif
441
442 boolean_t lastvavalid;
443 #if defined(__arm64__)
444 vm_page_t m = VM_PAGE_NULL;
445 #endif
446
447 #if defined(__x86_64__)
448 assert(!is_ept_pmap(pmap));
449 #endif
450
451 /* Assumes pmap is locked, or being called from the kernel debugger */
452 if (start > end) {
453 return KERN_INVALID_ARGUMENT;
454 }
455
456 ret = KERN_SUCCESS;
457 lastvavalid = FALSE;
458 for (vcur = vcurstart = start; (ret == KERN_SUCCESS) && (vcur < end);) {
459 ppnum_t ppn = 0;
460
461 #if defined(__arm64__)
462 /* We're at the start of the physmap, so pull out the pagetable pages that
463 * are accessed through that region.*/
464 if (vcur == phystokv(avail_start) && vm_object_lock_try_shared(&pmap_object_store)) {
465 m = (vm_page_t)vm_page_queue_first(&pmap_object_store.memq);
466 }
467
468 if (m != VM_PAGE_NULL) {
469 vm_map_offset_t vprev = vcur;
470 ppn = (ppnum_t)atop(avail_end);
471 while (!vm_page_queue_end(&pmap_object_store.memq, (vm_page_queue_entry_t)m)) {
472 /* Ignore pages that come from the static region and have already been dumped.*/
473 if (VM_PAGE_GET_PHYS_PAGE(m) >= atop(avail_start)) {
474 ppn = VM_PAGE_GET_PHYS_PAGE(m);
475 break;
476 }
477 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
478 }
479 vincr = PAGE_SIZE_64;
480 if (ppn == atop(avail_end)) {
481 vm_object_unlock(&pmap_object_store);
482 m = VM_PAGE_NULL;
483 // avail_end is not a valid physical address,
484 // so phystokv(avail_end) may not produce the expected result.
485 vcur = phystokv(avail_start) + (avail_end - avail_start);
486 } else {
487 m = (vm_page_t)vm_page_queue_next(&m->vmp_listq);
488 vcur = phystokv(ptoa(ppn));
489 }
490 if (vcur != vprev) {
491 ret = callback(vcurstart, vprev, context);
492 lastvavalid = FALSE;
493 }
494 }
495 if (m == VM_PAGE_NULL) {
496 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
497 }
498 #else /* defined(__arm64__) */
499 ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL);
500 #endif
501 if (ppn != 0 && kernel_vaddr_in_excluded_region(vcur, &vincr)) {
502 /* excluded region */
503 ppn = 0;
504 }
505 if (ppn != 0) {
506 if (((vcur < debug_start) || (vcur >= debug_end))
507 && !(
508 pmap_valid_page(ppn)
509 || bootloader_valid_page(ppn)
510 )
511 #if defined(XNU_TARGET_OS_BRIDGE)
512 // include the macOS panic region if it's mapped
513 && ((vcur < macos_panic_start) || (vcur >= macos_panic_end))
514 #endif /* defined(XNU_TARGET_OS_BRIDGE) */
515 ) {
516 /* not something we want */
517 ppn = 0;
518 }
519 /* include the phys carveout only if explictly marked */
520 if (debug_is_in_phys_carveout(vcur) &&
521 !debug_can_coredump_phys_carveout()) {
522 ppn = 0;
523 }
524 }
525
526 if (ppn != 0) {
527 if (!lastvavalid) {
528 /* Start of a new virtual region */
529 vcurstart = vcur;
530 lastvavalid = TRUE;
531 }
532 } else {
533 if (lastvavalid) {
534 /* end of a virtual region */
535 ret = callback(vcurstart, vcur, context);
536 lastvavalid = FALSE;
537 }
538
539 #if defined(__x86_64__)
540 /* Try to skip by 2MB if possible */
541 if ((vcur & PDMASK) == 0) {
542 pd_entry_t *pde;
543 pde = pmap_pde(pmap, vcur);
544 if (0 == pde || ((*pde & INTEL_PTE_VALID) == 0)) {
545 /* Make sure we wouldn't overflow */
546 if (vcur < (end - NBPD)) {
547 vincr = NBPD;
548 }
549 }
550 }
551 #endif /* defined(__x86_64__) */
552 }
553 vcur += vincr;
554 }
555
556 if ((ret == KERN_SUCCESS) && lastvavalid) {
557 /* send previous run */
558 ret = callback(vcurstart, vcur, context);
559 }
560
561 #if KASAN
562 if (ret == KERN_SUCCESS) {
563 ret = kasan_traverse_mappings(callback, context);
564 }
565 #endif
566
567
568 return ret;
569 }
570
571 struct kern_dump_preflight_context {
572 uint32_t region_count;
573 uint64_t dumpable_bytes;
574 };
575
576 int
kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)577 kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start,
578 vm_map_offset_t end,
579 void *context)
580 {
581 struct kern_dump_preflight_context *kdc = (struct kern_dump_preflight_context *)context;
582 IOReturn ret = KERN_SUCCESS;
583
584 kdc->region_count++;
585 kdc->dumpable_bytes += (end - start);
586
587 return ret;
588 }
589
590
591 struct kern_dump_send_seg_desc_context {
592 core_save_segment_descriptions_cb callback;
593 void *context;
594 };
595
596 int
kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)597 kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start,
598 vm_map_offset_t end,
599 void *context)
600 {
601 struct kern_dump_send_seg_desc_context *kds_context = (struct kern_dump_send_seg_desc_context *)context;
602 uint64_t seg_start = (uint64_t) start;
603 uint64_t seg_end = (uint64_t) end;
604
605 return kds_context->callback(seg_start, seg_end, kds_context->context);
606 }
607
608 struct kern_dump_send_segdata_context {
609 core_save_segment_data_cb callback;
610 void *context;
611 };
612
613 int
kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,vm_map_offset_t end,void * context)614 kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start,
615 vm_map_offset_t end,
616 void *context)
617 {
618 struct kern_dump_send_segdata_context *kds_context = (struct kern_dump_send_segdata_context *)context;
619
620 return kds_context->callback((void *)start, (uint64_t)(end - start), kds_context->context);
621 }
622
623 static kern_return_t
kern_dump_init(__unused void * refcon,void * context)624 kern_dump_init(__unused void *refcon, void *context)
625 {
626 /* TODO: consider doing mmu flush from an init function */
627
628 // If excluded regions list is locked, it is unsafe to dump the kernel.
629 if (kdp_lck_mtx_lock_spin_is_acquired(&excluded_regions_mtx)) {
630 kern_coredump_log(context, "%s: skipping kernel because excluded regions list is locked\n",
631 __func__);
632 #if defined(__arm64__)
633 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
634 #else
635 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_KERNEL_COREDUMP_SKIPPED_EXCLUDE_REGIONS_UNAVAILABLE;
636 #endif
637 paniclog_flush();
638 return KERN_NODE_DOWN;
639 }
640
641 return KERN_SUCCESS;
642 }
643
644 static int
kern_dump_save_summary(__unused void * refcon,core_save_summary_cb callback,void * context)645 kern_dump_save_summary(__unused void *refcon, core_save_summary_cb callback, void *context)
646 {
647 struct kern_dump_preflight_context kdc_preflight = { };
648 uint64_t thread_state_size = 0, thread_count = 0;
649 vm_map_offset_t vstart = kdp_core_start_addr();
650 kern_return_t ret;
651
652 ret = pmap_traverse_present_mappings(kernel_pmap,
653 vstart,
654 VM_MAX_KERNEL_ADDRESS,
655 kern_dump_pmap_traverse_preflight_callback,
656 &kdc_preflight);
657 if (ret != KERN_SUCCESS) {
658 kern_coredump_log(context, "save_summary: pmap traversal failed: %d\n", ret);
659 return ret;
660 }
661
662 kern_collectth_state_size(&thread_count, &thread_state_size);
663
664 ret = callback(kdc_preflight.region_count, kdc_preflight.dumpable_bytes,
665 thread_count, thread_state_size, 0, context);
666 return ret;
667 }
668
669 static int
kern_dump_save_seg_descriptions(__unused void * refcon,core_save_segment_descriptions_cb callback,void * context)670 kern_dump_save_seg_descriptions(__unused void *refcon, core_save_segment_descriptions_cb callback, void *context)
671 {
672 vm_map_offset_t vstart = kdp_core_start_addr();
673 kern_return_t ret;
674 struct kern_dump_send_seg_desc_context kds_context;
675
676 kds_context.callback = callback;
677 kds_context.context = context;
678
679 ret = pmap_traverse_present_mappings(kernel_pmap,
680 vstart,
681 VM_MAX_KERNEL_ADDRESS,
682 kern_dump_pmap_traverse_send_segdesc_callback,
683 &kds_context);
684 if (ret != KERN_SUCCESS) {
685 kern_coredump_log(context, "save_seg_desc: pmap traversal failed: %d\n", ret);
686 return ret;
687 }
688
689 return KERN_SUCCESS;
690 }
691
692 static int
kern_dump_save_thread_state(__unused void * refcon,void * buf,core_save_thread_state_cb callback,void * context)693 kern_dump_save_thread_state(__unused void *refcon, void *buf, core_save_thread_state_cb callback, void *context)
694 {
695 kern_return_t ret;
696 uint64_t thread_state_size = 0, thread_count = 0;
697
698 kern_collectth_state_size(&thread_count, &thread_state_size);
699
700 if (thread_state_size > 0) {
701 void * iter = NULL;
702 do {
703 kern_collectth_state(current_thread(), buf, thread_state_size, &iter);
704
705 ret = callback(buf, context);
706 if (ret != KERN_SUCCESS) {
707 return ret;
708 }
709 } while (iter);
710 }
711
712 return KERN_SUCCESS;
713 }
714
715
716 static int
kern_dump_save_sw_vers_detail(__unused void * refcon,core_save_sw_vers_detail_cb callback,void * context)717 kern_dump_save_sw_vers_detail(__unused void *refcon, core_save_sw_vers_detail_cb callback, void *context)
718 {
719 return callback(vm_kernel_stext, kernel_uuid, 0, context);
720 }
721
722 static int
kern_dump_save_segment_data(__unused void * refcon,core_save_segment_data_cb callback,void * context)723 kern_dump_save_segment_data(__unused void *refcon, core_save_segment_data_cb callback, void *context)
724 {
725 vm_map_offset_t vstart = kdp_core_start_addr();
726 kern_return_t ret;
727 struct kern_dump_send_segdata_context kds_context;
728
729 kds_context.callback = callback;
730 kds_context.context = context;
731
732 ret = pmap_traverse_present_mappings(kernel_pmap,
733 vstart,
734 VM_MAX_KERNEL_ADDRESS, kern_dump_pmap_traverse_send_segdata_callback, &kds_context);
735 if (ret != KERN_SUCCESS) {
736 kern_coredump_log(context, "save_seg_data: pmap traversal failed: %d\n", ret);
737 return ret;
738 }
739
740 return KERN_SUCCESS;
741 }
742
743 kern_return_t
kdp_reset_output_vars(void * kdp_core_out_state,uint64_t totalbytes,bool encrypt_core,bool * out_should_skip_coredump,const char * corename,kern_coredump_type_t coretype)744 kdp_reset_output_vars(void *kdp_core_out_state, uint64_t totalbytes, bool encrypt_core, bool *out_should_skip_coredump,
745 const char *corename, kern_coredump_type_t coretype)
746 {
747 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
748 struct kdp_output_stage *current_stage = NULL;
749
750 /* Re-initialize kdp_outstate */
751 outstate->kcos_totalbytes = totalbytes;
752 outstate->kcos_bytes_written = 0;
753 outstate->kcos_lastpercent = 0;
754 outstate->kcos_error = KERN_SUCCESS;
755
756 /* Reset the output stages */
757 STAILQ_FOREACH(current_stage, &outstate->kcos_out_stage, kos_next) {
758 kern_return_t res = current_stage->kos_funcs.kosf_reset(current_stage, corename, coretype);
759
760 /* Skip coredump if requested by an output stage. */
761 if (res == KERN_NODE_DOWN) {
762 *out_should_skip_coredump = true;
763 return KERN_SUCCESS;
764 }
765
766 if (res != KERN_SUCCESS) {
767 return res;
768 }
769 }
770
771 *out_should_skip_coredump = false;
772 if (encrypt_core) {
773 if (outstate->kcos_enforce_encryption && !outstate->kcos_encryption_stage) {
774 *out_should_skip_coredump = true;
775 #if defined(__arm64__)
776 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
777 #else
778 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_ENCRYPTED_COREDUMP_SKIPPED;
779 #endif
780 kern_coredump_log(NULL, "(kdp_reset_output_vars) Encryption requested, is unavailable, and enforcement is active. Skipping current core.\n");
781 }
782 } else if (outstate->kcos_encryption_stage) {
783 outstate->kcos_encryption_stage->kos_bypass = true;
784 }
785
786 return KERN_SUCCESS;
787 }
788
789 static kern_return_t
kern_dump_update_header(struct kdp_core_out_state * outstate)790 kern_dump_update_header(struct kdp_core_out_state *outstate)
791 {
792 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
793 uint64_t foffset;
794 kern_return_t ret;
795
796 /* Write the file header -- first seek to the beginning of the file */
797 foffset = 0;
798 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
799 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
800 sizeof(foffset), &foffset, foffset, ret);
801 return ret;
802 }
803
804 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header_size, kdp_core_header)) != KERN_SUCCESS) {
805 kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
806 kdp_core_header_size, kdp_core_header, ret);
807 return ret;
808 }
809
810 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
811 kern_coredump_log(NULL, "(kern_dump_update_header) outproc data flush returned 0x%x\n", ret);
812 return ret;
813 }
814
815 #if defined(__arm64__)
816 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
817 kern_coredump_log(NULL, "(kern_dump_update_header) outproc explicit flush returned 0x%x\n", ret);
818 return ret;
819 }
820 #endif /* defined(__arm64__) */
821
822 return ret;
823 }
824
825 kern_return_t
kern_dump_record_file(void * kdp_core_out_state,const char * filename,uint64_t file_offset,uint64_t * out_file_length,uint64_t details_flags)826 kern_dump_record_file(void *kdp_core_out_state, const char *filename, uint64_t file_offset, uint64_t *out_file_length, uint64_t details_flags)
827 {
828 kern_return_t ret = KERN_SUCCESS;
829 uint64_t bytes_written = 0;
830 struct mach_core_details_v2 *core_details = NULL;
831 struct kdp_output_stage *last_stage;
832 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
833
834 assert(kdp_core_header->num_files < KERN_COREDUMP_MAX_CORES);
835 assert(out_file_length != NULL);
836 *out_file_length = 0;
837
838 last_stage = STAILQ_LAST(&outstate->kcos_out_stage, kdp_output_stage, kos_next);
839 bytes_written = last_stage->kos_bytes_written;
840
841 core_details = &(kdp_core_header->files[kdp_core_header->num_files]);
842 core_details->flags = details_flags;
843 core_details->offset = file_offset;
844 core_details->length = bytes_written;
845 strncpy((char *)&core_details->core_name, filename,
846 MACH_CORE_FILEHEADER_NAMELEN);
847 core_details->core_name[MACH_CORE_FILEHEADER_NAMELEN - 1] = '\0';
848
849 kdp_core_header->num_files++;
850
851 ret = kern_dump_update_header(outstate);
852 if (ret == KERN_SUCCESS) {
853 *out_file_length = bytes_written;
854 }
855
856 return ret;
857 }
858
859 kern_return_t
kern_dump_seek_to_next_file(void * kdp_core_out_state,uint64_t next_file_offset)860 kern_dump_seek_to_next_file(void *kdp_core_out_state, uint64_t next_file_offset)
861 {
862 struct kdp_core_out_state *outstate = (struct kdp_core_out_state *)kdp_core_out_state;
863 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
864 kern_return_t ret;
865
866 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(next_file_offset), &next_file_offset)) != KERN_SUCCESS) {
867 kern_coredump_log(NULL, "(kern_dump_seek_to_next_file) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
868 sizeof(next_file_offset), &next_file_offset, next_file_offset, ret);
869 }
870
871 return ret;
872 }
873
874 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
875
876 static kern_return_t
kern_dump_write_public_key(struct kdp_core_out_state * outstate)877 kern_dump_write_public_key(struct kdp_core_out_state *outstate)
878 {
879 struct kdp_output_stage *first_stage = STAILQ_FIRST(&outstate->kcos_out_stage);
880 uint64_t foffset;
881 uint64_t remainder = PUBLIC_KEY_RESERVED_LENGTH - kdp_core_header->pub_key_length;
882 kern_return_t ret;
883
884 if (kdp_core_header->pub_key_offset == 0 || kdp_core_header->pub_key_length == 0) {
885 // Nothing to do
886 return KERN_SUCCESS;
887 }
888
889 /* Write the public key -- first seek to the appropriate offset */
890 foffset = kdp_core_header->pub_key_offset;
891 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
892 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
893 sizeof(foffset), &foffset, foffset, ret);
894 return ret;
895 }
896
897 // Write the public key
898 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
899 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
900 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
901 return ret;
902 }
903
904 // Fill out the remainder of the block with zeroes
905 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
906 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
907 remainder, ret);
908 return ret;
909 }
910
911 // Do it once more to write the "next" public key
912 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, kdp_core_header->pub_key_length, kdp_core_public_key)) != KERN_SUCCESS) {
913 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %u, %p) returned 0x%x\n",
914 kdp_core_header->pub_key_length, kdp_core_public_key, ret);
915 return ret;
916 }
917
918 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, remainder, NULL)) != KERN_SUCCESS) {
919 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc(KDP_DATA, NULL, %llu, NULL) returned 0x%x\n",
920 remainder, ret);
921 return ret;
922 }
923
924 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_DATA, NULL, 0, NULL)) != KERN_SUCCESS) {
925 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc data flush returned 0x%x\n", ret);
926 return ret;
927 }
928
929 #if defined(__arm64__)
930 if ((ret = (first_stage->kos_funcs.kosf_outproc)(first_stage, KDP_FLUSH, NULL, 0, NULL)) != KERN_SUCCESS) {
931 kern_coredump_log(NULL, "(kern_dump_write_public_key) outproc explicit flush returned 0x%x\n", ret);
932 return ret;
933 }
934 #endif /* defined(__arm64__) */
935
936 return ret;
937 }
938
939 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
940
941 static kern_return_t
chain_output_stages(enum kern_dump_type kd_variant,struct kdp_core_out_state * outstate,uint64_t * details_flags)942 chain_output_stages(enum kern_dump_type kd_variant, struct kdp_core_out_state *outstate, uint64_t *details_flags)
943 {
944 struct kdp_output_stage *current = NULL;
945
946 assert(details_flags);
947 *details_flags = 0;
948
949 switch (kd_variant) {
950 case KERN_DUMP_STACKSHOT_DISK:
951 OS_FALLTHROUGH;
952 case KERN_DUMP_DISK:
953 #if defined(__arm64__)
954 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &memory_backing_aware_buffer_output_stage, kos_next);
955 #endif
956 if (!kdp_corezip_disabled) {
957 if (kdp_core_is_initializing_lz4_stage) {
958 kern_coredump_log(NULL, "We were in the middle of initializing LZ4 stage. Cannot write a coredump to disk\n");
959 return KERN_FAILURE;
960 } else if (!lz4_output_stage.kos_initialized) {
961 kern_coredump_log(NULL, "LZ4 stage is not yet initialized. Cannot write a coredump to disk\n");
962 return KERN_FAILURE;
963 }
964 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &lz4_output_stage, kos_next);
965 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_LZ4;
966 }
967 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
968 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
969 if (kdp_core_is_initializing_encryption_stage) {
970 kern_coredump_log(NULL, "We were in the middle of initializing encryption. Marking it as unavailable\n");
971 } else if (aea_output_stage.kos_initialized) {
972 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &aea_output_stage, kos_next);
973 outstate->kcos_encryption_stage = &aea_output_stage;
974 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_ENCRYPTED_AEA;
975 }
976 outstate->kcos_enforce_encryption = kern_dump_should_enforce_encryption();
977 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
978 if (kdp_core_is_initializing_disk_stage) {
979 kern_coredump_log(NULL, "We were in the middle of initializing the disk stage. Cannot write a coredump to disk\n");
980 return KERN_FAILURE;
981 } else if (disk_output_stage.kos_initialized == false) {
982 kern_coredump_log(NULL, "Corefile is not yet initialized. Cannot write a coredump to disk\n");
983 return KERN_FAILURE;
984 }
985 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &disk_output_stage, kos_next);
986 break;
987 case KERN_DUMP_NET:
988 if (!kdp_corezip_disabled) {
989 if (!zlib_output_stage.kos_initialized) {
990 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to the network\n");
991 return KERN_FAILURE;
992 }
993 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
994 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
995 }
996 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &progress_notify_output_stage, kos_next);
997 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &buffer_output_stage, kos_next);
998 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &net_output_stage, kos_next);
999 break;
1000 #if defined(__arm64__)
1001 case KERN_DUMP_HW_SHMEM_DBG:
1002 if (!kdp_corezip_disabled) {
1003 if (!zlib_output_stage.kos_initialized) {
1004 kern_coredump_log(NULL, "Zlib stage is not initialized. Cannot write a coredump to shared memory\n");
1005 return KERN_FAILURE;
1006 }
1007 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &zlib_output_stage, kos_next);
1008 *details_flags |= MACH_CORE_DETAILS_V2_FLAG_COMPRESSED_ZLIB;
1009 }
1010 STAILQ_INSERT_TAIL(&outstate->kcos_out_stage, &shmem_output_stage, kos_next);
1011 break;
1012 #endif /* defined(__arm64__) */
1013 }
1014
1015 STAILQ_FOREACH(current, &outstate->kcos_out_stage, kos_next) {
1016 current->kos_outstate = outstate;
1017 }
1018
1019 return KERN_SUCCESS;
1020 }
1021
1022 #if defined(__arm64__)
1023
1024 static const char *panic_buf_filename = "panic_region";
1025
1026 static kern_return_t
dump_panic_buffer(struct kdp_core_out_state * outstate,char * panic_buf,size_t panic_len,uint64_t * foffset,uint64_t details_flags)1027 dump_panic_buffer(struct kdp_core_out_state *outstate, char *panic_buf, size_t panic_len,
1028 uint64_t *foffset, uint64_t details_flags)
1029 {
1030 kern_return_t ret = KERN_SUCCESS;
1031 bool should_skip = false;
1032
1033 kern_coredump_log(NULL, "\nBeginning dump of panic region of size 0x%zx\n", panic_len);
1034
1035 ret = kdp_reset_output_vars(outstate, panic_len, true, &should_skip,
1036 panic_buf_filename, RAW_COREDUMP);
1037 if (KERN_SUCCESS != ret) {
1038 return ret;
1039 }
1040
1041 if (should_skip) {
1042 kern_coredump_log(NULL, "Skipping panic region dump\n");
1043 return ret;
1044 }
1045
1046 uint64_t compressed_panic_region_len = 0;
1047 ret = kdp_core_output(outstate, panic_len, panic_buf);
1048 if (KERN_SUCCESS != ret) {
1049 kern_coredump_log(NULL, "Failed to write panic region to file, kdp_coreoutput(outstate, %zu, %p) returned 0x%x\n",
1050 panic_len, panic_buf, ret);
1051 return ret;
1052 }
1053
1054 ret = kdp_core_output(outstate, 0, NULL);
1055 if (KERN_SUCCESS != ret) {
1056 kern_coredump_log(NULL, "Failed to flush panic region data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", outstate, ret);
1057 return ret;
1058 }
1059
1060 ret = kern_dump_record_file(outstate, panic_buf_filename, *foffset, &compressed_panic_region_len,
1061 details_flags);
1062 if (KERN_SUCCESS != ret) {
1063 kern_coredump_log(NULL, "Failed to record panic region in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1064 return ret;
1065 }
1066
1067 kern_coredump_log(NULL, "Recorded panic region in corefile at offset 0x%llx, compressed to %llu bytes\n", *foffset, compressed_panic_region_len);
1068 *foffset = roundup((*foffset + compressed_panic_region_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1069
1070 ret = kern_dump_seek_to_next_file(outstate, *foffset);
1071 if (KERN_SUCCESS != ret) {
1072 kern_coredump_log(NULL, "Failed to seek to panic region file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", *foffset, ret);
1073 return ret;
1074 }
1075
1076 return ret;
1077 }
1078 #endif /* defined(__arm64__) */
1079
1080 static int
do_kern_dump(enum kern_dump_type kd_variant)1081 do_kern_dump(enum kern_dump_type kd_variant)
1082 {
1083 struct kdp_core_out_state outstate = { };
1084 struct kdp_output_stage *first_stage = NULL;
1085 char *coredump_log_start = NULL, *buf = NULL;
1086 size_t reserved_debug_logsize = 0, prior_debug_logsize = 0;
1087 uint64_t foffset = 0;
1088 kern_return_t ret = KERN_SUCCESS;
1089 boolean_t output_opened = FALSE, dump_succeeded = TRUE;
1090 uint64_t details_flags = 0;
1091
1092 /* Initialize output context */
1093
1094 bzero(&outstate, sizeof(outstate));
1095 STAILQ_INIT(&outstate.kcos_out_stage);
1096 ret = chain_output_stages(kd_variant, &outstate, &details_flags);
1097 if (KERN_SUCCESS != ret) {
1098 dump_succeeded = FALSE;
1099 goto exit;
1100 }
1101 first_stage = STAILQ_FIRST(&outstate.kcos_out_stage);
1102
1103 /*
1104 * Record the initial panic log buffer length so we can dump the coredump log
1105 * and panic log to disk
1106 */
1107 coredump_log_start = debug_buf_ptr;
1108 #if defined(__arm64__)
1109 assert(panic_info->eph_other_log_offset != 0);
1110 assert(panic_info->eph_panic_log_len != 0);
1111 /* Include any data from before the panic log as well */
1112 prior_debug_logsize = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1113 panic_info->eph_panic_log_len + panic_info->eph_other_log_len;
1114 #else /* defined(__arm64__) */
1115 if (panic_info->mph_panic_log_offset != 0) {
1116 prior_debug_logsize = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1117 panic_info->mph_panic_log_len + panic_info->mph_other_log_len;
1118 }
1119 #endif /* defined(__arm64__) */
1120
1121 assert(prior_debug_logsize <= debug_buf_size);
1122
1123 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1124 /* Open the file for output */
1125 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_WRQ, NULL, 0, NULL)) != KERN_SUCCESS) {
1126 kern_coredump_log(NULL, "outproc(KDP_WRQ, NULL, 0, NULL) returned 0x%x\n", ret);
1127 dump_succeeded = FALSE;
1128 goto exit;
1129 }
1130 }
1131 output_opened = true;
1132
1133 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1134 const size_t aligned_corefile_header_size = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1135 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1136 const size_t aligned_public_key_size = PUBLIC_KEY_RESERVED_LENGTH * 2;
1137 #else
1138 const size_t aligned_public_key_size = 0;
1139 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1140
1141 reserved_debug_logsize = prior_debug_logsize + KERN_COREDUMP_MAXDEBUGLOGSIZE;
1142
1143 /* Space for file header, public key, panic log, core log */
1144 foffset = roundup(aligned_corefile_header_size + aligned_public_key_size + reserved_debug_logsize, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1145 kdp_core_header->log_offset = aligned_corefile_header_size + aligned_public_key_size;
1146
1147 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1148 /* Write the public key */
1149 ret = kern_dump_write_public_key(&outstate);
1150 if (KERN_SUCCESS != ret) {
1151 kern_coredump_log(NULL, "(do_kern_dump write public key) returned 0x%x\n", ret);
1152 dump_succeeded = FALSE;
1153 goto exit;
1154 }
1155 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1156
1157 /* Seek the calculated offset (we'll scrollback later to flush the logs and header) */
1158 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1159 kern_coredump_log(NULL, "(do_kern_dump seek begin) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1160 sizeof(foffset), &foffset, foffset, ret);
1161 dump_succeeded = FALSE;
1162 goto exit;
1163 }
1164 }
1165
1166 #if defined(__arm64__)
1167 flush_mmu_tlb();
1168 #endif
1169
1170 kern_coredump_log(NULL, "%s", (kd_variant == KERN_DUMP_DISK) ? "Writing local cores...\n" :
1171 "Transmitting kernel state, please wait:\n");
1172
1173 #if defined (__arm64__)
1174 char *panic_buf = (char *)gPanicBase;
1175 size_t panic_len = (vm_offset_t)debug_buf_ptr - gPanicBase;
1176 if (kd_variant == KERN_DUMP_DISK && (panic_buf && panic_len)) {
1177 ret = dump_panic_buffer(&outstate, panic_buf, panic_len, &foffset, details_flags);
1178 if (KERN_SUCCESS != ret) {
1179 dump_succeeded = FALSE;
1180 }
1181 }
1182 #endif
1183
1184 #if defined(__x86_64__)
1185 if (((kd_variant == KERN_DUMP_STACKSHOT_DISK) || (kd_variant == KERN_DUMP_DISK)) && ((panic_stackshot_buf != 0) && (panic_stackshot_len != 0))) {
1186 bool should_skip = false;
1187 static const char *stackshot_filename = "panic_stackshot.kcdata";
1188
1189 kern_coredump_log(NULL, "\nBeginning dump of kernel stackshot\n");
1190
1191 ret = kdp_reset_output_vars(&outstate, panic_stackshot_len, true, &should_skip, stackshot_filename, RAW_COREDUMP);
1192
1193 if (ret != KERN_SUCCESS) {
1194 kern_coredump_log(NULL, "Failed to reset outstate for stackshot with len 0x%zx, returned 0x%x\n", panic_stackshot_len, ret);
1195 dump_succeeded = FALSE;
1196 } else if (!should_skip) {
1197 uint64_t compressed_stackshot_len = 0;
1198 if ((ret = kdp_core_output(&outstate, panic_stackshot_len, (void *)panic_stackshot_buf)) != KERN_SUCCESS) {
1199 kern_coredump_log(NULL, "Failed to write panic stackshot to file, kdp_coreoutput(outstate, %lu, %p) returned 0x%x\n",
1200 panic_stackshot_len, (void *) panic_stackshot_buf, ret);
1201 dump_succeeded = FALSE;
1202 } else if ((ret = kdp_core_output(&outstate, 0, NULL)) != KERN_SUCCESS) {
1203 kern_coredump_log(NULL, "Failed to flush stackshot data : kdp_core_output(%p, 0, NULL) returned 0x%x\n", &outstate, ret);
1204 dump_succeeded = FALSE;
1205 } else if ((ret = kern_dump_record_file(&outstate, stackshot_filename, foffset, &compressed_stackshot_len, details_flags)) != KERN_SUCCESS) {
1206 kern_coredump_log(NULL, "Failed to record panic stackshot in corefile header, kern_dump_record_file returned 0x%x\n", ret);
1207 dump_succeeded = FALSE;
1208 } else {
1209 kern_coredump_log(NULL, "Recorded panic stackshot in corefile at offset 0x%llx, compressed to %llu bytes\n", foffset, compressed_stackshot_len);
1210 foffset = roundup((foffset + compressed_stackshot_len), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1211 if ((ret = kern_dump_seek_to_next_file(&outstate, foffset)) != KERN_SUCCESS) {
1212 kern_coredump_log(NULL, "Failed to seek to stackshot file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", foffset, ret);
1213 dump_succeeded = FALSE;
1214 }
1215 }
1216 } else {
1217 kern_coredump_log(NULL, "Skipping stackshot dump\n");
1218 }
1219 }
1220 #endif
1221
1222 if (kd_variant == KERN_DUMP_DISK) {
1223 /*
1224 * Dump co-processors as well, foffset will be overwritten with the
1225 * offset of the next location in the file to be written to.
1226 */
1227 if (kern_do_coredump(&outstate, KCF_NONE, foffset, &foffset, details_flags) != 0) {
1228 dump_succeeded = FALSE;
1229 }
1230 #if defined (__arm64__)
1231 } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) {
1232 kern_coredump_log(NULL, "Writing all cores through shared memory debugger\n");
1233 if (kern_do_coredump(&outstate, KCF_ABORT_ON_FAILURE, foffset, &foffset, details_flags) != 0) {
1234 dump_succeeded = FALSE;
1235 }
1236 #endif /* __arm64__ */
1237 } else if (kd_variant != KERN_DUMP_STACKSHOT_DISK) {
1238 /* Only the kernel */
1239 if (kern_do_coredump(&outstate, KCF_KERNEL_ONLY, foffset, &foffset, details_flags) != 0) {
1240 dump_succeeded = FALSE;
1241 }
1242 }
1243
1244 if (kd_variant == KERN_DUMP_DISK) {
1245 assert(reserved_debug_logsize != 0);
1246 size_t remaining_debug_logspace = reserved_debug_logsize;
1247
1248 /* Write the debug log -- first seek to the end of the corefile header */
1249 foffset = kdp_core_header->log_offset;
1250 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_SEEK, NULL, sizeof(foffset), &foffset)) != KERN_SUCCESS) {
1251 kern_coredump_log(NULL, "(do_kern_dump seek logfile) outproc(KDP_SEEK, NULL, %lu, %p) foffset = 0x%llx returned 0x%x\n",
1252 sizeof(foffset), &foffset, foffset, ret);
1253 dump_succeeded = FALSE;
1254 goto exit;
1255 }
1256
1257 /* First flush the data from just the paniclog */
1258 size_t initial_log_length = 0;
1259 #if defined(__arm64__)
1260 initial_log_length = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
1261 panic_info->eph_panic_log_len;
1262 #else
1263 if (panic_info->mph_panic_log_offset != 0) {
1264 initial_log_length = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
1265 panic_info->mph_panic_log_len;
1266 }
1267 #endif
1268
1269 buf = debug_buf_base;
1270 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, initial_log_length, buf)) != KERN_SUCCESS) {
1271 kern_coredump_log(NULL, "(do_kern_dump paniclog) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1272 initial_log_length, buf, ret);
1273 dump_succeeded = FALSE;
1274 goto exit;
1275 }
1276
1277 remaining_debug_logspace -= initial_log_length;
1278
1279 /* Next include any log data from after the stackshot (the beginning of the 'other' log). */
1280 #if defined(__arm64__)
1281 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->eph_other_log_offset);
1282 #else
1283 /*
1284 * There may be no paniclog if we're doing a coredump after a call to Debugger() on x86 if debugger_is_panic was
1285 * configured to FALSE based on the boot-args. In that case just start from where the debug buffer was when
1286 * we began taking a coredump.
1287 */
1288 if (panic_info->mph_other_log_offset != 0) {
1289 buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->mph_other_log_offset);
1290 } else {
1291 buf = coredump_log_start;
1292 }
1293 #endif
1294 assert(debug_buf_ptr >= buf);
1295
1296 size_t other_log_length = debug_buf_ptr - buf;
1297 if (other_log_length > remaining_debug_logspace) {
1298 other_log_length = remaining_debug_logspace;
1299 }
1300
1301 /* Write the coredump log */
1302 if ((ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_DATA, NULL, other_log_length, buf)) != KERN_SUCCESS) {
1303 kern_coredump_log(NULL, "(do_kern_dump coredump log) outproc(KDP_DATA, NULL, %lu, %p) returned 0x%x\n",
1304 other_log_length, buf, ret);
1305 dump_succeeded = FALSE;
1306 goto exit;
1307 }
1308
1309 kdp_core_header->log_length = initial_log_length + other_log_length;
1310 kern_dump_update_header(&outstate);
1311 }
1312
1313 exit:
1314 /* close / last packet */
1315 if (output_opened && (ret = first_stage->kos_funcs.kosf_outproc(first_stage, KDP_EOF, NULL, 0, ((void *) 0))) != KERN_SUCCESS) {
1316 kern_coredump_log(NULL, "(do_kern_dump close) outproc(KDP_EOF, NULL, 0, 0) returned 0x%x\n", ret);
1317 dump_succeeded = FALSE;
1318 }
1319
1320 /* If applicable, update the panic header and flush it so we update the CRC */
1321 #if defined(__arm64__)
1322 panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1323 EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1324 paniclog_flush();
1325 #else
1326 if (panic_info->mph_panic_log_offset != 0) {
1327 panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
1328 MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED);
1329 paniclog_flush();
1330 }
1331 #endif
1332
1333 return dump_succeeded ? 0 : -1;
1334 }
1335
1336 boolean_t
dumped_kernel_core(void)1337 dumped_kernel_core(void)
1338 {
1339 return kern_dump_successful;
1340 }
1341
1342 int
kern_dump(enum kern_dump_type kd_variant)1343 kern_dump(enum kern_dump_type kd_variant)
1344 {
1345 static boolean_t local_dump_in_progress = FALSE, dumped_local = FALSE;
1346 int ret = -1;
1347 #if KASAN
1348 kasan_kdp_disable();
1349 #endif
1350 if ((kd_variant == KERN_DUMP_DISK) || (kd_variant == KERN_DUMP_STACKSHOT_DISK)) {
1351 if (dumped_local) {
1352 return 0;
1353 }
1354 if (local_dump_in_progress) {
1355 return -1;
1356 }
1357 local_dump_in_progress = TRUE;
1358 ret = do_kern_dump(kd_variant);
1359 if (ret == 0) {
1360 dumped_local = TRUE;
1361 kern_dump_successful = TRUE;
1362 local_dump_in_progress = FALSE;
1363 }
1364
1365 return ret;
1366 #if defined(__arm64__)
1367 } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) {
1368 ret = do_kern_dump(kd_variant);
1369 if (ret == 0) {
1370 kern_dump_successful = TRUE;
1371 }
1372 return ret;
1373 #endif
1374 } else {
1375 ret = do_kern_dump(kd_variant);
1376 if (ret == 0) {
1377 kern_dump_successful = TRUE;
1378 }
1379 return ret;
1380 }
1381 }
1382
1383 static kern_return_t
kdp_core_init_output_stages(void)1384 kdp_core_init_output_stages(void)
1385 {
1386 kern_return_t ret = KERN_SUCCESS;
1387
1388 // We only zero-out the disk stage. It will be initialized
1389 // later on when the corefile is initialized
1390 bzero(&disk_output_stage, sizeof(disk_output_stage));
1391
1392 // We only zero-out the LZ4 stage. It will be initialized
1393 // later on when the kext is loaded.
1394 bzero(&lz4_output_stage, sizeof(lz4_output_stage));
1395 lz4_stage_monitor_availability();
1396
1397 // We only initialize the zlib output stage if we can reach the debugger.
1398 // This saves us from wasting some wired memory that will never be used
1399 // in other configurations.
1400 bzero(&zlib_output_stage, sizeof(zlib_output_stage));
1401 if (debug_boot_arg && (debug_boot_arg & DB_REBOOT_ALWAYS) == 0) {
1402 ret = zlib_stage_initialize(&zlib_output_stage);
1403 if (KERN_SUCCESS != ret) {
1404 return ret;
1405 }
1406 }
1407
1408 bzero(&buffer_output_stage, sizeof(buffer_output_stage));
1409 ret = buffer_stage_initialize(&buffer_output_stage, kdp_crashdump_pkt_size);
1410 if (KERN_SUCCESS != ret) {
1411 return ret;
1412 }
1413
1414 bzero(&net_output_stage, sizeof(net_output_stage));
1415 ret = net_stage_initialize(&net_output_stage);
1416 if (KERN_SUCCESS != ret) {
1417 return ret;
1418 }
1419
1420 bzero(&progress_notify_output_stage, sizeof(progress_notify_output_stage));
1421 ret = progress_notify_stage_initialize(&progress_notify_output_stage);
1422 if (KERN_SUCCESS != ret) {
1423 return ret;
1424 }
1425
1426 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1427 // We only zero-out the AEA stage. It will be initialized
1428 // later on, if it's supported and needed
1429 bzero(&aea_output_stage, sizeof(aea_output_stage));
1430 aea_stage_monitor_availability();
1431 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1432
1433 #if defined(__arm64__)
1434 bzero(&shmem_output_stage, sizeof(shmem_output_stage));
1435 if (PE_consistent_debug_enabled() && PE_i_can_has_debugger(NULL)) {
1436 ret = shmem_stage_initialize(&shmem_output_stage);
1437 if (KERN_SUCCESS != ret) {
1438 return ret;
1439 }
1440 }
1441 #endif /* defined(__arm64__) */
1442
1443 #if defined(__arm64__)
1444 bzero(&memory_backing_aware_buffer_output_stage, sizeof(memory_backing_aware_buffer_output_stage));
1445 ret = memory_backing_aware_buffer_stage_initialize(&memory_backing_aware_buffer_output_stage);
1446 if (KERN_SUCCESS != ret) {
1447 return ret;
1448 }
1449 #endif /* defined(__arm64__) */
1450
1451 return ret;
1452 }
1453
1454 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1455
1456 bool
kern_dump_should_enforce_encryption(void)1457 kern_dump_should_enforce_encryption(void)
1458 {
1459 static int enforce_encryption = -1;
1460
1461 // Only check once
1462 if (enforce_encryption == -1) {
1463 uint32_t coredump_encryption_flags = 0;
1464
1465 // When set, the boot-arg is the sole decider
1466 if (!kernel_debugging_restricted() &&
1467 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags))) {
1468 enforce_encryption = (coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_ENFORCEMENT) != 0 ? 1 : 0;
1469 } else {
1470 enforce_encryption = 0;
1471 }
1472 }
1473
1474 return enforce_encryption != 0;
1475 }
1476
1477 static bool
kern_dump_is_encryption_available(void)1478 kern_dump_is_encryption_available(void)
1479 {
1480 // Default to feature enabled unless boot-arg says otherwise
1481 uint32_t coredump_encryption_flags = COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY;
1482
1483 if (!kernel_debugging_restricted()) {
1484 PE_parse_boot_argn("coredump_encryption", &coredump_encryption_flags, sizeof(coredump_encryption_flags));
1485 }
1486
1487 if ((coredump_encryption_flags & COREDUMP_ENCRYPTION_OVERRIDES_AVAILABILITY) == 0) {
1488 return false;
1489 }
1490
1491 return aea_stage_is_available();
1492 }
1493
1494 /*
1495 * Initialize (or de-initialize) the encryption stage. This is done in a way such that if initializing the
1496 * encryption stage with a new key fails, then the existing encryption stage is left untouched. Once
1497 * the new stage is initialized, the old stage is uninitialized.
1498 *
1499 * This function is called whenever we have a new public key (whether from someone calling our sysctl, or because
1500 * we read it out of a corefile), or when encryption becomes available.
1501 *
1502 * Parameters:
1503 * - public_key: The public key to use when initializing the encryption stage. Can be NULL to indicate that
1504 * the encryption stage should be de-initialized.
1505 * - public_key_size: The size of the given public key.
1506 */
1507 static kern_return_t
kdp_core_init_encryption_stage(void * public_key,size_t public_key_size)1508 kdp_core_init_encryption_stage(void *public_key, size_t public_key_size)
1509 {
1510 kern_return_t ret = KERN_SUCCESS;
1511 struct kdp_output_stage new_encryption_stage = {};
1512 struct kdp_output_stage old_encryption_stage = {};
1513
1514 lck_mtx_assert(kdp_core_encryption_stage_lock, LCK_MTX_ASSERT_OWNED);
1515
1516 bzero(&new_encryption_stage, sizeof(new_encryption_stage));
1517
1518 if (public_key && kern_dump_is_encryption_available()) {
1519 ret = aea_stage_initialize(&new_encryption_stage, public_key, public_key_size);
1520 if (KERN_SUCCESS != ret) {
1521 printf("(kdp_core_init_encryption_stage) Failed to initialize the encryption stage. Error 0x%x\n", ret);
1522 return ret;
1523 }
1524 }
1525
1526 bcopy(&aea_output_stage, &old_encryption_stage, sizeof(aea_output_stage));
1527
1528 bcopy(&new_encryption_stage, &aea_output_stage, sizeof(new_encryption_stage));
1529
1530 if (old_encryption_stage.kos_initialized && old_encryption_stage.kos_funcs.kosf_free) {
1531 old_encryption_stage.kos_funcs.kosf_free(&old_encryption_stage);
1532 }
1533
1534 return KERN_SUCCESS;
1535 }
1536
1537 kern_return_t
kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data,void * access_context,void * recipient_context)1538 kdp_core_handle_new_encryption_key(IOCoreFileAccessCallback access_data, void *access_context, void *recipient_context)
1539 {
1540 kern_return_t ret = KERN_SUCCESS;
1541 struct kdp_core_encryption_key_descriptor *key_descriptor = (struct kdp_core_encryption_key_descriptor *) recipient_context;
1542 void *old_public_key = NULL;
1543 size_t old_public_key_size = 0;
1544
1545 if (!key_descriptor) {
1546 return kIOReturnBadArgument;
1547 }
1548
1549 lck_mtx_lock(kdp_core_encryption_stage_lock);
1550 kdp_core_is_initializing_encryption_stage = true;
1551
1552 do {
1553 // Do the risky part first, and bail out cleanly if it fails
1554 ret = kdp_core_init_encryption_stage(key_descriptor->kcekd_key, key_descriptor->kcekd_size);
1555 if (ret != KERN_SUCCESS) {
1556 printf("kdp_core_handle_new_encryption_key failed to re-initialize encryption stage. Error 0x%x\n", ret);
1557 break;
1558 }
1559
1560 // The rest of this function should technically never fail
1561
1562 old_public_key = kdp_core_public_key;
1563 old_public_key_size = kdp_core_header->pub_key_length;
1564
1565 kdp_core_public_key = key_descriptor->kcekd_key;
1566 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1567 kdp_core_header->flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_EXISTING_COREFILE_KEY_FORMAT_MASK;
1568 if (key_descriptor->kcekd_key) {
1569 kdp_core_header->flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1570 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(key_descriptor->kcekd_format);
1571 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1572 kdp_core_header->pub_key_length = key_descriptor->kcekd_size;
1573 } else {
1574 kdp_core_header->pub_key_offset = 0;
1575 kdp_core_header->pub_key_length = 0;
1576 }
1577
1578 /*
1579 * Return the old key to the caller to free
1580 */
1581 key_descriptor->kcekd_key = old_public_key;
1582 key_descriptor->kcekd_size = (uint16_t)old_public_key_size;
1583
1584 // If this stuff fails, we have bigger problems
1585 struct mach_core_fileheader_v2 existing_header;
1586 bool used_existing_header = false;
1587 ret = access_data(access_context, FALSE, 0, sizeof(existing_header), &existing_header);
1588 if (ret != KERN_SUCCESS) {
1589 printf("kdp_core_handle_new_encryption_key failed to read the existing corefile header. Error 0x%x\n", ret);
1590 break;
1591 }
1592
1593 if (existing_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1594 && existing_header.version == 2
1595 && (existing_header.pub_key_length == 0
1596 || kdp_core_header->pub_key_length == 0
1597 || existing_header.pub_key_length == kdp_core_header->pub_key_length)) {
1598 used_existing_header = true;
1599 existing_header.flags &= ~MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1600
1601 if (kdp_core_public_key) {
1602 existing_header.flags |= key_descriptor->kcekd_format & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1603
1604 if (existing_header.pub_key_offset == 0) {
1605 existing_header.pub_key_offset = kdp_core_header->pub_key_offset;
1606 existing_header.pub_key_length = kdp_core_header->pub_key_length;
1607 }
1608 }
1609
1610 ret = access_data(access_context, TRUE, 0, sizeof(existing_header), &existing_header);
1611 if (ret != KERN_SUCCESS) {
1612 printf("kdp_core_handle_new_encryption_key failed to update the existing corefile header. Error 0x%x\n", ret);
1613 break;
1614 }
1615 } else {
1616 ret = access_data(access_context, TRUE, 0, sizeof(struct mach_core_fileheader_v2), kdp_core_header);
1617 if (ret != KERN_SUCCESS) {
1618 printf("kdp_core_handle_new_encryption_key failed to write the corefile header. Error 0x%x\n", ret);
1619 break;
1620 }
1621 }
1622
1623 if (kdp_core_header->pub_key_length) {
1624 uint64_t offset = used_existing_header ? existing_header.pub_key_offset : kdp_core_header->pub_key_offset;
1625 ret = access_data(access_context, TRUE, offset + PUBLIC_KEY_RESERVED_LENGTH, kdp_core_header->pub_key_length, kdp_core_public_key);
1626 if (ret != KERN_SUCCESS) {
1627 printf("kdp_core_handle_new_encryption_key failed to write the next public key. Error 0x%x\n", ret);
1628 break;
1629 }
1630
1631 if (!used_existing_header) {
1632 // Everything that happens here is optional. It's not the end of the world if this stuff fails, so we don't return
1633 // any errors
1634 // Since we're writing out a completely new header, we make sure to zero-out the region that's reserved for the public key.
1635 // This allows us consumers of the corefile to know for sure that this corefile is not encrypted (yet). Once we actually
1636 // write out a corefile, we'll overwrite this region with the key that we ended up using at the time.
1637 // If we fail to zero-out this region, consumers would read garbage data and properly fail to interpret it as a public key,
1638 // which is why it is OK for us to fail here (it's hard to interpret garbage data as a valid key, and even then, they wouldn't
1639 // find a matching private key anyway)
1640 void *empty_key = NULL;
1641 kern_return_t temp_ret = KERN_SUCCESS;
1642
1643 empty_key = kalloc_data(PUBLIC_KEY_RESERVED_LENGTH,
1644 Z_WAITOK | Z_ZERO | Z_NOFAIL);
1645
1646 temp_ret = access_data(access_context, TRUE, offset, PUBLIC_KEY_RESERVED_LENGTH, empty_key);
1647 kfree_data(empty_key, PUBLIC_KEY_RESERVED_LENGTH);
1648
1649 if (temp_ret != KERN_SUCCESS) {
1650 printf("kdp_core_handle_new_encryption_key failed to zero-out the public key region. Error 0x%x\n", temp_ret);
1651 break;
1652 }
1653 }
1654 }
1655 } while (0);
1656
1657 kdp_core_is_initializing_encryption_stage = false;
1658 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1659
1660 return ret;
1661 }
1662
1663 kern_return_t
kdp_core_handle_encryption_available(void)1664 kdp_core_handle_encryption_available(void)
1665 {
1666 kern_return_t ret;
1667
1668 lck_mtx_lock(kdp_core_encryption_stage_lock);
1669 kdp_core_is_initializing_encryption_stage = true;
1670
1671 ret = kdp_core_init_encryption_stage(kdp_core_public_key, kdp_core_header->pub_key_length);
1672
1673 kdp_core_is_initializing_encryption_stage = false;
1674 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1675
1676 return ret;
1677 }
1678
1679 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1680
1681 kern_return_t
kdp_core_handle_lz4_available(void)1682 kdp_core_handle_lz4_available(void)
1683 {
1684 kern_return_t ret;
1685 lck_mtx_lock(kdp_core_lz4_stage_lock);
1686 kdp_core_is_initializing_lz4_stage = true;
1687
1688 ret = lz4_stage_initialize(&lz4_output_stage);
1689
1690 kdp_core_is_initializing_lz4_stage = false;
1691 lck_mtx_unlock(kdp_core_lz4_stage_lock);
1692
1693 return ret;
1694 }
1695
1696 kern_return_t
kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data,void * access_context,__unused void * recipient_context)1697 kdp_core_polled_io_polled_file_available(IOCoreFileAccessCallback access_data, void *access_context, __unused void *recipient_context)
1698 {
1699 kern_return_t ret = KERN_SUCCESS;
1700
1701 lck_mtx_lock(kdp_core_disk_stage_lock);
1702 kdp_core_is_initializing_disk_stage = true;
1703
1704 ret = disk_stage_initialize(&disk_output_stage);
1705
1706 kdp_core_is_initializing_disk_stage = false;
1707 lck_mtx_unlock(kdp_core_disk_stage_lock);
1708
1709 if (KERN_SUCCESS != ret) {
1710 return ret;
1711 }
1712
1713 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1714 // If someone has already provided a new public key,
1715 // there's no sense in reading the old one from the corefile.
1716 if (kdp_core_public_key != NULL) {
1717 return KERN_SUCCESS;
1718 }
1719
1720 // The kernel corefile is now available. Let's try to retrieve the public key from its
1721 // header (if available and supported).
1722
1723 // First let's read the corefile header itself
1724 struct mach_core_fileheader_v2 temp_header = {};
1725 ret = access_data(access_context, FALSE, 0, sizeof(temp_header), &temp_header);
1726 if (KERN_SUCCESS != ret) {
1727 printf("kdp_core_polled_io_polled_file_available failed to read corefile header. Error 0x%x\n", ret);
1728 return ret;
1729 }
1730
1731 // Check if the corefile header is initialized, and whether it's initialized to values that we support
1732 // (for backwards and forwards) compatibility, and check whether the header indicates that the corefile has
1733 // has a public key stashed inside of it.
1734 if (temp_header.signature == MACH_CORE_FILEHEADER_V2_SIGNATURE
1735 && temp_header.version == 2
1736 && temp_header.pub_key_offset != 0
1737 && temp_header.pub_key_length != 0
1738 /* Future-proofing: make sure it's the key format that we support */
1739 && (temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK) == MACH_CORE_FILEHEADER_V2_FLAG_NEXT_COREFILE_KEY_FORMAT_NIST_P256
1740 /* Add some extra sanity checks. These are not necessary */
1741 && temp_header.pub_key_length <= 4096
1742 && temp_header.pub_key_offset < 65535) {
1743 // The corefile header is properly initialized, is supported, and contains a public key.
1744 // Let's adopt that public key for our encryption needs
1745 void *public_key = NULL;
1746
1747 public_key = kalloc_data(temp_header.pub_key_length,
1748 Z_ZERO | Z_WAITOK | Z_NOFAIL);
1749
1750 // Read the public key from the corefile. Note that the key we're trying to adopt is the "next" key, which is
1751 // PUBLIC_KEY_RESERVED_LENGTH bytes after the public key.
1752 ret = access_data(access_context, FALSE, temp_header.pub_key_offset + PUBLIC_KEY_RESERVED_LENGTH, temp_header.pub_key_length, public_key);
1753 if (KERN_SUCCESS != ret) {
1754 printf("kdp_core_polled_io_polled_file_available failed to read the public key. Error 0x%x\n", ret);
1755 kfree_data(public_key, temp_header.pub_key_length);
1756 return ret;
1757 }
1758
1759 lck_mtx_lock(kdp_core_encryption_stage_lock);
1760 kdp_core_is_initializing_encryption_stage = true;
1761
1762 ret = kdp_core_init_encryption_stage(public_key, temp_header.pub_key_length);
1763 if (KERN_SUCCESS == ret) {
1764 kdp_core_header->flags |= temp_header.flags & MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_COREFILE_KEY_FORMAT_MASK;
1765 kdp_core_header->flags |= MACH_CORE_FILEHEADER_V2_FLAGS_NEXT_KEY_FORMAT_TO_KEY_FORMAT(temp_header.flags);
1766 kdp_core_header->pub_key_offset = roundup(kdp_core_header_size, KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN);
1767 kdp_core_header->pub_key_length = temp_header.pub_key_length;
1768 kdp_core_public_key = public_key;
1769 }
1770
1771 kdp_core_is_initializing_encryption_stage = false;
1772 lck_mtx_unlock(kdp_core_encryption_stage_lock);
1773 }
1774 #else
1775 #pragma unused(access_data, access_context)
1776 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1777
1778 return ret;
1779 }
1780
1781 kern_return_t
kdp_core_polled_io_polled_file_unavailable(void)1782 kdp_core_polled_io_polled_file_unavailable(void)
1783 {
1784 lck_mtx_lock(kdp_core_disk_stage_lock);
1785 kdp_core_is_initializing_disk_stage = true;
1786
1787 if (disk_output_stage.kos_initialized && disk_output_stage.kos_funcs.kosf_free) {
1788 disk_output_stage.kos_funcs.kosf_free(&disk_output_stage);
1789 }
1790
1791 kdp_core_is_initializing_disk_stage = false;
1792 lck_mtx_unlock(kdp_core_disk_stage_lock);
1793
1794 return KERN_SUCCESS;
1795 }
1796
1797 void
kdp_core_init(void)1798 kdp_core_init(void)
1799 {
1800 kern_return_t kr;
1801 kern_coredump_callback_config core_config = { };
1802
1803 /* Initialize output stages */
1804 kr = kdp_core_init_output_stages();
1805 assert(KERN_SUCCESS == kr);
1806
1807 kmem_alloc(kernel_map, (vm_offset_t*)&kdp_core_header,
1808 kdp_core_header_size,
1809 KMA_NOFAIL | KMA_ZERO | KMA_PERMANENT | KMA_KOBJECT | KMA_DATA_SHARED,
1810 VM_KERN_MEMORY_DIAG);
1811
1812 kdp_core_header->signature = MACH_CORE_FILEHEADER_V2_SIGNATURE;
1813 kdp_core_header->version = 2;
1814
1815 kdp_core_initialization_lock_group = lck_grp_alloc_init("KDPCoreStageInit", LCK_GRP_ATTR_NULL);
1816 kdp_core_disk_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1817
1818 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
1819 kdp_core_encryption_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1820
1821 (void) kern_dump_should_enforce_encryption();
1822 #endif // CONFIG_KDP_COREDUMP_ENCRYPTION
1823
1824 kdp_core_lz4_stage_lock = lck_mtx_alloc_init(kdp_core_initialization_lock_group, LCK_ATTR_NULL);
1825
1826 core_config.kcc_coredump_init = kern_dump_init;
1827 core_config.kcc_coredump_get_summary = kern_dump_save_summary;
1828 core_config.kcc_coredump_save_segment_descriptions = kern_dump_save_seg_descriptions;
1829 core_config.kcc_coredump_save_thread_state = kern_dump_save_thread_state;
1830 core_config.kcc_coredump_save_sw_vers_detail = kern_dump_save_sw_vers_detail;
1831 core_config.kcc_coredump_save_segment_data = kern_dump_save_segment_data;
1832 core_config.kcc_coredump_save_note_summary = kern_dump_save_note_summary;
1833 core_config.kcc_coredump_save_note_descriptions = kern_dump_save_note_descriptions;
1834 core_config.kcc_coredump_save_note_data = kern_dump_save_note_data;
1835
1836 kr = kern_register_xnu_coredump_helper(&core_config);
1837 assert(KERN_SUCCESS == kr);
1838 }
1839
1840 /*
1841 * Additional LC_NOTES added to the core.
1842 */
1843
1844 static kern_return_t
kern_dump_save_note_summary(void * refcon __unused,core_save_note_summary_cb callback,void * context)1845 kern_dump_save_note_summary(void *refcon __unused, core_save_note_summary_cb callback, void *context)
1846 {
1847 int count = 1;
1848 size_t size = sizeof(addrable_bits_note_t);
1849
1850 #ifdef CONFIG_SPTM
1851 /* Load binary spec note */
1852
1853 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1854
1855 if (debug_header != NULL &&
1856 debug_header->magic == DEBUG_HEADER_MAGIC_VAL &&
1857 debug_header->version == DEBUG_HEADER_CURRENT_VERSION) {
1858 /* Also add SPTM, TXM, and xnu kc load binary specs if present */
1859 count += debug_header->count;
1860 size += debug_header->count * sizeof(load_binary_spec_note_t);
1861 }
1862 #endif /* CONFIG_SPTM */
1863
1864 return callback(count, size, context);
1865 }
1866
1867 static kern_return_t
kern_dump_save_note_descriptions(void * refcon __unused,core_save_note_descriptions_cb callback,void * context)1868 kern_dump_save_note_descriptions(void *refcon __unused, core_save_note_descriptions_cb callback, void *context)
1869 {
1870 int max_ret = KERN_SUCCESS;
1871 int ret;
1872
1873 max_ret = ret = callback(ADDRABLE_BITS_DATA_OWNER, sizeof(addrable_bits_note_t), context);
1874
1875 #if CONFIG_SPTM
1876 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1877
1878 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1879 ret = callback(LOAD_BINARY_SPEC_DATA_OWNER, sizeof(load_binary_spec_note_t), context);
1880 max_ret = MAX(ret, max_ret);
1881 }
1882 #endif /* CONFIG_SPTM */
1883
1884 return max_ret;
1885 }
1886
1887 static kern_return_t
kern_dump_save_note_data(void * refcon __unused,core_save_note_data_cb callback,void * context)1888 kern_dump_save_note_data(void *refcon __unused, core_save_note_data_cb callback, void *context)
1889 {
1890 int max_ret = KERN_SUCCESS;
1891 int ret;
1892
1893 addrable_bits_note_t note = {
1894 .version = ADDRABLE_BITS_VER,
1895 .addressing_bits = pmap_kernel_va_bits(),
1896 .unused = 0
1897 };
1898
1899 max_ret = ret = callback(¬e, sizeof(addrable_bits_note_t), context);
1900
1901 #if CONFIG_SPTM
1902 struct debug_header const *debug_header = SPTMArgs != NULL ? SPTMArgs->debug_header : NULL;
1903
1904 for (int i = 0; i < (debug_header != NULL ? debug_header->count : 0); i++) {
1905 load_binary_spec_note_t load_binary_spec = {
1906 .version = LOAD_BINARY_SPEC_VERSION,
1907 .uuid = {0},
1908 .address = (uint64_t)debug_header->image[i],
1909 .slide = UINT64_MAX // unknown, load address specified
1910 };
1911
1912 char const *name;
1913 switch (i) {
1914 case DEBUG_HEADER_ENTRY_SPTM:
1915 name = "sptm";
1916 break;
1917 case DEBUG_HEADER_ENTRY_XNU:
1918 name = "xnu";
1919 break;
1920 case DEBUG_HEADER_ENTRY_TXM:
1921 name = "txm";
1922 break;
1923 default:
1924 name = "UNKNOWN";
1925 kern_coredump_log(context, "%s(): encountered unknown debug header entry %d, "
1926 "including anyway with name '%s'\n", __func__, i, name);
1927 }
1928
1929 strlcpy(load_binary_spec.name_cstring, name, LOAD_BINARY_NAME_BUF_SIZE);
1930
1931 ret = callback(&load_binary_spec, sizeof(load_binary_spec), context);
1932
1933 if (ret != KERN_SUCCESS) {
1934 kern_coredump_log(context, "%s(): failed to write load binary spec structure "
1935 "for binary #%d ('%s'): callback returned 0x%x\n",
1936 __func__, i, name, ret);
1937 max_ret = MAX(ret, max_ret);
1938 }
1939 }
1940 #endif /* CONFIG_SPTM */
1941
1942 return max_ret;
1943 }
1944
1945 #else
1946
1947 void
kdp_core_exclude_region(__unused vm_offset_t addr,__unused vm_size_t size)1948 kdp_core_exclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
1949 {
1950 }
1951
1952 void
kdp_core_unexclude_region(__unused vm_offset_t addr,__unused vm_size_t size)1953 kdp_core_unexclude_region(__unused vm_offset_t addr, __unused vm_size_t size)
1954 {
1955 }
1956
1957 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1958