1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @Apple_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
21 */
22
23 #include <sys/kdebug_common.h>
24 #include <sys/kdebug_triage.h>
25
26 #define TRIAGE_KDCOPYBUF_COUNT 128
27 #define TRIAGE_KDCOPYBUF_SIZE (TRIAGE_KDCOPYBUF_COUNT * sizeof(kd_buf))
28
29 struct kd_control kd_control_triage = {
30 .kds_free_list = { .raw = KDS_PTR_NULL },
31 .mode = KDEBUG_MODE_TRIAGE,
32 .kdebug_events_per_storage_unit = TRIAGE_EVENTS_PER_STORAGE_UNIT,
33 .kdebug_min_storage_units_per_cpu = TRIAGE_MIN_STORAGE_UNITS_PER_CPU,
34 .kdebug_kdcopybuf_count = TRIAGE_KDCOPYBUF_COUNT,
35 .kdebug_kdcopybuf_size = TRIAGE_KDCOPYBUF_SIZE,
36 .kdc_flags = KDBG_DEBUGID_64,
37 .kdc_emit = KDEMIT_DISABLE,
38 .kdc_oldest_time = 0
39 };
40
41 struct kd_buffer kd_buffer_triage = {
42 .kdb_event_count = 0,
43 .kdb_storage_count = 0,
44 .kdb_storage_threshold = 0,
45 .kdb_region_count = 0,
46 .kdb_info = NULL,
47 .kd_bufs = NULL,
48 .kdcopybuf = NULL
49 };
50
51 static LCK_GRP_DECLARE(ktriage_grp, "ktriage");
52 static LCK_MTX_DECLARE(ktriage_mtx, &ktriage_grp);
53
54 static void
ktriage_lock(void)55 ktriage_lock(void)
56 {
57 lck_mtx_lock(&ktriage_mtx);
58 }
59
60 static void
ktriage_unlock(void)61 ktriage_unlock(void)
62 {
63 lck_mtx_unlock(&ktriage_mtx);
64 }
65
66 int
create_buffers_triage(void)67 create_buffers_triage(void)
68 {
69 int error = 0;
70 int events_per_storage_unit, min_storage_units_per_cpu;
71
72 if (kd_control_triage.kdc_flags & KDBG_BUFINIT) {
73 panic("create_buffers_triage shouldn't be called once we have inited the triage system.");
74 }
75
76 events_per_storage_unit = kd_control_triage.kdebug_events_per_storage_unit;
77 min_storage_units_per_cpu = kd_control_triage.kdebug_min_storage_units_per_cpu;
78
79 kd_control_triage.kdebug_cpus = kdbg_cpu_count();
80 kd_control_triage.alloc_cpus = kd_control_triage.kdebug_cpus;
81 kd_control_triage.kdc_coprocs = NULL;
82
83 if (kd_buffer_triage.kdb_event_count < (kd_control_triage.kdebug_cpus * events_per_storage_unit * min_storage_units_per_cpu)) {
84 kd_buffer_triage.kdb_storage_count = kd_control_triage.kdebug_cpus * min_storage_units_per_cpu;
85 } else {
86 kd_buffer_triage.kdb_storage_count = kd_buffer_triage.kdb_event_count / events_per_storage_unit;
87 }
88
89 kd_buffer_triage.kdb_event_count = kd_buffer_triage.kdb_storage_count * events_per_storage_unit;
90
91 kd_buffer_triage.kd_bufs = NULL;
92
93 error = create_buffers(&kd_control_triage, &kd_buffer_triage, VM_KERN_MEMORY_TRIAGE);
94
95 if (!error) {
96 kd_control_triage.kdc_oldest_time = mach_continuous_time();
97 kd_control_triage.enabled = 1;
98 kd_buffer_triage.kdb_storage_threshold = kd_buffer_triage.kdb_storage_count / 2;
99 }
100
101 return error;
102 }
103
104 __attribute__((noreturn))
105 void
delete_buffers_triage(void)106 delete_buffers_triage(void)
107 {
108 /*
109 * If create_buffers() for triage mode fails, it will call the generic delete_buffers() to
110 * free the resources. This specific call should never be invoked because we expect the
111 * triage system to always be ON.
112 */
113 panic("delete_buffers_triage shouldn't be invoked");
114 }
115
116 const char *ktriage_error_index_invalid_str = "Error descr index invalid\n";
117 const char *ktriage_error_subsyscode_invalid_str = "KTriage: Subsystem code invalid\n";
118 ktriage_strings_t subsystems_triage_strings[KDBG_TRIAGE_SUBSYS_MAX + 1];
119
120 static void
ktriage_convert_to_string(uint64_t debugid,char * buf,uint32_t bufsz)121 ktriage_convert_to_string(uint64_t debugid, char *buf, uint32_t bufsz)
122 {
123 if (buf == NULL) {
124 return;
125 }
126
127 int subsystem = KDBG_TRIAGE_EXTRACT_CLASS(debugid);
128
129 /* zero subsystem means there is nothing to log */
130 if (subsystem == 0) {
131 return;
132 }
133
134 size_t prefixlen, msglen;
135 if (subsystem <= KDBG_TRIAGE_SUBSYS_MAX) {
136 int subsystem_num_strings = subsystems_triage_strings[subsystem].num_strings;
137 const char **subsystem_strings = subsystems_triage_strings[subsystem].strings;
138
139 prefixlen = strlen(subsystem_strings[0]);
140 strlcpy(buf, (const char*)(subsystem_strings[0]), bufsz);
141
142 int strindx = KDBG_TRIAGE_EXTRACT_CODE(debugid);
143 if (strindx >= 1) { /* 0 is reserved for prefix */
144 if (strindx < subsystem_num_strings) {
145 msglen = MIN(bufsz - prefixlen, strlen(subsystem_strings[strindx]) + 1); /* incl. NULL termination */
146 strlcpy(buf + prefixlen, (subsystem_strings[strindx]), msglen);
147 } else {
148 msglen = MIN(bufsz - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
149 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
150 }
151 } else {
152 msglen = MIN(bufsz - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
153 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
154 }
155 } else {
156 msglen = MIN(bufsz, strlen(ktriage_error_subsyscode_invalid_str) + 1); /* incl. NULL termination */
157 strlcpy(buf, ktriage_error_subsyscode_invalid_str, msglen);
158 }
159
160 return;
161 }
162
163 void
ktriage_record(uint64_t thread_id,uint64_t debugid,uintptr_t arg)164 ktriage_record(
165 uint64_t thread_id,
166 uint64_t debugid,
167 uintptr_t arg)
168 {
169 struct kd_record kd_rec;
170
171 if (thread_id == 0) {
172 thread_id = thread_tid(current_thread());
173 }
174
175 kd_rec.cpu = -1;
176 kd_rec.timestamp = -1;
177
178 /*
179 * use 64-bit debugid per our flag KDBG_DEBUGID_64
180 * that is set in kd_control_triage (on LP64 only).
181 */
182 assert(kd_control_triage.kdc_flags & KDBG_DEBUGID_64);
183
184 kd_rec.debugid = 0;
185 kd_rec.arg4 = (uintptr_t)debugid;
186
187 kd_rec.arg1 = arg;
188 kd_rec.arg2 = 0;
189 kd_rec.arg3 = 0;
190 kd_rec.arg5 = (uintptr_t)thread_id;
191
192 kernel_debug_write(&kd_control_triage,
193 &kd_buffer_triage,
194 kd_rec);
195 }
196
197 void
ktriage_extract(uint64_t thread_id,void * buf,uint32_t bufsz)198 ktriage_extract(
199 uint64_t thread_id,
200 void *buf,
201 uint32_t bufsz)
202 {
203 size_t i, record_bytes, record_cnt, record_bufsz;
204 void *record_buf;
205 void *local_buf;
206 int ret;
207
208
209 if (thread_id == 0 || buf == NULL || bufsz < KDBG_TRIAGE_MAX_STRLEN) {
210 return;
211 }
212
213 local_buf = buf;
214 bzero(local_buf, bufsz);
215
216 record_bytes = record_bufsz = kd_buffer_triage.kdb_event_count * sizeof(kd_buf);
217 record_buf = kalloc_data(record_bufsz, Z_WAITOK);
218
219 if (record_buf == NULL) {
220 ret = ENOMEM;
221 } else {
222 ktriage_lock();
223 ret = kernel_debug_read(&kd_control_triage,
224 &kd_buffer_triage,
225 (user_addr_t) record_buf, &record_bytes, NULL, NULL, 0);
226 ktriage_unlock();
227 }
228
229 if (ret) {
230 printf("ktriage_extract: kernel_debug_read failed with %d\n", ret);
231 kfree_data(record_buf, record_bufsz);
232 return;
233 }
234
235 kd_buf *kd = (kd_buf*) record_buf;
236 i = 0;
237 record_cnt = record_bytes; /* kernel_debug_read() takes number of bytes that it
238 * converts to kd_bufs. It processes a max of those and
239 * returns number of kd_buf read/processed. We use a
240 * different variable here to make our units clear.
241 */
242
243 while (i < record_cnt) {
244 if (kd->arg5 == (uintptr_t)thread_id) {
245 ktriage_convert_to_string(kd->arg4, local_buf, KDBG_TRIAGE_MAX_STRLEN);
246 local_buf = (void *)((uintptr_t)local_buf + KDBG_TRIAGE_MAX_STRLEN);
247 bufsz -= KDBG_TRIAGE_MAX_STRLEN;
248 if (bufsz < KDBG_TRIAGE_MAX_STRLEN) {
249 break;
250 }
251 }
252 i++;
253 kd++;
254 }
255
256 kfree_data(record_buf, record_bufsz);
257 }
258
259 /* KDBG_TRIAGE_CODE_* section */
260 /* VM begin */
261
262 const char *vm_triage_strings[] =
263 {
264 [KDBG_TRIAGE_VM_PREFIX] = "VM - ",
265 [KDBG_TRIAGE_VM_NO_DATA] = "Didn't get back data for this file\n",
266 [KDBG_TRIAGE_VM_TEXT_CORRUPTION] = "A memory corruption was found in executable text\n",
267 [KDBG_TRIAGE_VM_ADDRESS_NOT_FOUND] = "Found no valid range containing this address\n",
268 [KDBG_TRIAGE_VM_PROTECTION_FAILURE] = "Fault hit protection failure\n",
269 [KDBG_TRIAGE_VM_FAULT_MEMORY_SHORTAGE] = "VM Fault hit memory shortage\n",
270 [KDBG_TRIAGE_VM_FAULT_COPY_MEMORY_SHORTAGE] = "vm_fault_copy hit memory shortage\n",
271 [KDBG_TRIAGE_VM_FAULT_OBJCOPYSLOWLY_MEMORY_SHORTAGE] = "vm_object_copy_slowly fault hit memory shortage\n",
272 [KDBG_TRIAGE_VM_FAULT_OBJIOPLREQ_MEMORY_SHORTAGE] = "vm_object_iopl_request fault hit memory shortage\n",
273 [KDBG_TRIAGE_VM_FAULT_INTERRUPTED] = "Fault was interrupted\n",
274 [KDBG_TRIAGE_VM_SUCCESS_NO_PAGE] = "Returned success with no page\n",
275 [KDBG_TRIAGE_VM_GUARDPAGE_FAULT] = "Guard page fault\n",
276 [KDBG_TRIAGE_VM_NONZERO_PREEMPTION_LEVEL] = "Fault entered with non-zero preemption level\n",
277 [KDBG_TRIAGE_VM_BUSYPAGE_WAIT_INTERRUPTED] = "Waiting on busy page was interrupted\n",
278 [KDBG_TRIAGE_VM_PURGEABLE_FAULT_ERROR] = "Purgeable object hit an error in fault\n",
279 [KDBG_TRIAGE_VM_OBJECT_SHADOW_SEVERED] = "Object has a shadow severed\n",
280 [KDBG_TRIAGE_VM_OBJECT_NOT_ALIVE] = "Object is not alive\n",
281 [KDBG_TRIAGE_VM_OBJECT_NO_PAGER] = "Object has no pager\n",
282 [KDBG_TRIAGE_VM_PAGE_HAS_ERROR] = "Page has error bit set\n",
283 [KDBG_TRIAGE_VM_PAGE_HAS_RESTART] = "Page has restart bit set\n",
284 [KDBG_TRIAGE_VM_FAILED_IMMUTABLE_PAGE_WRITE] = "Failed a writable mapping of an immutable page\n",
285 [KDBG_TRIAGE_VM_FAILED_NX_PAGE_EXEC_MAPPING] = "Failed an executable mapping of a nx page\n",
286 [KDBG_TRIAGE_VM_PMAP_ENTER_RESOURCE_SHORTAGE] = "pmap_enter retried due to resource shortage\n",
287 [KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE] = "Compressor offset requested out of range\n",
288 [KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE] = "Compressor doesn't have this page\n",
289 [KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED] = "Decompressor hit a failure\n",
290 [KDBG_TRIAGE_VM_SUBMAP_NO_COW_ON_EXECUTABLE] = "Submap disallowed cow on executable range\n",
291 [KDBG_TRIAGE_VM_SUBMAP_COPY_SLOWLY_FAILED] = "Submap object copy_slowly failed\n",
292 [KDBG_TRIAGE_VM_SUBMAP_COPY_STRAT_FAILED] = "Submap object copy_strategically failed\n",
293 [KDBG_TRIAGE_VM_VNODEPAGER_CLREAD_NO_UPL] = "vnode_pager_cluster_read couldn't create a UPL\n",
294 [KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UBCINFO] = "vnode_pagein got a vnode with no ubcinfo\n",
295 [KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL] = "Filesystem pagein returned an error in vnode_pagein\n",
296 [KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UPL] = "vnode_pagein couldn't create a UPL\n",
297 [KDBG_TRIAGE_VM_ECC_DIRTY] = "Accessed a page that has uncorrected ECC error\n",
298 [KDBG_TRIAGE_VM_ECC_CLEAN] = "Clean page had an uncorrected ECC error\n",
299 };
300 /* VM end */
301
302 /* Cluster begin */
303
304 const char *cluster_triage_strings[] =
305 {
306 [KDBG_TRIAGE_CL_PREFIX] = "CL - ",
307 [KDBG_TRIAGE_CL_PGIN_PAST_EOF] = "cluster_pagein past EOF\n",
308 };
309 /* Cluster end */
310
311 /* Shared Region begin */
312
313 const char *shared_region_triage_strings[] =
314 {
315 [KDBG_TRIAGE_SHARED_REGION_PREFIX] = "SR - ",
316 [KDBG_TRIAGE_SHARED_REGION_NO_UPL] = "shared_region_pager_data_request couldn't create a upl\n",
317 [KDBG_TRIAGE_SHARED_REGION_SLIDE_ERROR] = "shared_region_pager_data_request hit a page sliding error\n",
318 [KDBG_TRIAGE_SHARED_REGION_PAGER_MEMORY_SHORTAGE] = "shared_region_pager_data_request hit memory shortage\n",
319 };
320 /* Shared Region end */
321
322 /* Dyld Pager begin */
323
324 const char *dyld_pager_triage_strings[] =
325 {
326 [KDBG_TRIAGE_DYLD_PAGER_PREFIX] = "DP - ",
327 [KDBG_TRIAGE_DYLD_PAGER_NO_UPL] = "dyld_pager_data_request couldn't create a upl\n",
328 [KDBG_TRIAGE_DYLD_PAGER_SLIDE_ERROR] = "dyld_pager_data_request hit a page sliding error\n",
329 [KDBG_TRIAGE_DYLD_PAGER_MEMORY_SHORTAGE] = "dyld_pager_data_request hit memory shortage\n",
330 };
331 /* Dyld Pager end */
332
333 /* Apple Protect Pager begin */
334
335 const char *apple_protect_pager_triage_strings[] =
336 {
337 [KDBG_TRIAGE_APPLE_PROTECT_PAGER_PREFIX] = "APP - ",
338 [KDBG_TRIAGE_APPLE_PROTECT_PAGER_MEMORY_SHORTAGE] = "apple_protect_pager_data_request hit memory shortage\n",
339 };
340 /* Apple Protect Pager end */
341
342 /* Fourk Pager begin */
343
344 const char *fourk_pager_triage_strings[] =
345 {
346 [KDBG_TRIAGE_FOURK_PAGER_PREFIX] = "FP - ",
347 [KDBG_TRIAGE_FOURK_PAGER_MEMORY_SHORTAGE] = "fourk_pager_data_request hit memory shortage\n",
348 };
349 /* Fourk Pager end */
350
351 /* APFS begin */
352
353 const char *apfs_triage_strings[] =
354 {
355 [KDBG_TRIAGE_APFS_PREFIX] = "APFS - ",
356 [KDBG_TRIAGE_APFS_PAGEIN_NOT_ALLOWED] = "The inode's protection class does not allow page in\n",
357 [KDBG_TRIAGE_APFS_INODE_DEAD] = "The inode has been deleted\n",
358 [KDBG_TRIAGE_APFS_INODE_RAW_ENCRYPTED] = "The inode is raw encrypted, file page in is prohibited\n",
359 [KDBG_TRIAGE_APFS_INODE_OF_RAW_DEVICE] = "The inode backing a raw device, file IO is prohibited\n",
360 [KDBG_TRIAGE_APFS_DISALLOW_READS] = "The APFS_DISALLOW_READS flag is turned on (dataless snapshot mount)\n",
361 [KDBG_TRIAGE_APFS_XATTR_GET_FAILED] = "Could not get namedstream xattr\n",
362 [KDBG_TRIAGE_APFS_NO_NAMEDSTREAM_PARENT_INODE] = "Could not get namedstream parent inode\n",
363 [KDBG_TRIAGE_APFS_INVALID_OFFSET] = "Invalid offset\n",
364 [KDBG_TRIAGE_APFS_COLLECT_HASH_RECORDS] = "Failed collecting all hash records\n",
365 [KDBG_TRIAGE_APFS_INVALID_FILE_INFO] = "Encountered an invalid file info\n",
366 [KDBG_TRIAGE_APFS_NO_HASH_RECORD] = "Verification context for inode is empty\n",
367 [KDBG_TRIAGE_APFS_DATA_HASH_MISMATCH] = "Encountered a data hash mismatch\n",
368 [KDBG_TRIAGE_APFS_COMPRESSED_DATA_HASH_MISMATCH] = "Encountered a compressed data hash mismatch\n",
369 };
370 /* APFS end */
371
372 /* subsystems starts at index 1 */
373 ktriage_strings_t subsystems_triage_strings[KDBG_TRIAGE_SUBSYS_MAX + 1] = {
374 [KDBG_TRIAGE_SUBSYS_VM] = {VM_MAX_TRIAGE_STRINGS, vm_triage_strings},
375 [KDBG_TRIAGE_SUBSYS_CLUSTER] = {CLUSTER_MAX_TRIAGE_STRINGS, cluster_triage_strings},
376 [KDBG_TRIAGE_SUBSYS_SHARED_REGION] = {SHARED_REGION_MAX_TRIAGE_STRINGS, shared_region_triage_strings},
377 [KDBG_TRIAGE_SUBSYS_DYLD_PAGER] = {DYLD_PAGER_MAX_TRIAGE_STRINGS, dyld_pager_triage_strings},
378 [KDBG_TRIAGE_SUBSYS_APPLE_PROTECT_PAGER] = {APPLE_PROTECT_PAGER_MAX_TRIAGE_STRINGS, apple_protect_pager_triage_strings},
379 [KDBG_TRIAGE_SUBSYS_FOURK_PAGER] = {FOURK_PAGER_MAX_TRIAGE_STRINGS, fourk_pager_triage_strings},
380 [KDBG_TRIAGE_SUBSYS_APFS] = {APFS_MAX_TRIAGE_STRINGS, apfs_triage_strings},
381 };
382 /* KDBG_TRIAGE_CODE_* section */
383