1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @Apple_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
21 */
22
23 #include <sys/kdebug_common.h>
24 #include <sys/kdebug_triage.h>
25
26 #define TRIAGE_KDCOPYBUF_COUNT 128
27 #define TRIAGE_KDCOPYBUF_SIZE (TRIAGE_KDCOPYBUF_COUNT * sizeof(kd_buf))
28
29 struct kd_ctrl_page_t kd_ctrl_page_triage = {
30 .kds_free_list = {.raw = KDS_PTR_NULL},
31 .mode = KDEBUG_MODE_TRIAGE,
32 .kdebug_events_per_storage_unit = TRIAGE_EVENTS_PER_STORAGE_UNIT,
33 .kdebug_min_storage_units_per_cpu = TRIAGE_MIN_STORAGE_UNITS_PER_CPU,
34 .kdebug_kdcopybuf_count = TRIAGE_KDCOPYBUF_COUNT,
35 .kdebug_kdcopybuf_size = TRIAGE_KDCOPYBUF_SIZE,
36 .kdebug_flags = KDBG_DEBUGID_64,
37 .kdebug_slowcheck = SLOW_NOLOG,
38 .oldest_time = 0
39 };
40
41 struct kd_data_page_t kd_data_page_triage = {
42 .nkdbufs = 0,
43 .n_storage_units = 0,
44 .n_storage_threshold = 0,
45 .n_storage_buffer = 0,
46 .kdbip = NULL,
47 .kd_bufs = NULL,
48 .kdcopybuf = NULL
49 };
50
51 static LCK_GRP_DECLARE(ktriage_grp, "ktriage");
52 static LCK_MTX_DECLARE(ktriage_mtx, &ktriage_grp);
53
54 static void
ktriage_lock(void)55 ktriage_lock(void)
56 {
57 lck_mtx_lock(&ktriage_mtx);
58 }
59
60 static void
ktriage_unlock(void)61 ktriage_unlock(void)
62 {
63 lck_mtx_unlock(&ktriage_mtx);
64 }
65
66 int
create_buffers_triage(bool early_trace)67 create_buffers_triage(bool early_trace)
68 {
69 int error = 0;
70 int events_per_storage_unit, min_storage_units_per_cpu;
71
72 if (kd_ctrl_page_triage.kdebug_flags & KDBG_BUFINIT) {
73 panic("create_buffers_triage shouldn't be called once we have inited the triage system.");
74 }
75
76 events_per_storage_unit = kd_ctrl_page_triage.kdebug_events_per_storage_unit;
77 min_storage_units_per_cpu = kd_ctrl_page_triage.kdebug_min_storage_units_per_cpu;
78
79 kd_ctrl_page_triage.kdebug_cpus = kdbg_cpu_count(early_trace);
80 kd_ctrl_page_triage.kdebug_iops = NULL;
81
82 if (kd_data_page_triage.nkdbufs < (kd_ctrl_page_triage.kdebug_cpus * events_per_storage_unit * min_storage_units_per_cpu)) {
83 kd_data_page_triage.n_storage_units = kd_ctrl_page_triage.kdebug_cpus * min_storage_units_per_cpu;
84 } else {
85 kd_data_page_triage.n_storage_units = kd_data_page_triage.nkdbufs / events_per_storage_unit;
86 }
87
88 kd_data_page_triage.nkdbufs = kd_data_page_triage.n_storage_units * events_per_storage_unit;
89
90 kd_data_page_triage.kd_bufs = NULL;
91
92 error = create_buffers(&kd_ctrl_page_triage, &kd_data_page_triage, VM_KERN_MEMORY_TRIAGE);
93
94 if (!error) {
95 kd_ctrl_page_triage.oldest_time = mach_continuous_time();
96 kd_ctrl_page_triage.enabled = 1;
97 kd_data_page_triage.n_storage_threshold = kd_data_page_triage.n_storage_units / 2;
98 }
99
100 return error;
101 }
102
103 __attribute__((noreturn))
104 void
delete_buffers_triage(void)105 delete_buffers_triage(void)
106 {
107 /*
108 * If create_buffers() for triage mode fails, it will call the generic delete_buffers() to
109 * free the resources. This specific call should never be invoked because we expect the
110 * triage system to always be ON.
111 */
112 panic("delete_buffers_triage shouldn't be invoked");
113 }
114
115 const char *ktriage_error_index_invalid_str = "Error descr index invalid\n";
116 const char *ktriage_error_subsyscode_invalid_str = "KTriage: Subsystem code invalid\n";
117
118 static void
kernel_triage_convert_to_string(uint64_t debugid,char * buf)119 kernel_triage_convert_to_string(uint64_t debugid, char *buf)
120 {
121 if (buf == NULL) {
122 return;
123 }
124
125 int subsystem = KDBG_TRIAGE_EXTRACT_CLASS(debugid);
126 size_t prefixlen, msglen;
127
128 if (subsystem <= KDBG_TRIAGE_SUBSYS_MAX) {
129 switch (subsystem) {
130 case KDBG_TRIAGE_SUBSYS_VM:
131 {
132 prefixlen = strlen(*vm_triage_strings[0]);
133 strlcpy(buf, (const char*)(*vm_triage_strings[0]), prefixlen + 1); /* we'll overwrite NULL with rest of string below */
134
135 int strindx = KDBG_TRIAGE_EXTRACT_CODE(debugid);
136 if (strindx >= 1) { /* 0 is reserved for prefix */
137 if (strindx < VM_MAX_TRIAGE_STRINGS) {
138 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(*vm_triage_strings[strindx]) + 1); /* incl. NULL termination */
139 strlcpy(buf + prefixlen, (const char*)(*vm_triage_strings[strindx]), msglen);
140 } else {
141 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
142 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
143 }
144 } else {
145 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
146 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
147 }
148 break;
149 }
150
151 case KDBG_TRIAGE_SUBSYS_CLUSTER:
152 {
153 prefixlen = strlen(*cluster_triage_strings[0]);
154 strlcpy(buf, (const char*)(*cluster_triage_strings[0]), prefixlen + 1); /* we'll overwrite NULL with rest of string below */
155
156 int strindx = KDBG_TRIAGE_EXTRACT_CODE(debugid);
157 if (strindx >= 1) { /* 0 is reserved for prefix */
158 if (strindx < CLUSTER_MAX_TRIAGE_STRINGS) {
159 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(*cluster_triage_strings[strindx]) + 1); /* incl. NULL termination */
160 strlcpy(buf + prefixlen, (const char*)(*cluster_triage_strings[strindx]), msglen);
161 } else {
162 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
163 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
164 }
165 } else {
166 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
167 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
168 }
169 break;
170 }
171 case KDBG_TRIAGE_SUBSYS_SHARED_REGION:
172 {
173 prefixlen = strlen(*shared_region_triage_strings[0]);
174 strlcpy(buf, (const char*)(*shared_region_triage_strings[0]), prefixlen + 1); /* we'll overwrite NULL with rest of string below */
175
176 int strindx = KDBG_TRIAGE_EXTRACT_CODE(debugid);
177 if (strindx >= 1) { /* 0 is reserved for prefix */
178 if (strindx < SHARED_REGION_MAX_TRIAGE_STRINGS) {
179 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(*shared_region_triage_strings[strindx]) + 1); /* incl. NULL termination */
180 strlcpy(buf + prefixlen, (const char*)(*shared_region_triage_strings[strindx]), msglen);
181 } else {
182 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
183 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
184 }
185 } else {
186 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN - prefixlen, strlen(ktriage_error_index_invalid_str) + 1); /* incl. NULL termination */
187 strlcpy(buf + prefixlen, ktriage_error_index_invalid_str, msglen);
188 }
189 break;
190 }
191 default:
192 break;
193 }
194 ;
195 } else {
196 msglen = MIN(KDBG_TRIAGE_MAX_STRLEN, strlen(ktriage_error_subsyscode_invalid_str) + 1); /* incl. NULL termination */
197 strlcpy(buf, ktriage_error_subsyscode_invalid_str, msglen);
198 }
199
200 return;
201 }
202
203 void
kernel_triage_record(uint64_t thread_id,uint64_t debugid,uintptr_t arg)204 kernel_triage_record(
205 uint64_t thread_id,
206 uint64_t debugid,
207 uintptr_t arg)
208 {
209 struct kd_record kd_rec;
210
211 if (thread_id == 0) {
212 thread_id = thread_tid(current_thread());
213 }
214
215 kd_rec.cpu = -1;
216 kd_rec.timestamp = -1;
217
218 /*
219 * use 64-bit debugid per our flag KDBG_DEBUGID_64
220 * that is set in kd_ctrl_page_triage (on LP64 only).
221 */
222 assert(kd_ctrl_page_triage.kdebug_flags & KDBG_DEBUGID_64);
223
224 kd_rec.debugid = 0;
225 kd_rec.arg4 = (uintptr_t)debugid;
226
227 kd_rec.arg1 = arg;
228 kd_rec.arg2 = 0;
229 kd_rec.arg3 = 0;
230 kd_rec.arg5 = (uintptr_t)thread_id;
231
232 kernel_debug_write(&kd_ctrl_page_triage,
233 &kd_data_page_triage,
234 kd_rec);
235 }
236
237 void
kernel_triage_extract(uint64_t thread_id,void * buf,uint32_t bufsz)238 kernel_triage_extract(
239 uint64_t thread_id,
240 void *buf,
241 uint32_t bufsz)
242 {
243 size_t i, record_bytes, record_cnt, record_bufsz;
244 void *record_buf;
245 void *local_buf;
246 int ret;
247
248
249 if (thread_id == 0 || buf == NULL || bufsz < KDBG_TRIAGE_MAX_STRLEN) {
250 return;
251 }
252
253 local_buf = buf;
254 bzero(local_buf, bufsz);
255
256 record_bytes = record_bufsz = kd_data_page_triage.nkdbufs * sizeof(kd_buf);
257 record_buf = kalloc_data(record_bufsz, Z_WAITOK);
258
259 if (record_buf == NULL) {
260 ret = ENOMEM;
261 } else {
262 ktriage_lock();
263 ret = kernel_debug_read(&kd_ctrl_page_triage,
264 &kd_data_page_triage,
265 (user_addr_t) record_buf, &record_bytes, NULL, NULL, 0);
266 ktriage_unlock();
267 }
268
269 if (ret) {
270 printf("kernel_triage_extract: kernel_debug_read failed with %d\n", ret);
271 kfree_data(record_buf, record_bufsz);
272 return;
273 }
274
275 kd_buf *kd = (kd_buf*) record_buf;
276 i = 0;
277 record_cnt = record_bytes; /* kernel_debug_read() takes number of bytes that it
278 * converts to kd_bufs. It processes a max of those and
279 * returns number of kd_buf read/processed. We use a
280 * different variable here to make our units clear.
281 */
282
283 while (i < record_cnt) {
284 if (kd->arg5 == (uintptr_t)thread_id) {
285 kernel_triage_convert_to_string(kd->arg4, local_buf);
286 local_buf = (void *)((uintptr_t)local_buf + KDBG_TRIAGE_MAX_STRLEN);
287 bufsz -= KDBG_TRIAGE_MAX_STRLEN;
288 if (bufsz < KDBG_TRIAGE_MAX_STRLEN) {
289 break;
290 }
291 }
292 i++;
293 kd++;
294 }
295
296 kfree_data(record_buf, record_bufsz);
297 }
298
299
300 /* KDBG_TRIAGE_CODE_* section */
301 /* VM begin */
302
303 const char *vm_triage_strings[VM_MAX_TRIAGE_STRINGS][KDBG_TRIAGE_MAX_STRLEN] =
304 {
305 {"VM - "}, /* VM message prefix */
306 {"Didn't get back data for this file\n"}, /* KDBG_TRIAGE_VM_NO_DATA */
307 {"A memory corruption was found in executable text\n"}, /* KDBG_TRIAGE_VM_TEXT_CORRUPTION */
308 {"Found no valid range containing this address\n"}, /* KDBG_TRIAGE_VM_ADDRESS_NOT_FOUND */
309 {"Fault hit protection failure\n"}, /* KDBG_TRIAGE_VM_PROTECTION_FAILURE */
310 {"Fault hit memory shortage\n"}, /* KDBG_TRIAGE_VM_MEMORY_SHORTAGE */
311 {"Fault was interrupted\n"}, /* KDBG_TRIAGE_VM_FAULT_INTERRUPTED */
312 {"Returned success with no page\n"}, /* KDBG_TRIAGE_VM_SUCCESS_NO_PAGE */
313 {"Guard page fault\n"}, /* KDBG_TRIAGE_VM_GUARDPAGE_FAULT */
314 {"Fault entered with non-zero preemption level\n"}, /* KDBG_TRIAGE_VM_NONZERO_PREEMPTION_LEVEL */
315 {"Waiting on busy page was interrupted\n"}, /* KDBG_TRIAGE_VM_BUSYPAGE_WAIT_INTERRUPTED */
316 {"Purgeable object hit an error in fault\n"}, /* KDBG_TRIAGE_VM_PURGEABLE_FAULT_ERROR */
317 {"Object has a shadow severed\n"}, /* KDBG_TRIAGE_VM_OBJECT_SHADOW_SEVERED */
318 {"Object is not alive\n"}, /* KDBG_TRIAGE_VM_OBJECT_NOT_ALIVE */
319 {"Object has no pager\n"}, /* KDBG_TRIAGE_VM_OBJECT_NO_PAGER */
320 {"Page has error bit set\n"}, /* KDBG_TRIAGE_VM_PAGE_HAS_ERROR */
321 {"Page has restart bit set\n"}, /* KDBG_TRIAGE_VM_PAGE_HAS_RESTART */
322 {"Failed a writable mapping of an immutable page\n"}, /* KDBG_TRIAGE_VM_FAILED_IMMUTABLE_PAGE_WRITE */
323 {"Failed an executable mapping of a nx page\n"}, /* KDBG_TRIAGE_VM_FAILED_NX_PAGE_EXEC_MAPPING */
324 {"pmap_enter failed with resource shortage\n"}, /* KDBG_TRIAGE_VM_PMAP_ENTER_RESOURCE_SHORTAGE */
325 {"Compressor offset requested out of range\n"}, /* KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE */
326 {"Compressor doesn't have this page\n"}, /* KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE */
327 {"Decompressor hit a failure\n"}, /* KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED */
328 {"Compressor failed a blocking pager_get\n"}, /* KDBG_TRIAGE_VM_COMPRESSOR_BLOCKING_OP_FAILED */
329 {"Submap disallowed cow on executable range\n"}, /* KDBG_TRIAGE_VM_SUBMAP_NO_COW_ON_EXECUTABLE */
330 {"Submap object copy_slowly failed\n"}, /* KDBG_TRIAGE_VM_SUBMAP_COPY_SLOWLY_FAILED */
331 {"Submap object copy_strategically failed\n"}, /* KDBG_TRIAGE_VM_SUBMAP_COPY_STRAT_FAILED */
332 {"vnode_pager_cluster_read couldn't create a UPL\n"}, /* KDBG_TRIAGE_VM_VNODEPAGER_CLREAD_NO_UPL */
333 {"vnode_pagein got a vnode with no ubcinfo\n"}, /* KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UBCINFO */
334 {"Filesystem pagein returned an error in vnode_pagein\n"}, /* KDBG_TRIAGE_VM_VNODEPAGEIN_FSPAGEIN_FAIL */
335 {"vnode_pagein couldn't create a UPL\n"} /* KDBG_TRIAGE_VM_VNODEPAGEIN_NO_UPL*/
336 };
337 /* VM end */
338
339 /* Cluster begin */
340
341 const char *cluster_triage_strings[CLUSTER_MAX_TRIAGE_STRINGS][KDBG_TRIAGE_MAX_STRLEN] =
342 {
343 {"CL - "}, /* Cluster message prefix */
344 {"cluster_pagein past EOF\n"} /* KDBG_TRIAGE_CL_PGIN_PAST_EOF */
345 };
346 /* Cluster end */
347
348 /* Shared Region begin */
349
350 const char *shared_region_triage_strings[SHARED_REGION_MAX_TRIAGE_STRINGS][KDBG_TRIAGE_MAX_STRLEN] =
351 {
352 {"SR - "}, /* Shared region message prefix */
353 {"shared_region_pager_data_request couldn't create a upl\n"}, /* KDBG_TRIAGE_SHARED_REGION_NO_UPL */
354 {"shared_region_pager_data_request hit a page sliding error\n"} /* KDBG_TRIAGE_SHARED_REGION_SLIDE_ERROR */
355 };
356 /* Shared Region end */
357 /* KDBG_TRIAGE_CODE_* section */
358