1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
30
31 #include <mach/mach_types.h>
32 #include <kdp/output_stages/output_stages.h>
33 #include <kdp/kdp_core.h>
34 #include <kdp/processor_core.h>
35 #include <IOKit/IOPolledInterface.h>
36 #include <IOKit/IOBSD.h>
37 #include <vm/vm_kern_xnu.h>
38
39 struct disk_stage_data {
40 bool last_operation_was_write;
41 uint64_t current_offset;
42 uint64_t furthest_written_offset;
43 size_t alignment;
44 };
45
46 kern_return_t
disk_stage_write(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,const void * data)47 disk_stage_write(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, const void *data)
48 {
49 kern_return_t err = KERN_SUCCESS;
50
51 assert(stage != NULL);
52 assert(stage->kos_initialized == true);
53
54 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
55 bool already_seeked_this_chunk = false;
56
57 if ((offset < stage_data->furthest_written_offset) || (offset != stage_data->current_offset)) {
58 // We need to seek to the proper offset and prefill the IOPolledInterface internal buffers
59 uint64_t offset_misalignment = offset % stage_data->alignment;
60 uint64_t aligned_offset = offset - offset_misalignment;
61 err = disk_stage_read(stage, offset, 0, NULL);
62 if (kIOReturnSuccess != err) {
63 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during seek) returned 0x%x\n", err);
64 return err;
65 }
66
67 // Now seek back to the aligned offset
68 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
69 if (kIOReturnSuccess != err) {
70 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
71 return err;
72 }
73
74 // Adjust the position forward
75 gIOPolledCoreFileVars->position += offset_misalignment;
76 already_seeked_this_chunk = true;
77 }
78
79 while (KERN_SUCCESS == err && length != 0) {
80 bool read_modify_write = false;
81 uint64_t chunk = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
82 if (chunk > length) {
83 chunk = length;
84
85 // If we're about to write to a region that we've written to before,
86 // we'll need to prefill the IOPolledInterface internal buffers with the contents
87 // of that region
88 if (offset + chunk < stage_data->furthest_written_offset) {
89 read_modify_write = true;
90
91 if (!already_seeked_this_chunk) {
92 uint64_t offset_misalignment = offset % stage_data->alignment;
93 uint64_t aligned_offset = offset - offset_misalignment;
94 err = disk_stage_read(stage, offset, 0, NULL);
95 if (kIOReturnSuccess != err) {
96 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during final chunk seek) returned 0x%x\n", err);
97 break;
98 }
99
100 // Now seek back to the aligned offset
101 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
102 if (kIOReturnSuccess != err) {
103 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
104 break;
105 }
106
107 // Adjust the position forward
108 gIOPolledCoreFileVars->position += offset_misalignment;
109 }
110 }
111 }
112
113 already_seeked_this_chunk = false;
114
115 stage_data->last_operation_was_write = true;
116
117 // Now write the chunk
118 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) chunk, NULL);
119 if (kIOReturnSuccess != err) {
120 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
121 data, chunk, err);
122 break;
123 }
124
125 if (read_modify_write) {
126 // We flush the entirety of the IOPolledInterface buffer back to disk
127 uint32_t remainder = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
128 gIOPolledCoreFileVars->bufferOffset += remainder;
129 gIOPolledCoreFileVars->position += remainder;
130 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
131 if (kIOReturnSuccess != err) {
132 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite (during final flush) returned 0x%x\n", err);
133 break;
134 }
135 }
136
137 data = (const void *) (((uintptr_t) data) + chunk);
138 length -= chunk;
139 offset += chunk;
140 stage_data->current_offset += chunk;
141 if (offset > stage_data->furthest_written_offset) {
142 stage_data->furthest_written_offset = offset;
143 }
144 }
145
146 return err;
147 }
148
149 kern_return_t
disk_stage_read(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,void * data)150 disk_stage_read(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, void *data)
151 {
152 kern_return_t err = KERN_SUCCESS;
153
154 assert(stage != NULL);
155 assert(stage->kos_initialized == true);
156
157 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
158
159 // Flush out any prior data
160 if (stage_data->last_operation_was_write) {
161 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
162 if (kIOReturnSuccess != err) {
163 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileWrite (during seek) returned 0x%x\n", err);
164 return err;
165 }
166 stage_data->last_operation_was_write = false;
167 }
168
169 uint64_t offset_misalignment = offset % stage_data->alignment;
170 uint64_t aligned_offset = offset - offset_misalignment;
171
172 // First seek to the aligned position (this will update the position variable and whatnot)
173 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
174 if (kIOReturnSuccess != err) {
175 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
176 return err;
177 }
178
179 // kick off the read ahead (this is mostly taken from IOHibernateIO.cpp)
180 gIOPolledCoreFileVars->bufferHalf = 0;
181 gIOPolledCoreFileVars->bufferLimit = 0;
182 gIOPolledCoreFileVars->lastRead = 0;
183 gIOPolledCoreFileVars->readEnd = roundup(gIOPolledCoreFileVars->fileSize, stage_data->alignment);
184 gIOPolledCoreFileVars->bufferOffset = 0;
185
186 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, 0, NULL);
187 if (kIOReturnSuccess != err) {
188 kern_coredump_log(NULL, "(disk_stage_read) Kickstarting IOPolledFileRead(0) returned 0x%x\n", err);
189 return err;
190 }
191
192 // This read will (even if offset_misalignment is 0) wait for the previous read to actually complete
193 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, (IOByteCount) offset_misalignment, NULL);
194 if (kIOReturnSuccess != err) {
195 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", offset_misalignment, err);
196 return err;
197 }
198
199 stage_data->current_offset = offset;
200
201 err = IOPolledFileRead(gIOPolledCoreFileVars, (uint8_t *) data, (IOByteCount) length, NULL);
202 if (kIOReturnSuccess != err) {
203 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", length, err);
204 return err;
205 }
206
207 stage_data->current_offset += length;
208
209 return err;
210 }
211
212 static kern_return_t
disk_stage_reset(struct kdp_output_stage * stage,__unused const char * corename,__unused kern_coredump_type_t coretype)213 disk_stage_reset(struct kdp_output_stage *stage, __unused const char *corename, __unused kern_coredump_type_t coretype)
214 {
215 stage->kos_bypass = false;
216 stage->kos_bytes_written = 0;
217
218 return KERN_SUCCESS;
219 }
220
221 static kern_return_t
disk_stage_outproc(struct kdp_output_stage * stage,unsigned int request,__unused char * corename,uint64_t length,void * data)222 disk_stage_outproc(struct kdp_output_stage *stage, unsigned int request,
223 __unused char *corename, uint64_t length, void * data)
224 {
225 kern_return_t err = KERN_SUCCESS;
226 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
227
228 assert(STAILQ_NEXT(stage, kos_next) == NULL);
229
230 switch (request) {
231 case KDP_WRQ:
232 err = IOPolledFileSeek(gIOPolledCoreFileVars, 0);
233 if (kIOReturnSuccess != err) {
234 kern_coredump_log(NULL, "IOPolledFileSeek(gIOPolledCoreFileVars, 0) returned 0x%x\n", err);
235 break;
236 }
237 err = IOPolledFilePollersOpen(gIOPolledCoreFileVars, kIOPolledBeforeSleepState, false);
238 if (kIOReturnSuccess != err) {
239 kern_coredump_log(NULL, "IOPolledFilePollersOpen returned 0x%x\n", err);
240 break;
241 }
242 break;
243
244 case KDP_SEEK:
245 {
246 uint64_t noffset = *((uint64_t *) data);
247 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
248 if (kIOReturnSuccess != err) {
249 kern_coredump_log(NULL, "IOPolledFileWrite (during seek) returned 0x%x\n", err);
250 break;
251 }
252 err = IOPolledFileSeek(gIOPolledCoreFileVars, noffset);
253 if (kIOReturnSuccess != err) {
254 kern_coredump_log(NULL, "IOPolledFileSeek(0x%llx) returned 0x%x\n", noffset, err);
255 }
256 stage_data->current_offset = noffset;
257 break;
258 }
259
260 case KDP_DATA:
261 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) length, NULL);
262 if (kIOReturnSuccess != err) {
263 kern_coredump_log(NULL, "IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
264 data, length, err);
265 break;
266 }
267 stage_data->last_operation_was_write = true;
268 stage_data->current_offset += length;
269 stage->kos_bytes_written += length;
270 break;
271
272 #if defined(__arm64__)
273 /* Only supported on embedded by the underlying polled mode driver */
274 case KDP_FLUSH:
275 err = IOPolledFileFlush(gIOPolledCoreFileVars);
276 if (kIOReturnSuccess != err) {
277 kern_coredump_log(NULL, "IOPolledFileFlush() returned 0x%x\n", err);
278 break;
279 }
280 break;
281 #endif /* defined(__arm64__) */
282
283 case KDP_EOF:
284 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
285 if (kIOReturnSuccess != err) {
286 kern_coredump_log(NULL, "IOPolledFileWrite (during EOF) returned 0x%x\n", (uint32_t)err);
287 break;
288 }
289 err = IOPolledFilePollersClose(gIOPolledCoreFileVars, kIOPolledBeforeSleepState);
290 if (kIOReturnSuccess != err) {
291 kern_coredump_log(NULL, "IOPolledFilePollersClose (during EOF) returned 0x%x\n", (uint32_t)err);
292 break;
293 }
294 break;
295 }
296
297 return err;
298 }
299
300 static void
disk_stage_free(struct kdp_output_stage * stage)301 disk_stage_free(struct kdp_output_stage *stage)
302 {
303 kmem_free(kernel_map, (vm_offset_t) stage->kos_data, stage->kos_data_size);
304 stage->kos_data = NULL;
305 stage->kos_data_size = 0;
306 stage->kos_initialized = false;
307 }
308
309 kern_return_t
disk_stage_initialize(struct kdp_output_stage * stage)310 disk_stage_initialize(struct kdp_output_stage *stage)
311 {
312 kern_return_t ret = KERN_SUCCESS;
313 struct disk_stage_data *data = NULL;
314
315 assert(stage != NULL);
316 assert(stage->kos_initialized == false);
317 assert(stage->kos_data == NULL);
318
319 stage->kos_data_size = sizeof(struct disk_stage_data);
320 ret = kmem_alloc(kernel_map, (vm_offset_t*) &stage->kos_data, stage->kos_data_size,
321 KMA_DATA_SHARED, VM_KERN_MEMORY_DIAG);
322 if (KERN_SUCCESS != ret) {
323 return ret;
324 }
325
326 data = (struct disk_stage_data *) stage->kos_data;
327 data->last_operation_was_write = false;
328 data->current_offset = 0;
329 data->furthest_written_offset = 0;
330 data->alignment = KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN;
331
332 stage->kos_funcs.kosf_reset = disk_stage_reset;
333 stage->kos_funcs.kosf_outproc = disk_stage_outproc;
334 stage->kos_funcs.kosf_free = disk_stage_free;
335
336 stage->kos_initialized = true;
337
338 return KERN_SUCCESS;
339 }
340
341 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
342