1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
30
31 #include <mach/mach_types.h>
32 #include <kdp/output_stages/output_stages.h>
33 #include <kdp/kdp_core.h>
34 #include <kdp/processor_core.h>
35 #include <IOKit/IOPolledInterface.h>
36 #include <IOKit/IOBSD.h>
37
38 struct disk_stage_data {
39 bool last_operation_was_write;
40 uint64_t current_offset;
41 uint64_t furthest_written_offset;
42 size_t alignment;
43 };
44
45 kern_return_t
disk_stage_write(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,const void * data)46 disk_stage_write(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, const void *data)
47 {
48 kern_return_t err = KERN_SUCCESS;
49
50 assert(stage != NULL);
51 assert(stage->kos_initialized == true);
52
53 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
54 bool already_seeked_this_chunk = false;
55
56 if ((offset < stage_data->furthest_written_offset) || (offset != stage_data->current_offset)) {
57 // We need to seek to the proper offset and prefill the IOPolledInterface internal buffers
58 uint64_t offset_misalignment = offset % stage_data->alignment;
59 uint64_t aligned_offset = offset - offset_misalignment;
60 err = disk_stage_read(stage, offset, 0, NULL);
61 if (kIOReturnSuccess != err) {
62 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during seek) returned 0x%x\n", err);
63 return err;
64 }
65
66 // Now seek back to the aligned offset
67 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
68 if (kIOReturnSuccess != err) {
69 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
70 return err;
71 }
72
73 // Adjust the position forward
74 gIOPolledCoreFileVars->position += offset_misalignment;
75 already_seeked_this_chunk = true;
76 }
77
78 while (KERN_SUCCESS == err && length != 0) {
79 bool read_modify_write = false;
80 uint64_t chunk = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
81 if (chunk > length) {
82 chunk = length;
83
84 // If we're about to write to a region that we've written to before,
85 // we'll need to prefill the IOPolledInterface internal buffers with the contents
86 // of that region
87 if (offset + chunk < stage_data->furthest_written_offset) {
88 read_modify_write = true;
89
90 if (!already_seeked_this_chunk) {
91 uint64_t offset_misalignment = offset % stage_data->alignment;
92 uint64_t aligned_offset = offset - offset_misalignment;
93 err = disk_stage_read(stage, offset, 0, NULL);
94 if (kIOReturnSuccess != err) {
95 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during final chunk seek) returned 0x%x\n", err);
96 break;
97 }
98
99 // Now seek back to the aligned offset
100 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
101 if (kIOReturnSuccess != err) {
102 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
103 break;
104 }
105
106 // Adjust the position forward
107 gIOPolledCoreFileVars->position += offset_misalignment;
108 }
109 }
110 }
111
112 already_seeked_this_chunk = false;
113
114 stage_data->last_operation_was_write = true;
115
116 // Now write the chunk
117 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) chunk, NULL);
118 if (kIOReturnSuccess != err) {
119 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
120 data, chunk, err);
121 break;
122 }
123
124 if (read_modify_write) {
125 // We flush the entirety of the IOPolledInterface buffer back to disk
126 uint32_t remainder = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
127 gIOPolledCoreFileVars->bufferOffset += remainder;
128 gIOPolledCoreFileVars->position += remainder;
129 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
130 if (kIOReturnSuccess != err) {
131 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite (during final flush) returned 0x%x\n", err);
132 break;
133 }
134 }
135
136 data = (const void *) (((uintptr_t) data) + chunk);
137 length -= chunk;
138 offset += chunk;
139 stage_data->current_offset += chunk;
140 if (offset > stage_data->furthest_written_offset) {
141 stage_data->furthest_written_offset = offset;
142 }
143 }
144
145 return err;
146 }
147
148 kern_return_t
disk_stage_read(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,void * data)149 disk_stage_read(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, void *data)
150 {
151 kern_return_t err = KERN_SUCCESS;
152
153 assert(stage != NULL);
154 assert(stage->kos_initialized == true);
155
156 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
157
158 // Flush out any prior data
159 if (stage_data->last_operation_was_write) {
160 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
161 if (kIOReturnSuccess != err) {
162 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileWrite (during seek) returned 0x%x\n", err);
163 return err;
164 }
165 stage_data->last_operation_was_write = false;
166 }
167
168 uint64_t offset_misalignment = offset % stage_data->alignment;
169 uint64_t aligned_offset = offset - offset_misalignment;
170
171 // First seek to the aligned position (this will update the position variable and whatnot)
172 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
173 if (kIOReturnSuccess != err) {
174 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
175 return err;
176 }
177
178 // kick off the read ahead (this is mostly taken from IOHibernateIO.cpp)
179 gIOPolledCoreFileVars->bufferHalf = 0;
180 gIOPolledCoreFileVars->bufferLimit = 0;
181 gIOPolledCoreFileVars->lastRead = 0;
182 gIOPolledCoreFileVars->readEnd = roundup(gIOPolledCoreFileVars->fileSize, stage_data->alignment);
183 gIOPolledCoreFileVars->bufferOffset = 0;
184
185 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, 0, NULL);
186 if (kIOReturnSuccess != err) {
187 kern_coredump_log(NULL, "(disk_stage_read) Kickstarting IOPolledFileRead(0) returned 0x%x\n", err);
188 return err;
189 }
190
191 // This read will (even if offset_misalignment is 0) wait for the previous read to actually complete
192 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, (IOByteCount) offset_misalignment, NULL);
193 if (kIOReturnSuccess != err) {
194 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", offset_misalignment, err);
195 return err;
196 }
197
198 stage_data->current_offset = offset;
199
200 err = IOPolledFileRead(gIOPolledCoreFileVars, (uint8_t *) data, (IOByteCount) length, NULL);
201 if (kIOReturnSuccess != err) {
202 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", length, err);
203 return err;
204 }
205
206 stage_data->current_offset += length;
207
208 return err;
209 }
210
211 static void
disk_stage_reset(struct kdp_output_stage * stage)212 disk_stage_reset(struct kdp_output_stage *stage)
213 {
214 stage->kos_bypass = false;
215 stage->kos_bytes_written = 0;
216 }
217
218 static kern_return_t
disk_stage_outproc(struct kdp_output_stage * stage,unsigned int request,__unused char * corename,uint64_t length,void * data)219 disk_stage_outproc(struct kdp_output_stage *stage, unsigned int request,
220 __unused char *corename, uint64_t length, void * data)
221 {
222 kern_return_t err = KERN_SUCCESS;
223 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
224
225 assert(STAILQ_NEXT(stage, kos_next) == NULL);
226
227 switch (request) {
228 case KDP_WRQ:
229 err = IOPolledFileSeek(gIOPolledCoreFileVars, 0);
230 if (kIOReturnSuccess != err) {
231 kern_coredump_log(NULL, "IOPolledFileSeek(gIOPolledCoreFileVars, 0) returned 0x%x\n", err);
232 break;
233 }
234 err = IOPolledFilePollersOpen(gIOPolledCoreFileVars, kIOPolledBeforeSleepState, false);
235 if (kIOReturnSuccess != err) {
236 kern_coredump_log(NULL, "IOPolledFilePollersOpen returned 0x%x\n", err);
237 break;
238 }
239 break;
240
241 case KDP_SEEK:
242 {
243 uint64_t noffset = *((uint64_t *) data);
244 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
245 if (kIOReturnSuccess != err) {
246 kern_coredump_log(NULL, "IOPolledFileWrite (during seek) returned 0x%x\n", err);
247 break;
248 }
249 err = IOPolledFileSeek(gIOPolledCoreFileVars, noffset);
250 if (kIOReturnSuccess != err) {
251 kern_coredump_log(NULL, "IOPolledFileSeek(0x%llx) returned 0x%x\n", noffset, err);
252 }
253 stage_data->current_offset = noffset;
254 break;
255 }
256
257 case KDP_DATA:
258 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) length, NULL);
259 if (kIOReturnSuccess != err) {
260 kern_coredump_log(NULL, "IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
261 data, length, err);
262 break;
263 }
264 stage_data->last_operation_was_write = true;
265 stage_data->current_offset += length;
266 stage->kos_bytes_written += length;
267 break;
268
269 #if defined(__arm64__)
270 /* Only supported on embedded by the underlying polled mode driver */
271 case KDP_FLUSH:
272 err = IOPolledFileFlush(gIOPolledCoreFileVars);
273 if (kIOReturnSuccess != err) {
274 kern_coredump_log(NULL, "IOPolledFileFlush() returned 0x%x\n", err);
275 break;
276 }
277 break;
278 #endif /* defined(__arm64__) */
279
280 case KDP_EOF:
281 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
282 if (kIOReturnSuccess != err) {
283 kern_coredump_log(NULL, "IOPolledFileWrite (during EOF) returned 0x%x\n", err);
284 break;
285 }
286 err = IOPolledFilePollersClose(gIOPolledCoreFileVars, kIOPolledBeforeSleepState);
287 if (kIOReturnSuccess != err) {
288 kern_coredump_log(NULL, "IOPolledFilePollersClose (during EOF) returned 0x%x\n", err);
289 break;
290 }
291 break;
292 }
293
294 return err;
295 }
296
297 static void
disk_stage_free(struct kdp_output_stage * stage)298 disk_stage_free(struct kdp_output_stage *stage)
299 {
300 kmem_free(kernel_map, (vm_offset_t) stage->kos_data, stage->kos_data_size);
301 stage->kos_data = NULL;
302 stage->kos_data_size = 0;
303 stage->kos_initialized = false;
304 }
305
306 kern_return_t
disk_stage_initialize(struct kdp_output_stage * stage)307 disk_stage_initialize(struct kdp_output_stage *stage)
308 {
309 kern_return_t ret = KERN_SUCCESS;
310 struct disk_stage_data *data = NULL;
311
312 assert(stage != NULL);
313 assert(stage->kos_initialized == false);
314 assert(stage->kos_data == NULL);
315
316 stage->kos_data_size = sizeof(struct disk_stage_data);
317 ret = kmem_alloc(kernel_map, (vm_offset_t*) &stage->kos_data, stage->kos_data_size,
318 KMA_DATA, VM_KERN_MEMORY_DIAG);
319 if (KERN_SUCCESS != ret) {
320 return ret;
321 }
322
323 data = (struct disk_stage_data *) stage->kos_data;
324 data->last_operation_was_write = false;
325 data->current_offset = 0;
326 data->furthest_written_offset = 0;
327 data->alignment = KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN;
328
329 stage->kos_funcs.kosf_reset = disk_stage_reset;
330 stage->kos_funcs.kosf_outproc = disk_stage_outproc;
331 stage->kos_funcs.kosf_free = disk_stage_free;
332
333 stage->kos_initialized = true;
334
335 return KERN_SUCCESS;
336 }
337
338 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
339