1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING
30
31 #include <mach/mach_types.h>
32 #include <kdp/output_stages/output_stages.h>
33 #include <kdp/kdp_core.h>
34 #include <kdp/processor_core.h>
35 #include <IOKit/IOPolledInterface.h>
36 #include <IOKit/IOBSD.h>
37 #include <vm/vm_kern_xnu.h>
38
39 struct disk_stage_data {
40 bool last_operation_was_write;
41 uint64_t current_offset;
42 uint64_t furthest_written_offset;
43 size_t alignment;
44 };
45
46 kern_return_t
disk_stage_write(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,const void * data)47 disk_stage_write(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, const void *data)
48 {
49 kern_return_t err = KERN_SUCCESS;
50
51 assert(stage != NULL);
52 assert(stage->kos_initialized == true);
53
54 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
55 bool already_seeked_this_chunk = false;
56
57 if ((offset < stage_data->furthest_written_offset) || (offset != stage_data->current_offset)) {
58 // We need to seek to the proper offset and prefill the IOPolledInterface internal buffers
59 uint64_t offset_misalignment = offset % stage_data->alignment;
60 uint64_t aligned_offset = offset - offset_misalignment;
61 err = disk_stage_read(stage, offset, 0, NULL);
62 if (kIOReturnSuccess != err) {
63 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during seek) returned 0x%x\n", err);
64 return err;
65 }
66
67 // Now seek back to the aligned offset
68 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
69 if (kIOReturnSuccess != err) {
70 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
71 return err;
72 }
73
74 // Adjust the position forward
75 gIOPolledCoreFileVars->position += offset_misalignment;
76 already_seeked_this_chunk = true;
77 }
78
79 while (KERN_SUCCESS == err && length != 0) {
80 bool read_modify_write = false;
81 uint64_t chunk = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
82 if (chunk > length) {
83 chunk = length;
84
85 // If we're about to write to a region that we've written to before,
86 // we'll need to prefill the IOPolledInterface internal buffers with the contents
87 // of that region
88 if (offset + chunk < stage_data->furthest_written_offset) {
89 read_modify_write = true;
90
91 if (!already_seeked_this_chunk) {
92 uint64_t offset_misalignment = offset % stage_data->alignment;
93 uint64_t aligned_offset = offset - offset_misalignment;
94 err = disk_stage_read(stage, offset, 0, NULL);
95 if (kIOReturnSuccess != err) {
96 kern_coredump_log(NULL, "(disk_stage_write) disk_stage_read (during final chunk seek) returned 0x%x\n", err);
97 break;
98 }
99
100 // Now seek back to the aligned offset
101 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
102 if (kIOReturnSuccess != err) {
103 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
104 break;
105 }
106
107 // Adjust the position forward
108 gIOPolledCoreFileVars->position += offset_misalignment;
109 }
110 }
111 }
112
113 already_seeked_this_chunk = false;
114
115 stage_data->last_operation_was_write = true;
116
117 // Now write the chunk
118 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) chunk, NULL);
119 if (kIOReturnSuccess != err) {
120 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
121 data, chunk, err);
122 break;
123 }
124
125 if (read_modify_write) {
126 // We flush the entirety of the IOPolledInterface buffer back to disk
127 uint32_t remainder = gIOPolledCoreFileVars->bufferLimit - gIOPolledCoreFileVars->bufferOffset;
128 gIOPolledCoreFileVars->bufferOffset += remainder;
129 gIOPolledCoreFileVars->position += remainder;
130 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
131 if (kIOReturnSuccess != err) {
132 kern_coredump_log(NULL, "(disk_stage_write) IOPolledFileWrite (during final flush) returned 0x%x\n", err);
133 break;
134 }
135 }
136
137 data = (const void *) (((uintptr_t) data) + chunk);
138 length -= chunk;
139 offset += chunk;
140 stage_data->current_offset += chunk;
141 if (offset > stage_data->furthest_written_offset) {
142 stage_data->furthest_written_offset = offset;
143 }
144 }
145
146 return err;
147 }
148
149 kern_return_t
disk_stage_read(struct kdp_output_stage * stage,uint64_t offset,uint64_t length,void * data)150 disk_stage_read(struct kdp_output_stage *stage, uint64_t offset, uint64_t length, void *data)
151 {
152 kern_return_t err = KERN_SUCCESS;
153
154 assert(stage != NULL);
155 assert(stage->kos_initialized == true);
156
157 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
158
159 // Flush out any prior data
160 if (stage_data->last_operation_was_write) {
161 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
162 if (kIOReturnSuccess != err) {
163 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileWrite (during seek) returned 0x%x\n", err);
164 return err;
165 }
166 stage_data->last_operation_was_write = false;
167 }
168
169 uint64_t offset_misalignment = offset % stage_data->alignment;
170 uint64_t aligned_offset = offset - offset_misalignment;
171
172 // First seek to the aligned position (this will update the position variable and whatnot)
173 err = IOPolledFileSeek(gIOPolledCoreFileVars, aligned_offset);
174 if (kIOReturnSuccess != err) {
175 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileSeek(0x%llx) returned 0x%x\n", aligned_offset, err);
176 return err;
177 }
178
179 // kick off the read ahead (this is mostly taken from IOHibernateIO.cpp)
180 gIOPolledCoreFileVars->bufferHalf = 0;
181 gIOPolledCoreFileVars->bufferLimit = 0;
182 gIOPolledCoreFileVars->lastRead = 0;
183 gIOPolledCoreFileVars->readEnd = roundup(gIOPolledCoreFileVars->fileSize, stage_data->alignment);
184 gIOPolledCoreFileVars->bufferOffset = 0;
185
186 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, 0, NULL);
187 if (kIOReturnSuccess != err) {
188 kern_coredump_log(NULL, "(disk_stage_read) Kickstarting IOPolledFileRead(0) returned 0x%x\n", err);
189 return err;
190 }
191
192 // This read will (even if offset_misalignment is 0) wait for the previous read to actually complete
193 err = IOPolledFileRead(gIOPolledCoreFileVars, NULL, (IOByteCount) offset_misalignment, NULL);
194 if (kIOReturnSuccess != err) {
195 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", offset_misalignment, err);
196 return err;
197 }
198
199 stage_data->current_offset = offset;
200
201 err = IOPolledFileRead(gIOPolledCoreFileVars, (uint8_t *) data, (IOByteCount) length, NULL);
202 if (kIOReturnSuccess != err) {
203 kern_coredump_log(NULL, "(disk_stage_read) IOPolledFileRead(%llu) returned 0x%x\n", length, err);
204 return err;
205 }
206
207 stage_data->current_offset += length;
208
209 return err;
210 }
211
212 static void
disk_stage_reset(struct kdp_output_stage * stage)213 disk_stage_reset(struct kdp_output_stage *stage)
214 {
215 stage->kos_bypass = false;
216 stage->kos_bytes_written = 0;
217 }
218
219 static kern_return_t
disk_stage_outproc(struct kdp_output_stage * stage,unsigned int request,__unused char * corename,uint64_t length,void * data)220 disk_stage_outproc(struct kdp_output_stage *stage, unsigned int request,
221 __unused char *corename, uint64_t length, void * data)
222 {
223 kern_return_t err = KERN_SUCCESS;
224 struct disk_stage_data *stage_data = (struct disk_stage_data *) stage->kos_data;
225
226 assert(STAILQ_NEXT(stage, kos_next) == NULL);
227
228 switch (request) {
229 case KDP_WRQ:
230 err = IOPolledFileSeek(gIOPolledCoreFileVars, 0);
231 if (kIOReturnSuccess != err) {
232 kern_coredump_log(NULL, "IOPolledFileSeek(gIOPolledCoreFileVars, 0) returned 0x%x\n", err);
233 break;
234 }
235 err = IOPolledFilePollersOpen(gIOPolledCoreFileVars, kIOPolledBeforeSleepState, false);
236 if (kIOReturnSuccess != err) {
237 kern_coredump_log(NULL, "IOPolledFilePollersOpen returned 0x%x\n", err);
238 break;
239 }
240 break;
241
242 case KDP_SEEK:
243 {
244 uint64_t noffset = *((uint64_t *) data);
245 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
246 if (kIOReturnSuccess != err) {
247 kern_coredump_log(NULL, "IOPolledFileWrite (during seek) returned 0x%x\n", err);
248 break;
249 }
250 err = IOPolledFileSeek(gIOPolledCoreFileVars, noffset);
251 if (kIOReturnSuccess != err) {
252 kern_coredump_log(NULL, "IOPolledFileSeek(0x%llx) returned 0x%x\n", noffset, err);
253 }
254 stage_data->current_offset = noffset;
255 break;
256 }
257
258 case KDP_DATA:
259 err = IOPolledFileWrite(gIOPolledCoreFileVars, data, (IOByteCount) length, NULL);
260 if (kIOReturnSuccess != err) {
261 kern_coredump_log(NULL, "IOPolledFileWrite(gIOPolledCoreFileVars, %p, 0x%llx, NULL) returned 0x%x\n",
262 data, length, err);
263 break;
264 }
265 stage_data->last_operation_was_write = true;
266 stage_data->current_offset += length;
267 stage->kos_bytes_written += length;
268 break;
269
270 #if defined(__arm64__)
271 /* Only supported on embedded by the underlying polled mode driver */
272 case KDP_FLUSH:
273 err = IOPolledFileFlush(gIOPolledCoreFileVars);
274 if (kIOReturnSuccess != err) {
275 kern_coredump_log(NULL, "IOPolledFileFlush() returned 0x%x\n", err);
276 break;
277 }
278 break;
279 #endif /* defined(__arm64__) */
280
281 case KDP_EOF:
282 err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL);
283 if (kIOReturnSuccess != err) {
284 kern_coredump_log(NULL, "IOPolledFileWrite (during EOF) returned 0x%x\n", err);
285 break;
286 }
287 err = IOPolledFilePollersClose(gIOPolledCoreFileVars, kIOPolledBeforeSleepState);
288 if (kIOReturnSuccess != err) {
289 kern_coredump_log(NULL, "IOPolledFilePollersClose (during EOF) returned 0x%x\n", err);
290 break;
291 }
292 break;
293 }
294
295 return err;
296 }
297
298 static void
disk_stage_free(struct kdp_output_stage * stage)299 disk_stage_free(struct kdp_output_stage *stage)
300 {
301 kmem_free(kernel_map, (vm_offset_t) stage->kos_data, stage->kos_data_size);
302 stage->kos_data = NULL;
303 stage->kos_data_size = 0;
304 stage->kos_initialized = false;
305 }
306
307 kern_return_t
disk_stage_initialize(struct kdp_output_stage * stage)308 disk_stage_initialize(struct kdp_output_stage *stage)
309 {
310 kern_return_t ret = KERN_SUCCESS;
311 struct disk_stage_data *data = NULL;
312
313 assert(stage != NULL);
314 assert(stage->kos_initialized == false);
315 assert(stage->kos_data == NULL);
316
317 stage->kos_data_size = sizeof(struct disk_stage_data);
318 ret = kmem_alloc(kernel_map, (vm_offset_t*) &stage->kos_data, stage->kos_data_size,
319 KMA_DATA, VM_KERN_MEMORY_DIAG);
320 if (KERN_SUCCESS != ret) {
321 return ret;
322 }
323
324 data = (struct disk_stage_data *) stage->kos_data;
325 data->last_operation_was_write = false;
326 data->current_offset = 0;
327 data->furthest_written_offset = 0;
328 data->alignment = KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN;
329
330 stage->kos_funcs.kosf_reset = disk_stage_reset;
331 stage->kos_funcs.kosf_outproc = disk_stage_outproc;
332 stage->kos_funcs.kosf_free = disk_stage_free;
333
334 stage->kos_initialized = true;
335
336 return KERN_SUCCESS;
337 }
338
339 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
340