1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if defined(CONFIG_KDP_INTERACTIVE_DEBUGGING)
30
31 #include <mach/kern_return.h>
32 #include <os/base.h>
33 #include <kern/assert.h>
34 #include <kern/kalloc.h>
35 #include <kern/misc_protos.h>
36 #include <kdp/kdp_core.h>
37 #include <kdp/kdp_out_stage.h>
38 #include <kdp/processor_core.h>
39 #include <kdp/output_stages/output_stages.h>
40 #include <libkern/compression/compression.h>
41
42 // One page seems to be a sweet-spot for both buffers; we tend to dump a page or less at a time so
43 // smaller sizes start incurring overhead for the loop, and larger sizes waste more memory without
44 // a noticable increase in throughput.
45 //
46 // Having a buffer full of zeroes wastes a bit of memory, but we need some way to pass zeroes to
47 // Compression when we are asked to pad the stream.
48 #define DST_BUF_SIZE PAGE_SIZE
49 #define ZERO_BUF_SIZE PAGE_SIZE
50
51 struct lz4_stage_data {
52 compression_stream_t stream;
53 compression_algorithm_t algorithm;
54 uint8_t *dst_buf;
55 const uint8_t *zero_buf;
56 bool reset_failed;
57 };
58
59 static kern_return_t
lz4_stage_reset(struct kdp_output_stage * stage,__unused const char * corename,__unused kern_coredump_type_t coretype)60 lz4_stage_reset(struct kdp_output_stage *stage, __unused const char *corename, __unused kern_coredump_type_t coretype)
61 {
62 struct lz4_stage_data *data;
63 compression_status_t status;
64
65 assert(compression_ki_ptr);
66
67 assert(stage);
68 assert(stage->kos_initialized);
69
70 data = stage->kos_data;
71 assert(data);
72
73 status = compression_ki_ptr->compression_stream_reinit(&data->stream,
74 COMPRESSION_STREAM_ENCODE, data->algorithm);
75 if (COMPRESSION_STATUS_OK != status) {
76 data->reset_failed = true;
77 }
78
79 data->stream.dst_ptr = data->dst_buf;
80 data->stream.dst_size = DST_BUF_SIZE;
81
82 stage->kos_bypass = false;
83 stage->kos_bytes_written = 0;
84
85 return KERN_SUCCESS;
86 }
87
88 static kern_return_t
lz4_stage_stream(struct lz4_stage_data * data,struct kdp_output_stage * next_stage,char * corename,const uint8_t * src_buf,size_t src_buf_size,uint64_t * written)89 lz4_stage_stream(struct lz4_stage_data *data, struct kdp_output_stage *next_stage, char *corename,
90 const uint8_t *src_buf, size_t src_buf_size, uint64_t *written)
91 {
92 bool finalize;
93 compression_status_t status;
94 kern_return_t ret;
95 size_t produced;
96
97 assert(compression_ki_ptr);
98
99 assert(data);
100 assert(next_stage);
101 assert(written);
102
103 finalize = !src_buf && !src_buf_size;
104 assert((src_buf && src_buf_size) || finalize);
105
106 data->stream.src_ptr = src_buf;
107 data->stream.src_size = src_buf_size;
108
109 // Every time we process more bytes, the stream source and destination pointers
110 // and sizes are adjusted.
111 do {
112 status = compression_ki_ptr->compression_stream_process(&data->stream,
113 finalize ? COMPRESSION_STREAM_FINALIZE : 0);
114 if (COMPRESSION_STATUS_ERROR == status) {
115 kern_coredump_log(NULL, "(%s) compression_stream_process failed\n", __func__);
116 return KERN_FAILURE;
117 }
118
119 // The difference between the original destination buffer size and the size of the
120 // remaining space tells us how many bytes were produced.
121 produced = DST_BUF_SIZE - data->stream.dst_size;
122 // Pass along those bytes to the next stage so that we empty the destination buffer.
123 ret = next_stage->kos_funcs.kosf_outproc(next_stage, KDP_DATA, corename,
124 produced, data->dst_buf);
125 if (KERN_SUCCESS != ret) {
126 kern_coredump_log(NULL, "(%s) next stage output failed with error 0x%x\n", __func__, ret);
127 return ret;
128 }
129 *written += produced;
130 data->stream.dst_ptr = data->dst_buf;
131 data->stream.dst_size = DST_BUF_SIZE;
132 // Continue processing while the source buffer is non-empty, or we are finalizing
133 // and there are still bytes being produced in the destination buffer.
134 } while (data->stream.src_size || (finalize && COMPRESSION_STATUS_END != status));
135
136 if (finalize) {
137 ret = next_stage->kos_funcs.kosf_outproc(next_stage, KDP_DATA, corename, 0, NULL);
138 if (KERN_SUCCESS != ret) {
139 kern_coredump_log(NULL, "(%s) next stage output failed with error 0x%x\n", __func__, ret);
140 }
141 return ret;
142 }
143
144 return KERN_SUCCESS;
145 }
146
147 static kern_return_t
lz4_stage_outproc(struct kdp_output_stage * stage,unsigned int request,char * corename,uint64_t length,void * panic_data)148 lz4_stage_outproc(struct kdp_output_stage *stage, unsigned int request,
149 char *corename, uint64_t length, void *panic_data)
150 {
151 struct lz4_stage_data *data;
152 struct kdp_output_stage *next_stage;
153 kern_return_t ret;
154 size_t pad_length;
155 size_t zero_size;
156
157 assert(stage);
158 assert(stage->kos_initialized);
159
160 data = stage->kos_data;
161 assert(data);
162
163 next_stage = STAILQ_NEXT(stage, kos_next);
164 assert(next_stage);
165
166 if (data->reset_failed) {
167 return KERN_FAILURE;
168 }
169
170 if (KDP_SEEK == request) {
171 stage->kos_bypass = true;
172 }
173
174 if (stage->kos_bypass || KDP_DATA != request) {
175 ret = next_stage->kos_funcs.kosf_outproc(next_stage, request, corename, length,
176 panic_data);
177 if (KERN_SUCCESS != ret) {
178 kern_coredump_log(NULL, "(%s) next stage output failed with error 0x%x\n", __func__, ret);
179 }
180 return ret;
181 }
182
183 if (panic_data) {
184 // Write panic data to the stream.
185 ret = lz4_stage_stream(data, next_stage, corename, panic_data, (size_t)length,
186 &stage->kos_bytes_written);
187 if (KERN_SUCCESS != ret) {
188 kern_coredump_log(NULL, "(%s) lz4_stage_stream failed with error 0x%x\n", __func__, ret);
189 }
190 return ret;
191 }
192
193 if (length) {
194 // Pad the stream with zeroes.
195 pad_length = (size_t)length;
196 do {
197 zero_size = MIN(pad_length, ZERO_BUF_SIZE);
198 ret = lz4_stage_stream(data, next_stage, corename, data->zero_buf,
199 zero_size, &stage->kos_bytes_written);
200 if (KERN_SUCCESS != ret) {
201 kern_coredump_log(NULL, "(%s) lz4_stage_stream failed with error 0x%x\n", __func__, ret);
202 return ret;
203 }
204 pad_length -= zero_size;
205 } while (pad_length);
206 return KERN_SUCCESS;
207 }
208
209 // Finalize the stream.
210 ret = lz4_stage_stream(data, next_stage, corename, NULL, 0, &stage->kos_bytes_written);
211 if (KERN_SUCCESS != ret) {
212 kern_coredump_log(NULL, "(%s) lz4_stage_stream failed with error 0x%x\n", __func__, ret);
213 }
214
215 return ret;
216 }
217
218 static void
lz4_stage_free(struct kdp_output_stage * stage)219 lz4_stage_free(struct kdp_output_stage *stage)
220 {
221 struct lz4_stage_data *data;
222
223 assert(compression_ki_ptr);
224
225 assert(stage);
226 assert(stage->kos_initialized);
227
228 data = stage->kos_data;
229 assert(data);
230
231 kfree_data(data->dst_buf, DST_BUF_SIZE);
232
233 compression_ki_ptr->compression_stream_destroy(&data->stream);
234
235 kfree_type(typeof(*data), data);
236
237 stage->kos_initialized = false;
238 }
239
240 kern_return_t
lz4_stage_initialize(struct kdp_output_stage * stage)241 lz4_stage_initialize(struct kdp_output_stage *stage)
242 {
243 struct lz4_stage_data *data;
244 compression_status_t status;
245
246 assert(compression_ki_ptr);
247
248 assert(stage);
249 assert(!stage->kos_initialized);
250 assert(!stage->kos_data);
251
252 data = kalloc_type(typeof(*data), Z_WAITOK);
253 assert(data);
254
255 data->algorithm = COMPRESSION_LZ4;
256
257 data->dst_buf = kalloc_data(DST_BUF_SIZE, Z_WAITOK);
258 assert(data->dst_buf);
259
260 data->zero_buf = kalloc_data(ZERO_BUF_SIZE, Z_WAITOK | Z_ZERO);
261 assert(data->zero_buf);
262
263 data->reset_failed = false;
264
265 status = compression_ki_ptr->compression_stream_init(&data->stream, COMPRESSION_STREAM_ENCODE,
266 data->algorithm);
267 if (COMPRESSION_STATUS_ERROR == status) {
268 return KERN_FAILURE;
269 }
270 data->stream.dst_ptr = data->dst_buf;
271 data->stream.dst_size = DST_BUF_SIZE;
272
273 stage->kos_data = data;
274 stage->kos_data_size = sizeof(*data);
275
276 stage->kos_funcs.kosf_reset = lz4_stage_reset;
277 stage->kos_funcs.kosf_outproc = lz4_stage_outproc;
278 stage->kos_funcs.kosf_free = lz4_stage_free;
279
280 stage->kos_initialized = true;
281
282 return KERN_SUCCESS;
283 }
284
285 static void
lz4_stage_registration_callback(void)286 lz4_stage_registration_callback(void)
287 {
288 kern_return_t ret = kdp_core_handle_lz4_available();
289 if (KERN_SUCCESS != ret) {
290 printf("(%s) Failed to handle availability of LZ4. Error 0x%x\n", __func__, ret);
291 }
292 }
293
294 void
lz4_stage_monitor_availability(void)295 lz4_stage_monitor_availability(void)
296 {
297 compression_interface_set_registration_callback(lz4_stage_registration_callback);
298 }
299
300 #endif /* defined(CONFIG_KDP_INTERACTIVE_DEBUGGING) */
301