1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <skywalk/packet/pbufpool_var.h>
31
32 static errno_t kern_pbufpool_alloc_common(const kern_pbufpool_t,
33 const uint32_t, kern_packet_t *, uint32_t);
34 static errno_t kern_pbufpool_alloc_batch_common(const kern_pbufpool_t,
35 const uint32_t, kern_packet_t *__counted_by(*size), uint32_t *size,
36 alloc_cb_func_t, const void *, uint32_t);
37
38 #define KBI_INVALID_CB_PAIRS(cb1, cb2) \
39 (!(init->kbi_##cb1 == NULL && init->kbi_##cb2 == NULL) && \
40 ((init->kbi_##cb1 == NULL) ^ (init->kbi_##cb2 == NULL)))
41
42 errno_t
kern_pbufpool_create(const struct kern_pbufpool_init * init,kern_pbufpool_t * ppp,struct kern_pbufpool_memory_info * pp_info)43 kern_pbufpool_create(const struct kern_pbufpool_init *init,
44 kern_pbufpool_t *ppp, struct kern_pbufpool_memory_info *pp_info)
45 {
46 /* XXX: woodford_s - find a way to get 'srp' off the kernel stack */
47 struct skmem_region_params srp[SKMEM_REGIONS];
48 struct kern_pbufpool *pp = NULL;
49 nexus_meta_type_t md_type;
50 nexus_meta_subtype_t md_subtype;
51 uint32_t buf_cnt;
52 uint16_t max_frags;
53 uint32_t ppcreatef = PPCREATEF_EXTERNAL;
54 uint32_t pkt_cnt;
55 uint32_t pp_region_flags = 0;
56 int err = 0;
57 bool kernel_only;
58 bool tx_pool = true;
59
60 if (ppp == NULL || init == NULL ||
61 init->kbi_version != KERN_PBUFPOOL_CURRENT_VERSION ||
62 init->kbi_packets == 0 || (init->kbi_buflets != 0 &&
63 init->kbi_buflets < init->kbi_packets &&
64 !(init->kbi_flags & KBIF_BUFFER_ON_DEMAND)) ||
65 init->kbi_bufsize == 0 || init->kbi_max_frags == 0 ||
66 ((init->kbi_flags & KBIF_QUANTUM) &&
67 (init->kbi_flags & KBIF_BUFFER_ON_DEMAND)) ||
68 KBI_INVALID_CB_PAIRS(buf_seg_ctor, buf_seg_dtor)) {
69 err = EINVAL;
70 goto done;
71 }
72
73 *ppp = NULL;
74
75 md_type = ((init->kbi_flags & KBIF_QUANTUM) ?
76 NEXUS_META_TYPE_QUANTUM : NEXUS_META_TYPE_PACKET);
77
78 /*
79 * If packet, we assume this is for a driver handling raw frames.
80 * This also implies that at present, we do not create mirrored
81 * regions for user space to conserve memory (since those regions
82 * aren't going to be used anyway.)
83 *
84 * XXX: [email protected] - to allow for "direct" channels from
85 * user process to driver, we will need to revisit this.
86 */
87 md_subtype = ((md_type == NEXUS_META_TYPE_QUANTUM) ?
88 NEXUS_META_SUBTYPE_PAYLOAD : NEXUS_META_SUBTYPE_RAW);
89 kernel_only = (md_type == NEXUS_META_TYPE_PACKET) &&
90 #if (DEVELOPMENT || DEBUG)
91 !skywalk_netif_direct_enabled() &&
92 #endif /* (DEVELOPMENT || DEBUG) */
93 ((init->kbi_flags & KBIF_USER_ACCESS) == 0);
94
95 VERIFY((init->kbi_max_frags != 0) &&
96 (init->kbi_max_frags <= UINT16_MAX));
97 max_frags = (uint16_t)init->kbi_max_frags;
98 if (md_type == NEXUS_META_TYPE_QUANTUM && max_frags > 1) {
99 err = EINVAL;
100 goto done;
101 }
102 if ((max_frags > 1) && !(init->kbi_flags & KBIF_BUFFER_ON_DEMAND)) {
103 err = EINVAL;
104 goto done;
105 }
106
107 bzero(&srp, sizeof(srp));
108 for (int i = 0; i < SKMEM_REGIONS; i++) {
109 srp[i] = *skmem_get_default(i);
110 }
111
112 switch (init->kbi_flags & (KBIF_IODIR_IN | KBIF_IODIR_OUT)) {
113 case KBIF_IODIR_IN:
114 pp_region_flags |= PP_REGION_CONFIG_BUF_IODIR_IN;
115 tx_pool = false;
116 break;
117 case KBIF_IODIR_OUT:
118 pp_region_flags |= PP_REGION_CONFIG_BUF_IODIR_OUT;
119 break;
120 case (KBIF_IODIR_IN | KBIF_IODIR_OUT):
121 default:
122 pp_region_flags |= PP_REGION_CONFIG_BUF_IODIR_BIDIR;
123 break;
124 }
125
126 if (init->kbi_flags & KBIF_BUFFER_ON_DEMAND) {
127 pp_region_flags |= PP_REGION_CONFIG_BUFLET;
128 }
129 if (kernel_only) {
130 pp_region_flags |= PP_REGION_CONFIG_KERNEL_ONLY;
131 }
132 if (init->kbi_flags & KBIF_KERNEL_READONLY) {
133 pp_region_flags |= PP_REGION_CONFIG_BUF_KREADONLY;
134 }
135 if (init->kbi_flags & KBIF_THREADSAFE) {
136 pp_region_flags |= PP_REGION_CONFIG_BUF_THREADSAFE;
137 }
138 /*
139 * Enable magazine layer for metadata.
140 */
141 if (!(init->kbi_flags & KBIF_NO_MAGAZINES)) {
142 pp_region_flags |= PP_REGION_CONFIG_MD_MAGAZINE_ENABLE;
143 }
144 pp_region_flags |= PP_REGION_CONFIG_MD_PERSISTENT;
145
146 pkt_cnt = init->kbi_packets;
147 /*
148 * For TCP to be able to send a 4MB window worth of data, packet pool
149 * must have at least 4MB/MTU packets. On devices which are not
150 * memory constrained, we can increase the pool to be atleast
151 * 4K packets.
152 */
153 if (tx_pool && !SKMEM_MEM_CONSTRAINED_DEVICE() &&
154 #if (DEVELOPMENT || DEBUG)
155 !skmem_test_enabled() &&
156 #endif /* (DEVELOPMENT || DEBUG) */
157 !(init->kbi_flags & KBIF_MONOLITHIC) &&
158 !(init->kbi_flags & KBIF_VIRTUAL_DEVICE) &&
159 !(init->kbi_flags & KBIF_PHYS_CONTIGUOUS) &&
160 !(init->kbi_flags & KBIF_KERNEL_READONLY) &&
161 !(init->kbi_flags & KBIF_QUANTUM)) {
162 pkt_cnt = MAX((4 * 1024), pkt_cnt);
163 }
164 #if (DEVELOPMENT || DEBUG)
165 if (sk_min_pool_size != 0) {
166 pkt_cnt = MAX(pkt_cnt, sk_min_pool_size);
167 }
168 #endif /* (DEVELOPMENT || DEBUG) */
169 /* make sure # of buffers is >= # of packets */
170 buf_cnt = MAX(pkt_cnt, init->kbi_buflets);
171
172 /*
173 * Apply same logic as in nxprov_create_common().
174 */
175 if (init->kbi_flags &
176 (KBIF_PERSISTENT | KBIF_MONOLITHIC | KBIF_INHIBIT_CACHE |
177 KBIF_PHYS_CONTIGUOUS)) {
178 if (init->kbi_flags & KBIF_PERSISTENT) {
179 pp_region_flags |= PP_REGION_CONFIG_BUF_PERSISTENT;
180 }
181 if (init->kbi_flags & KBIF_MONOLITHIC) {
182 pp_region_flags |= PP_REGION_CONFIG_BUF_MONOLITHIC;
183 }
184 if (init->kbi_flags & KBIF_INHIBIT_CACHE) {
185 pp_region_flags |= PP_REGION_CONFIG_BUF_NOCACHE;
186 }
187 if (init->kbi_flags & KBIF_PHYS_CONTIGUOUS) {
188 pp_region_flags |= PP_REGION_CONFIG_BUF_SEGPHYSCONTIG;
189 }
190 }
191
192 /* adjust region params */
193 pp_regions_params_adjust(srp, md_type, md_subtype, pkt_cnt, max_frags,
194 init->kbi_bufsize, 0, buf_cnt, init->kbi_buf_seg_size,
195 pp_region_flags);
196
197 /*
198 * Create packet pool.
199 */
200 ASSERT(ppcreatef & PPCREATEF_EXTERNAL);
201 if (kernel_only) {
202 ppcreatef |= PPCREATEF_KERNEL_ONLY;
203 }
204 if (init->kbi_flags & KBIF_BUFFER_ON_DEMAND) {
205 ppcreatef |= PPCREATEF_ONDEMAND_BUF;
206 }
207 /*
208 * Enable CPU-layer magazine resizing if this is a long-lived
209 * pbufpool, e.g. one that's allocated by a device driver.
210 */
211 if (!(init->kbi_flags & KBIF_VIRTUAL_DEVICE)) {
212 ppcreatef |= PPCREATEF_DYNAMIC;
213 }
214 if ((pp = pp_create(
215 __unsafe_null_terminated_from_indexable(init->kbi_name), srp,
216 init->kbi_buf_seg_ctor, init->kbi_buf_seg_dtor,
217 init->kbi_ctx, init->kbi_ctx_retain, init->kbi_ctx_release,
218 ppcreatef)) == NULL) {
219 err = ENOMEM;
220 goto done;
221 }
222
223 *ppp = pp;
224
225 if (pp_info != NULL) {
226 err = kern_pbufpool_get_memory_info(pp, pp_info);
227 VERIFY(err == 0);
228 }
229
230 done:
231 if (err != 0 && pp != NULL) {
232 /* callee drops reference */
233 pp_close(pp);
234 pp = NULL;
235 }
236
237 return err;
238 }
239
240 /*
241 * -fbounds-safety: This function is mainly used by kexts in C++, which we're
242 * not doing bound checks yet. So just leave it as __single
243 */
244 void *__single
kern_pbufpool_get_context(const kern_pbufpool_t pp)245 kern_pbufpool_get_context(const kern_pbufpool_t pp)
246 {
247 void *__single ctx = (pp->pp_flags & PPF_EXTERNAL) ? pp->pp_ctx : NULL;
248 if (ctx != NULL) {
249 pp->pp_ctx_retain(ctx);
250 }
251 return ctx;
252 }
253
254 errno_t
kern_pbufpool_get_memory_info(const kern_pbufpool_t pp,struct kern_pbufpool_memory_info * pp_info)255 kern_pbufpool_get_memory_info(const kern_pbufpool_t pp,
256 struct kern_pbufpool_memory_info *pp_info)
257 {
258 if (pp_info == NULL) {
259 return EINVAL;
260 }
261
262 bzero(pp_info, sizeof(*pp_info));
263 if (pp->pp_flags & PPF_EXTERNAL) {
264 pp_info->kpm_flags |= KPMF_EXTERNAL;
265 }
266 pp_info->kpm_packets = pp->pp_kmd_region->skr_c_obj_cnt;
267 pp_info->kpm_max_frags = pp->pp_max_frags;
268 pp_info->kpm_buflets = PP_BUF_REGION_DEF(pp)->skr_c_obj_cnt;
269 pp_info->kpm_bufsize = PP_BUF_SIZE_DEF(pp);
270 pp_info->kpm_buf_obj_size = PP_BUF_OBJ_SIZE_DEF(pp);
271 pp_info->kpm_bufsegs = PP_BUF_REGION_DEF(pp)->skr_seg_max_cnt;
272 pp_info->kpm_buf_seg_size = PP_BUF_REGION_DEF(pp)->skr_seg_size;
273
274 return 0;
275 }
276
277 kern_segment_idx_t
kern_segment_get_index(const kern_segment_t seg)278 kern_segment_get_index(const kern_segment_t seg)
279 {
280 return seg->sg_index;
281 }
282
283 static errno_t
kern_pbufpool_alloc_common(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * pph,uint32_t skmflag)284 kern_pbufpool_alloc_common(const kern_pbufpool_t pp, const uint32_t bufcnt,
285 kern_packet_t *pph, uint32_t skmflag)
286 {
287 struct __kern_quantum *kqum;
288
289 *pph = 0;
290
291 if (__improbable(bufcnt > pp->pp_max_frags)) {
292 return EINVAL;
293 }
294
295 if (__improbable((bufcnt != pp->pp_max_frags) &&
296 !PP_HAS_BUFFER_ON_DEMAND(pp))) {
297 return EINVAL;
298 }
299
300 kqum = SK_PTR_ADDR_KQUM(pp_alloc_packet(pp, (uint16_t)bufcnt, skmflag));
301 if (__probable(kqum != NULL)) {
302 *pph = SK_PTR_ENCODE(kqum, METADATA_TYPE(kqum),
303 METADATA_SUBTYPE(kqum));
304 }
305
306 return (kqum != NULL) ? 0 : ENOMEM;
307 }
308
309 errno_t
kern_pbufpool_alloc(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * pph)310 kern_pbufpool_alloc(const kern_pbufpool_t pp, const uint32_t bufcnt,
311 kern_packet_t *pph)
312 {
313 return kern_pbufpool_alloc_common(pp, bufcnt, pph, SKMEM_SLEEP);
314 }
315
316 errno_t
kern_pbufpool_alloc_nosleep(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * pph)317 kern_pbufpool_alloc_nosleep(const kern_pbufpool_t pp, const uint32_t bufcnt,
318 kern_packet_t *pph)
319 {
320 return kern_pbufpool_alloc_common(pp, bufcnt, pph, SKMEM_NOSLEEP);
321 }
322
323 static errno_t
kern_pbufpool_alloc_batch_common(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * __counted_by (* size)array,uint32_t * size,alloc_cb_func_t cb,const void * ctx,uint32_t skmflag)324 kern_pbufpool_alloc_batch_common(const kern_pbufpool_t pp,
325 const uint32_t bufcnt, kern_packet_t *__counted_by(*size)array,
326 uint32_t *size, alloc_cb_func_t cb, const void *ctx, uint32_t skmflag)
327 {
328 if (__improbable(array == NULL || size == NULL || *size == 0 ||
329 bufcnt > pp->pp_max_frags || (cb == NULL && ctx != NULL))) {
330 return EINVAL;
331 }
332
333 if (__improbable((bufcnt != pp->pp_max_frags) &&
334 !PP_HAS_BUFFER_ON_DEMAND(pp))) {
335 return EINVAL;
336 }
337
338 return pp_alloc_packet_batch(pp, (uint16_t)bufcnt, array, size, TRUE,
339 cb, ctx, skmflag);
340 }
341
342 errno_t
kern_pbufpool_alloc_batch(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * __counted_by (* size)array,uint32_t * size)343 kern_pbufpool_alloc_batch(const kern_pbufpool_t pp, const uint32_t bufcnt,
344 kern_packet_t *__counted_by(*size)array, uint32_t *size)
345 {
346 return kern_pbufpool_alloc_batch_common(pp, bufcnt, array,
347 size, NULL, NULL, SKMEM_SLEEP);
348 }
349
350 errno_t
kern_pbufpool_alloc_batch_callback(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * __counted_by (* size)array,uint32_t * size,alloc_cb_func_t cb,const void * ctx)351 kern_pbufpool_alloc_batch_callback(const kern_pbufpool_t pp,
352 const uint32_t bufcnt, kern_packet_t *__counted_by(*size)array,
353 uint32_t *size, alloc_cb_func_t cb, const void *ctx)
354 {
355 return kern_pbufpool_alloc_batch_common(pp, bufcnt, array,
356 size, cb, ctx, SKMEM_SLEEP);
357 }
358
359 errno_t
kern_pbufpool_alloc_batch_nosleep(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * __counted_by (* size)array,uint32_t * size)360 kern_pbufpool_alloc_batch_nosleep(const kern_pbufpool_t pp,
361 const uint32_t bufcnt, kern_packet_t *__counted_by(*size)array,
362 uint32_t *size)
363 {
364 return kern_pbufpool_alloc_batch_common(pp, bufcnt, array,
365 size, NULL, NULL, SKMEM_NOSLEEP);
366 }
367
368 errno_t
kern_pbufpool_alloc_batch_nosleep_callback(const kern_pbufpool_t pp,const uint32_t bufcnt,kern_packet_t * __counted_by (* size)array,uint32_t * size,alloc_cb_func_t cb,const void * ctx)369 kern_pbufpool_alloc_batch_nosleep_callback(const kern_pbufpool_t pp,
370 const uint32_t bufcnt, kern_packet_t *__counted_by(*size)array,
371 uint32_t *size, alloc_cb_func_t cb, const void *ctx)
372 {
373 return kern_pbufpool_alloc_batch_common(pp, bufcnt, array,
374 size, cb, ctx, SKMEM_NOSLEEP);
375 }
376
377 void
kern_pbufpool_free(const kern_pbufpool_t pp,kern_packet_t ph)378 kern_pbufpool_free(const kern_pbufpool_t pp, kern_packet_t ph)
379 {
380 pp_free_packet(pp, SK_PTR_ADDR(ph));
381 }
382
383 void
kern_pbufpool_free_batch(const kern_pbufpool_t pp,kern_packet_t * __counted_by (size)array,uint32_t size)384 kern_pbufpool_free_batch(const kern_pbufpool_t pp,
385 kern_packet_t *__counted_by(size)array, uint32_t size)
386 {
387 if (__improbable(array == NULL || size == 0)) {
388 return;
389 }
390
391 pp_free_packet_batch(pp, array, size);
392 }
393
394 void
kern_pbufpool_free_chain(const kern_pbufpool_t pp,kern_packet_t chain)395 kern_pbufpool_free_chain(const kern_pbufpool_t pp, kern_packet_t chain)
396 {
397 struct __kern_packet *pkt_chain = SK_PTR_ADDR_KPKT(chain);
398
399 VERIFY(pp == pkt_chain->pkt_qum.qum_pp);
400 pp_free_packet_chain(pkt_chain, NULL);
401 }
402
403 errno_t
kern_pbufpool_alloc_buffer(const kern_pbufpool_t pp,mach_vm_address_t * buf,kern_segment_t * sg,kern_obj_idx_seg_t * sg_idx)404 kern_pbufpool_alloc_buffer(const kern_pbufpool_t pp, mach_vm_address_t *buf,
405 kern_segment_t *sg, kern_obj_idx_seg_t *sg_idx)
406 {
407 return pp_alloc_buffer(pp, buf, sg, sg_idx, 0);
408 }
409
410
411 errno_t
kern_pbufpool_alloc_buffer_nosleep(const kern_pbufpool_t pp,mach_vm_address_t * buf,kern_segment_t * sg,kern_obj_idx_seg_t * sg_idx)412 kern_pbufpool_alloc_buffer_nosleep(const kern_pbufpool_t pp,
413 mach_vm_address_t *buf, kern_segment_t *sg, kern_obj_idx_seg_t *sg_idx)
414 {
415 return pp_alloc_buffer(pp, buf, sg, sg_idx, SKMEM_NOSLEEP);
416 }
417
418 void
kern_pbufpool_free_buffer(const kern_pbufpool_t pp,mach_vm_address_t baddr)419 kern_pbufpool_free_buffer(const kern_pbufpool_t pp, mach_vm_address_t baddr)
420 {
421 pp_free_buffer(pp, baddr);
422 }
423
424 void
kern_pbufpool_destroy(kern_pbufpool_t pp)425 kern_pbufpool_destroy(kern_pbufpool_t pp)
426 {
427 VERIFY(pp->pp_flags & PPF_EXTERNAL);
428 pp_close(pp);
429 }
430
431 errno_t
kern_pbufpool_alloc_buflet(const kern_pbufpool_t pp,kern_buflet_t * pbuf)432 kern_pbufpool_alloc_buflet(const kern_pbufpool_t pp, kern_buflet_t *pbuf)
433 {
434 return pp_alloc_buflet(pp, pbuf, SKMEM_SLEEP, false);
435 }
436
437 errno_t
kern_pbufpool_alloc_buflet_nosleep(const kern_pbufpool_t pp,kern_buflet_t * pbuf)438 kern_pbufpool_alloc_buflet_nosleep(const kern_pbufpool_t pp,
439 kern_buflet_t *pbuf)
440 {
441 return pp_alloc_buflet(pp, pbuf, SKMEM_NOSLEEP, false);
442 }
443