1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 #include <stdlib.h>
31 #include <stddef.h>
32 #include <string.h>
33 #include <strings.h>
34 #include <unistd.h>
35 #include <errno.h>
36 #include <skywalk/os_skywalk_private.h>
37 #include <skywalk/os_packet_private.h>
38
39 #ifndef LIBSYSCALL_INTERFACE
40 #error "LIBSYSCALL_INTERFACE not defined"
41 #endif /* !LIBSYSCALL_INTERFACE */
42
43 /*
44 * Defined here as we don't have Libc
45 */
46 extern int __getpid(void);
47 extern int __kill(int pid, int signum, int posix);
48 extern int __exit(int) __attribute__((noreturn));
49
50 static ring_id_t _ring_id(struct ch_info *cinfo, const ring_id_type_t type);
51 static void os_channel_info2attr(struct channel *chd, channel_attr_t cha);
52 static int _flowadv_id_equal(struct __flowadv_entry *, uuid_t);
53
54 #if defined(__arm__) || defined(__arm64__)
55 __attribute__((always_inline, visibility("hidden")))
56 static inline void
membar_sync(void)57 membar_sync(void)
58 {
59 __asm__ volatile ("dmb ish" ::: "memory");
60 }
61 #elif defined(__i386__) || defined(__x86_64__)
62 __attribute__((always_inline, visibility("hidden")))
63 static inline void
membar_sync(void)64 membar_sync(void)
65 {
66 __asm__ volatile ("mfence" ::: "memory");
67 }
68 #else /* !__arm__ && !__arm64__ && !__i386__ && !__x86_64__ */
69 #error "Unknown platform; membar_sync() not available"
70 #endif /* !__arm__ && !__arm64__ && !__i386__ && !__x86_64__ */
71
72 /*
73 * This is pretty much what an inlined memcmp() would do for UUID
74 * comparison; since we don't have access to memcmp() here, we
75 * manually handle it ourselves.
76 */
77 #define UUID_COMPARE(a, b) \
78 (a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3] && \
79 a[4] == b[4] && a[5] == b[5] && a[6] == b[6] && a[7] == b[7] && \
80 a[8] == b[8] && a[9] == b[9] && a[10] == b[10] && a[11] == b[11] && \
81 a[12] == b[12] && a[13] == b[13] && a[14] == b[14] && a[15] == b[15])
82
83 #define _SLOT_INDEX(_chrd, _slot) \
84 ((slot_idx_t)((_slot - (_chrd)->chrd_slot_desc)))
85
86 #define _SLOT_DESC(_chrd, _idx) \
87 (SLOT_DESC_USD(&(_chrd)->chrd_slot_desc[_idx]))
88
89 #define _METADATA(_chrd, _ring, _midx) \
90 ((void *)((_chrd)->chrd_md_base_addr + \
91 ((_midx) * (_ring)->ring_md_size) + METADATA_PREAMBLE_SZ))
92
93 #define _SLOT_METADATA(_chrd, _ring, _idx) \
94 _METADATA(_chrd, _ring, _SLOT_DESC(_chrd, _idx)->sd_md_idx)
95
96 #define _SLOT_METADATA_IDX_VERIFY(_chrd, _md, _midx) do { \
97 if (__improbable((_md) != _METADATA((_chrd), (_chrd)->chrd_ring, \
98 (_midx))) && !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \
99 SK_ABORT_WITH_CAUSE("bad packet handle", (_midx)); \
100 /* NOTREACHED */ \
101 __builtin_unreachable(); \
102 } \
103 } while (0)
104
105 #define _BFT_INDEX(_chrd, _bft) (_bft)->buf_bft_idx_reg
106
107 #define _SLOT_BFT_METADATA(_chrd, _ring, _idx) \
108 _CHANNEL_RING_BFT(_chrd, _ring, _SLOT_DESC(_chrd, _idx)->sd_md_idx)
109
110 #define _SLOT_BFT_METADATA_IDX_VERIFY(_chrd, _md, _midx) do { \
111 if (__improbable((mach_vm_address_t)(_md) != \
112 _CHANNEL_RING_BFT((_chrd), (_chrd)->chrd_ring, (_midx))) && \
113 !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \
114 SK_ABORT_WITH_CAUSE("bad buflet handle", (_midx)); \
115 /* NOTREACHED */ \
116 __builtin_unreachable(); \
117 } \
118 } while (0)
119
120 #define _SLOT_DESC_VERIFY(_chrd, _sdp) do { \
121 if (__improbable(!SD_VALID_METADATA(_sdp)) && \
122 !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \
123 SK_ABORT("Slot descriptor has no metadata"); \
124 /* NOTREACHED */ \
125 __builtin_unreachable(); \
126 } \
127 } while (0)
128
129 #define _METADATA_VERIFY(_chrd, _md) do { \
130 if (__improbable(METADATA_PREAMBLE(_md)->mdp_redzone != \
131 (((mach_vm_address_t)(_md) - (_chrd)->chrd_md_base_addr) ^ \
132 __os_ch_md_redzone_cookie)) && \
133 !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \
134 SK_ABORT_WITH_CAUSE("Metadata redzone corrupted", \
135 METADATA_PREAMBLE(_md)->mdp_redzone); \
136 /* NOTREACHED */ \
137 __builtin_unreachable(); \
138 } \
139 } while (0)
140
141 #define _PKT_BUFCNT_VERIFY(_chrd, _bcnt, _bmax) do { \
142 if (__improbable((_chrd)->chrd_max_bufs < (_bmax))) { \
143 SK_ABORT_WITH_CAUSE("Invalid max bufcnt", (_bmax)); \
144 /* NOTREACHED */ \
145 __builtin_unreachable(); \
146 } \
147 if (__improbable((_bcnt) > (_bmax))) { \
148 SK_ABORT_WITH_CAUSE("Invalid bufcnt", (_bcnt)); \
149 /* NOTREACHED */ \
150 __builtin_unreachable(); \
151 } \
152 } while (0)
153
154 #define _ABORT_MSGSZ 1024
155
156 #define _SCHEMA_VER_VERIFY(_chd) do { \
157 /* ensure all stores are globally visible */ \
158 membar_sync(); \
159 if (CHD_SCHEMA(_chd)->csm_ver != CSM_CURRENT_VERSION) { \
160 char *_msg = malloc(_ABORT_MSGSZ); \
161 uint32_t _ver = (uint32_t)CHD_SCHEMA(_chd)->csm_ver; \
162 /* we're stuck with %x and %s formatters */ \
163 (void) _mach_snprintf(_msg, _ABORT_MSGSZ, \
164 "Schema region version mismatch: 0x%x != 0x%x\n" \
165 "Kernel version: %s - did you forget to install " \
166 "a matching libsystem_kernel.dylib?\n" \
167 "Kernel UUID: %x%x%x%x-%x%x-%x%x-%x%x-%x%x%x%x%x%x", \
168 _ver, (uint32_t)CSM_CURRENT_VERSION, \
169 CHD_SCHEMA(_chd)->csm_kern_name, \
170 CHD_SCHEMA(_chd)->csm_kern_uuid[0], \
171 CHD_SCHEMA(_chd)->csm_kern_uuid[1], \
172 CHD_SCHEMA(_chd)->csm_kern_uuid[2], \
173 CHD_SCHEMA(_chd)->csm_kern_uuid[3], \
174 CHD_SCHEMA(_chd)->csm_kern_uuid[4], \
175 CHD_SCHEMA(_chd)->csm_kern_uuid[5], \
176 CHD_SCHEMA(_chd)->csm_kern_uuid[6], \
177 CHD_SCHEMA(_chd)->csm_kern_uuid[7], \
178 CHD_SCHEMA(_chd)->csm_kern_uuid[8], \
179 CHD_SCHEMA(_chd)->csm_kern_uuid[9], \
180 CHD_SCHEMA(_chd)->csm_kern_uuid[10], \
181 CHD_SCHEMA(_chd)->csm_kern_uuid[11], \
182 CHD_SCHEMA(_chd)->csm_kern_uuid[12], \
183 CHD_SCHEMA(_chd)->csm_kern_uuid[13], \
184 CHD_SCHEMA(_chd)->csm_kern_uuid[14], \
185 CHD_SCHEMA(_chd)->csm_kern_uuid[15]); \
186 SK_ABORT_DYNAMIC(_msg); \
187 /* NOTREACHED */ \
188 __builtin_unreachable(); \
189 } \
190 } while (0)
191
192 #define _SLOT_ATTACH_METADATA(_usd, _md_idx) do { \
193 (_usd)->sd_md_idx = (_md_idx); \
194 (_usd)->sd_flags |= SD_IDX_VALID; \
195 } while (0)
196
197 #define _SLOT_DETACH_METADATA(_usd) do { \
198 (_usd)->sd_md_idx = OBJ_IDX_NONE; \
199 (_usd)->sd_flags &= ~SD_IDX_VALID; \
200 } while (0)
201
202 #define _CHANNEL_OFFSET(_type, _ptr, _offset) \
203 ((_type)(void *)((uintptr_t)(_ptr) + (_offset)))
204
205 #define _CHANNEL_SCHEMA(_base, _off) \
206 _CHANNEL_OFFSET(struct __user_channel_schema *, _base, _off)
207
208 #define _CHANNEL_RING_DEF_BUF(_chrd, _ring, _idx, _off) \
209 ((_chrd)->chrd_def_buf_base_addr + \
210 ((_idx) * (_ring)->ring_def_buf_size) + _off)
211
212 #define _CHANNEL_RING_LARGE_BUF(_chrd, _ring, _idx, _off) \
213 ((_chrd)->chrd_large_buf_base_addr + \
214 ((_idx) * (_ring)->ring_large_buf_size) + _off)
215
216 #define _CHANNEL_RING_BUF(_chrd, _ring, _bft) \
217 BUFLET_HAS_LARGE_BUF(_bft) ? \
218 _CHANNEL_RING_LARGE_BUF(_chrd, _ring, (_bft)->buf_idx, (_bft)->buf_boff) : \
219 _CHANNEL_RING_DEF_BUF(_chrd, _ring, (_bft)->buf_idx, (_bft)->buf_boff)
220
221 #define _CHANNEL_RING_BFT(_chrd, _ring, _idx) \
222 ((_chrd)->chrd_bft_base_addr + ((_idx) * (_ring)->ring_bft_size))
223
224 #define _CHANNEL_RING_NEXT(_ring, _cur) \
225 (__improbable((_cur) + 1 == (_ring)->ring_num_slots) ? 0 : (_cur) + 1)
226
227 #define _CHANNEL_RING_IS_DEFUNCT(_chrd) \
228 (!(*(_chrd)->chrd_csm_flags & CSM_ACTIVE))
229
230 #define _CHANNEL_IS_DEFUNCT(_chd) \
231 (!(CHD_SCHEMA(_chd)->csm_flags & CSM_ACTIVE))
232
233 #define _CH_PKT_GET_FIRST_BUFLET(_pkt, _bft, _chrd, _ring) do { \
234 if (__probable((_pkt)->pkt_qum_buf.buf_idx != OBJ_IDX_NONE)) { \
235 (_bft) = &(_pkt)->pkt_qum_buf; \
236 } else if ((_pkt)->pkt_qum_buf.buf_nbft_idx != OBJ_IDX_NONE) { \
237 (_bft) = _CHANNEL_RING_BFT(_chrd, _ring, \
238 (_pkt)->pkt_qum_buf.buf_nbft_idx); \
239 } else { \
240 (_bft) = NULL; \
241 } \
242 } while (0)
243
244 /*
245 * A per process copy of the channel metadata redzone cookie.
246 */
247 __attribute__((visibility("hidden")))
248 static uint64_t __os_ch_md_redzone_cookie = 0;
249
250 __attribute__((always_inline, visibility("hidden")))
251 static inline uint32_t
_num_tx_rings(struct ch_info * ci)252 _num_tx_rings(struct ch_info *ci)
253 {
254 ring_id_t first, last;
255
256 first = _ring_id(ci, CHANNEL_FIRST_TX_RING);
257 last = _ring_id(ci, CHANNEL_LAST_TX_RING);
258
259 return (last - first) + 1;
260 }
261
262 __attribute__((always_inline, visibility("hidden")))
263 static inline uint32_t
_num_rx_rings(struct ch_info * ci)264 _num_rx_rings(struct ch_info *ci)
265 {
266 ring_id_t first, last;
267
268 first = _ring_id(ci, CHANNEL_FIRST_RX_RING);
269 last = _ring_id(ci, CHANNEL_LAST_RX_RING);
270
271 return (last - first) + 1;
272 }
273
274 __attribute__((always_inline, visibility("hidden")))
275 static inline uint32_t
_num_allocator_rings(const struct __user_channel_schema * csm)276 _num_allocator_rings(const struct __user_channel_schema *csm)
277 {
278 return csm->csm_allocator_ring_pairs << 1;
279 }
280
281 __attribute__((visibility("hidden")))
282 static void
os_channel_init_ring(struct channel_ring_desc * chrd,struct channel * chd,uint32_t ring_index)283 os_channel_init_ring(struct channel_ring_desc *chrd,
284 struct channel *chd, uint32_t ring_index)
285 {
286 struct __user_channel_schema *csm = CHD_SCHEMA(chd);
287 struct __user_channel_ring *ring = NULL;
288 struct __slot_desc *sd = NULL;
289 nexus_meta_type_t md_type;
290 nexus_meta_subtype_t md_subtype;
291
292 ring = _CHANNEL_OFFSET(struct __user_channel_ring *, csm,
293 csm->csm_ring_ofs[ring_index].ring_off);
294 sd = _CHANNEL_OFFSET(struct __slot_desc *, csm,
295 csm->csm_ring_ofs[ring_index].sd_off);
296 md_type = csm->csm_md_type;
297 md_subtype = csm->csm_md_subtype;
298
299 if (ring == NULL || sd == NULL) {
300 SK_ABORT("Channel schema not valid");
301 /* NOTREACHED */
302 __builtin_unreachable();
303 } else if (!(md_type == NEXUS_META_TYPE_QUANTUM ||
304 md_type == NEXUS_META_TYPE_PACKET)) {
305 SK_ABORT_WITH_CAUSE("Metadata type unknown", md_type);
306 /* NOTREACHED */
307 __builtin_unreachable();
308 } else if (!(md_subtype == NEXUS_META_SUBTYPE_PAYLOAD ||
309 md_subtype == NEXUS_META_SUBTYPE_RAW)) {
310 SK_ABORT_WITH_CAUSE("Metadata subtype unknown", md_subtype);
311 /* NOTREACHED */
312 __builtin_unreachable();
313 }
314
315 chrd->chrd_slot_desc = sd;
316 chrd->chrd_csm_flags = &chd->chd_schema->csm_flags;
317 /* const overrides */
318 *(struct channel **)(uintptr_t)&chrd->chrd_channel = chd;
319 *(struct __user_channel_ring **)(uintptr_t)&chrd->chrd_ring = ring;
320 *(nexus_meta_type_t *)(uintptr_t)&chrd->chrd_md_type = md_type;
321 *(nexus_meta_subtype_t *)(uintptr_t)&chrd->chrd_md_subtype = md_subtype;
322 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_shmem_base_addr =
323 CHD_INFO(chd)->cinfo_mem_base;
324 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_def_buf_base_addr =
325 (mach_vm_address_t)((uintptr_t)ring + ring->ring_def_buf_base);
326 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_md_base_addr =
327 (mach_vm_address_t)((uintptr_t)ring + ring->ring_md_base);
328 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_sd_base_addr =
329 (mach_vm_address_t)((uintptr_t)ring + ring->ring_sd_base);
330 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_bft_base_addr =
331 (mach_vm_address_t)((uintptr_t)ring + ring->ring_bft_base);
332 *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_large_buf_base_addr =
333 (mach_vm_address_t)((uintptr_t)ring + ring->ring_large_buf_base);
334 *(uint32_t *)(uintptr_t)&chrd->chrd_max_bufs =
335 CHD_PARAMS(chd)->nxp_max_frags;
336 }
337
338 __attribute__((always_inline, visibility("hidden")))
339 static inline mach_vm_address_t
_initialize_metadata_address(const channel_ring_t chrd,struct __user_quantum * q,uint16_t * bdoff)340 _initialize_metadata_address(const channel_ring_t chrd,
341 struct __user_quantum *q, uint16_t *bdoff)
342 {
343 int i;
344 struct __user_buflet *ubft0;
345 const struct __user_channel_ring *ring = chrd->chrd_ring;
346
347 switch (chrd->chrd_md_type) {
348 case NEXUS_META_TYPE_PACKET: {
349 struct __user_buflet *ubft, *pbft;
350 struct __user_packet *p = (struct __user_packet *)q;
351 uint16_t bcnt = p->pkt_bufs_cnt;
352 uint16_t bmax = p->pkt_bufs_max;
353
354 _CASSERT(sizeof(p->pkt_qum_buf.buf_addr) ==
355 sizeof(mach_vm_address_t));
356 /*
357 * In the event of a defunct, we'd be accessing zero-filled
358 * memory and end up with 0 for bcnt or bmax.
359 */
360 if (__improbable((bcnt == 0) || (bmax == 0))) {
361 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
362 SK_ABORT("bad bufcnt");
363 /* NOTREACHED */
364 __builtin_unreachable();
365 }
366 return 0;
367 }
368 _PKT_BUFCNT_VERIFY(chrd, bcnt, bmax);
369 _CH_PKT_GET_FIRST_BUFLET(p, ubft, chrd, ring);
370 if (__improbable(ubft == NULL)) {
371 SK_ABORT("bad packet: no buflet");
372 /* NOTREACHED */
373 __builtin_unreachable();
374 }
375 /*
376 * special handling for empty packet buflet.
377 */
378 if (__improbable(p->pkt_qum_buf.buf_idx == OBJ_IDX_NONE)) {
379 *__DECONST(mach_vm_address_t *,
380 &p->pkt_qum_buf.buf_addr) = 0;
381 *__DECONST(mach_vm_address_t *,
382 &p->pkt_qum_buf.buf_nbft_addr) =
383 (mach_vm_address_t)ubft;
384 }
385 ubft0 = ubft;
386 for (i = 0; (i < bcnt) && (ubft != NULL); i++) {
387 pbft = ubft;
388 if (__probable(pbft->buf_idx != OBJ_IDX_NONE)) {
389 *(mach_vm_address_t *)(uintptr_t)
390 &(pbft->buf_addr) = _CHANNEL_RING_BUF(chrd,
391 ring, pbft);
392 } else {
393 *(mach_vm_address_t *)(uintptr_t)
394 &(pbft->buf_addr) = NULL;
395 }
396 if (pbft->buf_nbft_idx != OBJ_IDX_NONE) {
397 ubft = _CHANNEL_RING_BFT(chrd, ring,
398 pbft->buf_nbft_idx);
399 } else {
400 ubft = NULL;
401 }
402 *__DECONST(mach_vm_address_t *, &pbft->buf_nbft_addr) =
403 (mach_vm_address_t)ubft;
404 }
405 if (__improbable(pbft->buf_nbft_idx != OBJ_IDX_NONE)) {
406 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
407 SK_ABORT("non terminating buflet chain");
408 /* NOTREACHED */
409 __builtin_unreachable();
410 }
411 return 0;
412 }
413 if (__improbable(i != bcnt)) {
414 SK_ABORT_WITH_CAUSE("invalid buflet count", bcnt);
415 /* NOTREACHED */
416 __builtin_unreachable();
417 }
418 break;
419 }
420 default:
421 ubft0 = &q->qum_buf[0];
422 _CASSERT(sizeof(q->qum_buf[0].buf_addr) ==
423 sizeof(mach_vm_address_t));
424 /* immutable: compute pointers from the index */
425 *(mach_vm_address_t *)(uintptr_t)&ubft0->buf_addr =
426 _CHANNEL_RING_BUF(chrd, ring, ubft0);
427 break;
428 }
429
430 /* return address and offset of the first buffer */
431 *bdoff = ubft0->buf_doff;
432 return ubft0->buf_addr;
433 }
434
435 /*
436 * _slot_index_is_valid
437 * - verify that the slot index is within valid bounds
438 * - if the head is less than (or equal to) the tail (case A below)
439 * head <= valid < tail
440 * - if the head is greater than the tail (case B below)
441 * valid < tail
442 * or
443 * head <= valid < num_slots
444 *
445 * case A: x x x x x x x H o o o o o T x x x x x x
446 * case B: o o o o o T x x x x H o o o o o o o o o
447 *
448 * 'H' - head
449 * 'T' - tail
450 * 'x' - invalid
451 * 'o' - valid
452 */
453 __attribute__((always_inline, visibility("hidden")))
454 static inline int
_slot_index_is_valid(const struct __user_channel_ring * ring,slot_idx_t idx)455 _slot_index_is_valid(const struct __user_channel_ring *ring, slot_idx_t idx)
456 {
457 int is_valid = 0;
458
459 if (ring->ring_head <= ring->ring_tail) {
460 if (__probable(idx >= ring->ring_head && idx < ring->ring_tail)) {
461 is_valid = 1;
462 }
463 } else {
464 if (__probable(idx < ring->ring_tail ||
465 (idx >= ring->ring_head && idx < ring->ring_num_slots))) {
466 is_valid = 1;
467 }
468 }
469
470 return is_valid;
471 }
472
473 channel_t
os_channel_create_extended(const uuid_t uuid,const nexus_port_t port,const ring_dir_t dir,const ring_id_t ring,const channel_attr_t cha)474 os_channel_create_extended(const uuid_t uuid, const nexus_port_t port,
475 const ring_dir_t dir, const ring_id_t ring, const channel_attr_t cha)
476 {
477 uint32_t num_tx_rings, num_rx_rings, num_allocator_rings;
478 uint32_t ring_offset, ring_index, num_event_rings;
479 struct __user_channel_schema *ucs;
480 struct channel *chd = NULL;
481 struct ch_info *ci = NULL;
482 struct ch_init init;
483 int i, fd = -1;
484 int err = 0;
485 size_t chd_sz;
486
487 SK_ALIGN64_CASSERT(struct ch_info, cinfo_mem_map_size);
488
489 switch (dir) {
490 case CHANNEL_DIR_TX_RX:
491 case CHANNEL_DIR_TX:
492 case CHANNEL_DIR_RX:
493 break;
494 default:
495 err = EINVAL;
496 goto done;
497 }
498
499 ci = malloc(CHD_INFO_SIZE);
500 if (ci == NULL) {
501 err = errno = ENOMEM;
502 goto done;
503 }
504 bzero(ci, CHD_INFO_SIZE);
505
506 bzero(&init, sizeof(init));
507 init.ci_version = CHANNEL_INIT_CURRENT_VERSION;
508 if (cha != NULL) {
509 if (cha->cha_exclusive != 0) {
510 init.ci_ch_mode |= CHMODE_EXCLUSIVE;
511 }
512 if (cha->cha_user_packet_pool != 0) {
513 init.ci_ch_mode |= CHMODE_USER_PACKET_POOL;
514 }
515 if (cha->cha_nexus_defunct_ok != 0) {
516 init.ci_ch_mode |= CHMODE_DEFUNCT_OK;
517 }
518 if (cha->cha_enable_event_ring != 0) {
519 /* User packet pool is required for event rings */
520 if (cha->cha_user_packet_pool == 0) {
521 err = EINVAL;
522 goto done;
523 }
524 init.ci_ch_mode |= CHMODE_EVENT_RING;
525 }
526 if (cha->cha_monitor != 0) {
527 if (dir == CHANNEL_DIR_TX_RX) {
528 init.ci_ch_mode |= CHMODE_MONITOR;
529 } else if (dir == CHANNEL_DIR_TX) {
530 init.ci_ch_mode |= CHMODE_MONITOR_TX;
531 } else if (dir == CHANNEL_DIR_RX) {
532 init.ci_ch_mode |= CHMODE_MONITOR_RX;
533 }
534 if (cha->cha_monitor == CHANNEL_MONITOR_NO_COPY) {
535 init.ci_ch_mode |= CHMODE_MONITOR_NO_COPY;
536 }
537 }
538 if (cha->cha_filter != 0) {
539 init.ci_ch_mode |= CHMODE_FILTER;
540 }
541 if (cha->cha_low_latency != 0) {
542 init.ci_ch_mode |= CHMODE_LOW_LATENCY;
543 }
544 init.ci_key_len = cha->cha_key_len;
545 init.ci_key = cha->cha_key;
546 init.ci_tx_lowat = cha->cha_tx_lowat;
547 init.ci_rx_lowat = cha->cha_rx_lowat;
548 }
549 init.ci_ch_ring_id = ring;
550 init.ci_nx_port = port;
551 bcopy(uuid, init.ci_nx_uuid, sizeof(uuid_t));
552
553 fd = __channel_open(&init, sizeof(init));
554 if (fd == -1) {
555 err = errno;
556 goto done;
557 }
558
559 err = __channel_get_info(fd, ci, CHD_INFO_SIZE);
560 if (err != 0) {
561 err = errno;
562 goto done;
563 }
564
565 ucs = _CHANNEL_SCHEMA(ci->cinfo_mem_base, ci->cinfo_schema_offset);
566 num_tx_rings = _num_tx_rings(ci); /* # of channel tx rings */
567 num_rx_rings = _num_rx_rings(ci); /* # of channel rx rings */
568 num_allocator_rings = _num_allocator_rings(ucs);
569 num_event_rings = ucs->csm_num_event_rings;
570
571 /*
572 * if the user requested packet allocation mode for channel, then
573 * check that channel was opened in packet allocation mode and
574 * allocator rings were created.
575 */
576 if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) &&
577 ((num_allocator_rings < 2) ||
578 !(ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL))) {
579 err = errno = ENXIO;
580 goto done;
581 }
582
583 if ((init.ci_ch_mode & CHMODE_EVENT_RING) && ((num_event_rings == 0) ||
584 !(ci->cinfo_ch_mode & CHMODE_EVENT_RING))) {
585 err = errno = ENXIO;
586 goto done;
587 }
588
589 chd_sz = CHD_SIZE(num_tx_rings + num_rx_rings + num_allocator_rings +
590 num_event_rings);
591 chd = malloc(chd_sz);
592 if (chd == NULL) {
593 err = errno = ENOMEM;
594 goto done;
595 }
596
597 bzero(chd, chd_sz);
598 chd->chd_fd = fd;
599 chd->chd_guard = init.ci_guard;
600
601 /* claim ch_info (will be freed along with the channel itself) */
602 CHD_INFO(chd) = ci;
603 ci = NULL;
604
605 /* const override */
606 *(struct __user_channel_schema **)(uintptr_t)&chd->chd_schema = ucs;
607
608 /* make sure we're running on the right kernel */
609 _SCHEMA_VER_VERIFY(chd);
610
611 *(nexus_meta_type_t *)&chd->chd_md_type = CHD_SCHEMA(chd)->csm_md_type;
612 *(nexus_meta_subtype_t *)&chd->chd_md_subtype =
613 CHD_SCHEMA(chd)->csm_md_subtype;
614
615 if (CHD_SCHEMA(chd)->csm_stats_ofs != 0) {
616 *(void **)(uintptr_t)&chd->chd_nx_stats =
617 _CHANNEL_OFFSET(void *, CHD_INFO(chd)->cinfo_mem_base,
618 CHD_SCHEMA(chd)->csm_stats_ofs);
619 }
620
621 if (CHD_SCHEMA(chd)->csm_flowadv_ofs != 0) {
622 *(struct __flowadv_entry **)(uintptr_t)&chd->chd_nx_flowadv =
623 _CHANNEL_OFFSET(struct __flowadv_entry *,
624 CHD_INFO(chd)->cinfo_mem_base,
625 CHD_SCHEMA(chd)->csm_flowadv_ofs);
626 }
627
628 if (CHD_SCHEMA(chd)->csm_nexusadv_ofs != 0) {
629 struct __kern_nexus_adv_metadata *adv_md;
630
631 *(struct __kern_nexus_adv_metadata **)
632 (uintptr_t)&chd->chd_nx_adv =
633 _CHANNEL_OFFSET(struct __kern_nexus_adv_metadata *,
634 CHD_INFO(chd)->cinfo_mem_base,
635 CHD_SCHEMA(chd)->csm_nexusadv_ofs);
636 adv_md = CHD_NX_ADV_MD(chd);
637 if (adv_md->knam_version != NX_ADVISORY_MD_CURRENT_VERSION &&
638 !_CHANNEL_IS_DEFUNCT(chd)) {
639 SK_ABORT_WITH_CAUSE("nexus advisory metadata version"
640 " mismatch", NX_ADVISORY_MD_CURRENT_VERSION);
641 /* NOTREACHED */
642 __builtin_unreachable();
643 }
644 if (chd->chd_nx_adv->knam_type == NEXUS_ADVISORY_TYPE_NETIF) {
645 struct netif_nexus_advisory *netif_adv;
646 netif_adv = CHD_NX_ADV_NETIF(adv_md);
647 if (netif_adv->nna_version !=
648 NX_NETIF_ADVISORY_CURRENT_VERSION &&
649 !_CHANNEL_IS_DEFUNCT(chd)) {
650 SK_ABORT_WITH_CAUSE("nexus advisory "
651 "version mismatch for netif",
652 NX_NETIF_ADVISORY_CURRENT_VERSION);
653 /* NOTREACHED */
654 __builtin_unreachable();
655 }
656 } else if (chd->chd_nx_adv->knam_type ==
657 NEXUS_ADVISORY_TYPE_FLOWSWITCH) {
658 struct sk_nexusadv *fsw_adv;
659 fsw_adv = CHD_NX_ADV_FSW(adv_md);
660 if (fsw_adv->nxadv_ver !=
661 NX_FLOWSWITCH_ADVISORY_CURRENT_VERSION &&
662 !_CHANNEL_IS_DEFUNCT(chd)) {
663 SK_ABORT_WITH_CAUSE("nexus advisory "
664 "version mismatch for flowswitch",
665 NX_FLOWSWITCH_ADVISORY_CURRENT_VERSION);
666 /* NOTREACHED */
667 __builtin_unreachable();
668 }
669 } else if (!_CHANNEL_IS_DEFUNCT(chd)) {
670 SK_ABORT_WITH_CAUSE("nexus advisory metadata type"
671 " unknown", NX_ADVISORY_MD_CURRENT_VERSION);
672 /* NOTREACHED */
673 __builtin_unreachable();
674 }
675 }
676
677 if (cha != NULL) {
678 os_channel_info2attr(chd, cha);
679 }
680
681 ring_offset = 0;
682 for (i = 0; i < num_tx_rings; i++) {
683 ring_index = ring_offset + i;
684 os_channel_init_ring(&chd->chd_rings[ring_index], chd,
685 ring_index);
686 }
687
688 ring_offset += num_tx_rings;
689 for (i = 0; i < num_rx_rings; i++) {
690 ring_index = ring_offset + i;
691 os_channel_init_ring(&chd->chd_rings[ring_index], chd,
692 ring_index);
693 }
694
695 ring_offset += num_rx_rings;
696 for (i = 0; i < num_allocator_rings; i++) {
697 ring_index = ring_offset + i;
698 os_channel_init_ring(&chd->chd_rings[ring_index], chd,
699 ring_index);
700 }
701
702 ring_offset += num_allocator_rings;
703 for (i = 0; i < num_event_rings; i++) {
704 ring_index = ring_offset + i;
705 os_channel_init_ring(&chd->chd_rings[ring_index], chd,
706 ring_index);
707 }
708
709 if (init.ci_ch_mode & CHMODE_USER_PACKET_POOL) {
710 chd->chd_sync_flags = CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE;
711 *__DECONST(uint8_t *, &chd->chd_alloc_ring_idx) =
712 num_tx_rings + num_rx_rings;
713 if (num_allocator_rings > 2) {
714 chd->chd_sync_flags |= CHANNEL_SYNCF_ALLOC_BUF;
715 *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) =
716 chd->chd_alloc_ring_idx + 1;
717 *__DECONST(uint8_t *, &chd->chd_free_ring_idx) =
718 chd->chd_buf_alloc_ring_idx + 1;
719 *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) =
720 chd->chd_free_ring_idx + 1;
721 } else {
722 *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) =
723 CHD_RING_IDX_NONE;
724 *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) =
725 CHD_RING_IDX_NONE;
726 *__DECONST(uint8_t *, &chd->chd_free_ring_idx) =
727 chd->chd_alloc_ring_idx + 1;
728 }
729 } else {
730 *__DECONST(uint8_t *, &chd->chd_alloc_ring_idx) =
731 CHD_RING_IDX_NONE;
732 *__DECONST(uint8_t *, &chd->chd_free_ring_idx) =
733 CHD_RING_IDX_NONE;
734 *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) =
735 CHD_RING_IDX_NONE;
736 *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) =
737 CHD_RING_IDX_NONE;
738 }
739
740 if (__os_ch_md_redzone_cookie == 0) {
741 __os_ch_md_redzone_cookie =
742 CHD_SCHEMA(chd)->csm_md_redzone_cookie;
743 }
744
745 /* ensure all stores are globally visible */
746 membar_sync();
747
748 done:
749 if (err != 0) {
750 if (fd != -1) {
751 (void) guarded_close_np(fd, &init.ci_guard);
752 }
753 if (chd != NULL) {
754 if (CHD_INFO(chd) != NULL) {
755 free(CHD_INFO(chd));
756 CHD_INFO(chd) = NULL;
757 }
758 free(chd);
759 chd = NULL;
760 }
761 if (ci != NULL) {
762 free(ci);
763 ci = NULL;
764 }
765 errno = err;
766 }
767 return chd;
768 }
769
770 channel_t
os_channel_create(const uuid_t uuid,const nexus_port_t port)771 os_channel_create(const uuid_t uuid, const nexus_port_t port)
772 {
773 return os_channel_create_extended(uuid, port, CHANNEL_DIR_TX_RX,
774 CHANNEL_RING_ID_ANY, NULL);
775 }
776
777 int
os_channel_get_fd(const channel_t chd)778 os_channel_get_fd(const channel_t chd)
779 {
780 return chd->chd_fd;
781 }
782
783 int
os_channel_read_attr(const channel_t chd,channel_attr_t cha)784 os_channel_read_attr(const channel_t chd, channel_attr_t cha)
785 {
786 int err;
787
788 if ((err = __channel_get_info(chd->chd_fd, CHD_INFO(chd),
789 CHD_INFO_SIZE)) == 0) {
790 os_channel_info2attr(chd, cha);
791 }
792
793 return err;
794 }
795
796 int
os_channel_write_attr(const channel_t chd,channel_attr_t cha)797 os_channel_write_attr(const channel_t chd, channel_attr_t cha)
798 {
799 int err = 0;
800
801 if (CHD_INFO(chd)->cinfo_tx_lowat.cet_unit !=
802 cha->cha_tx_lowat.cet_unit ||
803 CHD_INFO(chd)->cinfo_tx_lowat.cet_value !=
804 cha->cha_tx_lowat.cet_value) {
805 if ((err = __channel_set_opt(chd->chd_fd, CHOPT_TX_LOWAT_THRESH,
806 &cha->cha_tx_lowat, sizeof(cha->cha_tx_lowat))) != 0) {
807 goto done;
808 }
809
810 /* update local copy */
811 CHD_INFO(chd)->cinfo_tx_lowat = cha->cha_tx_lowat;
812 }
813
814 if (CHD_INFO(chd)->cinfo_rx_lowat.cet_unit !=
815 cha->cha_rx_lowat.cet_unit ||
816 CHD_INFO(chd)->cinfo_rx_lowat.cet_value !=
817 cha->cha_rx_lowat.cet_value) {
818 if ((err = __channel_set_opt(chd->chd_fd, CHOPT_RX_LOWAT_THRESH,
819 &cha->cha_rx_lowat, sizeof(cha->cha_rx_lowat))) != 0) {
820 goto done;
821 }
822
823 /* update local copy */
824 CHD_INFO(chd)->cinfo_rx_lowat = cha->cha_rx_lowat;
825 }
826 done:
827 return err;
828 }
829
830 int
os_channel_read_nexus_extension_info(const channel_t chd,nexus_type_t * nt,uint64_t * ext)831 os_channel_read_nexus_extension_info(const channel_t chd, nexus_type_t *nt,
832 uint64_t *ext)
833 {
834 struct nxprov_params *nxp;
835
836 nxp = &CHD_INFO(chd)->cinfo_nxprov_params;
837 if (nt != NULL) {
838 *nt = nxp->nxp_type;
839 }
840 if (ext != NULL) {
841 *ext = (uint64_t)nxp->nxp_extensions;
842 }
843
844 return 0;
845 }
846
847 int
os_channel_sync(const channel_t chd,const sync_mode_t mode)848 os_channel_sync(const channel_t chd, const sync_mode_t mode)
849 {
850 if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX)) {
851 return EINVAL;
852 }
853
854 return __channel_sync(chd->chd_fd, mode,
855 (mode == CHANNEL_SYNC_TX) ? chd->chd_sync_flags :
856 (chd->chd_sync_flags &
857 ~(CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_ALLOC_BUF)));
858 }
859
860 void
os_channel_destroy(channel_t chd)861 os_channel_destroy(channel_t chd)
862 {
863 if (chd->chd_fd != -1) {
864 (void) guarded_close_np(chd->chd_fd, &chd->chd_guard);
865 }
866
867 if (CHD_INFO(chd) != NULL) {
868 free(CHD_INFO(chd));
869 CHD_INFO(chd) = NULL;
870 }
871
872 free(chd);
873 }
874
875 int
os_channel_is_defunct(channel_t chd)876 os_channel_is_defunct(channel_t chd)
877 {
878 return _CHANNEL_IS_DEFUNCT(chd);
879 }
880
881 __attribute__((always_inline, visibility("hidden")))
882 static inline ring_id_t
_ring_id(struct ch_info * cinfo,const ring_id_type_t type)883 _ring_id(struct ch_info *cinfo, const ring_id_type_t type)
884 {
885 ring_id_t rid = CHANNEL_RING_ID_ANY; /* make it crash */
886
887 switch (type) {
888 case CHANNEL_FIRST_TX_RING:
889 rid = cinfo->cinfo_first_tx_ring;
890 break;
891
892 case CHANNEL_LAST_TX_RING:
893 rid = cinfo->cinfo_last_tx_ring;
894 break;
895
896 case CHANNEL_FIRST_RX_RING:
897 rid = cinfo->cinfo_first_rx_ring;
898 break;
899
900 case CHANNEL_LAST_RX_RING:
901 rid = cinfo->cinfo_last_rx_ring;
902 break;
903 }
904
905 return rid;
906 }
907
908 ring_id_t
os_channel_ring_id(const channel_t chd,const ring_id_type_t type)909 os_channel_ring_id(const channel_t chd, const ring_id_type_t type)
910 {
911 return _ring_id(CHD_INFO(chd), type);
912 }
913
914 channel_ring_t
os_channel_tx_ring(const channel_t chd,const ring_id_t rid)915 os_channel_tx_ring(const channel_t chd, const ring_id_t rid)
916 {
917 struct ch_info *ci = CHD_INFO(chd);
918
919 if (__improbable((ci->cinfo_ch_ring_id != CHANNEL_RING_ID_ANY &&
920 ci->cinfo_ch_ring_id != rid) ||
921 rid < _ring_id(ci, CHANNEL_FIRST_TX_RING) ||
922 rid > _ring_id(ci, CHANNEL_LAST_TX_RING))) {
923 return NULL;
924 }
925
926 return &chd->chd_rings[rid - _ring_id(ci, CHANNEL_FIRST_TX_RING)];
927 }
928
929 channel_ring_t
os_channel_rx_ring(const channel_t chd,const ring_id_t rid)930 os_channel_rx_ring(const channel_t chd, const ring_id_t rid)
931 {
932 struct ch_info *ci = CHD_INFO(chd);
933
934 if (__improbable((ci->cinfo_ch_ring_id != CHANNEL_RING_ID_ANY &&
935 ci->cinfo_ch_ring_id != rid) ||
936 rid < _ring_id(ci, CHANNEL_FIRST_RX_RING) ||
937 rid > _ring_id(ci, CHANNEL_LAST_RX_RING))) {
938 return NULL;
939 }
940
941 return &chd->chd_rings[_num_tx_rings(ci) + /* add tx rings */
942 (rid - _ring_id(ci, CHANNEL_FIRST_RX_RING))];
943 }
944
945 /*
946 * Return 1 if we have pending transmissions in the tx ring. When everything
947 * is complete ring->ring_head == ring->ring_khead.
948 */
949 int
os_channel_pending(const channel_ring_t chrd)950 os_channel_pending(const channel_ring_t chrd)
951 {
952 struct __user_channel_ring *ring =
953 __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
954 return ring->ring_head != ring->ring_khead;
955 }
956
957 uint64_t
os_channel_ring_sync_time(const channel_ring_t chrd)958 os_channel_ring_sync_time(const channel_ring_t chrd)
959 {
960 return chrd->chrd_ring->ring_sync_time;
961 }
962
963 uint64_t
os_channel_ring_notify_time(const channel_ring_t chrd)964 os_channel_ring_notify_time(const channel_ring_t chrd)
965 {
966 return chrd->chrd_ring->ring_notify_time;
967 }
968
969 uint32_t
os_channel_available_slot_count(const channel_ring_t chrd)970 os_channel_available_slot_count(const channel_ring_t chrd)
971 {
972 const struct __user_channel_ring *ring = chrd->chrd_ring;
973 uint32_t count;
974 int n;
975
976 if (ring->ring_kind == CR_KIND_TX) {
977 n = ring->ring_head - ring->ring_khead;
978 if (n < 0) {
979 n += ring->ring_num_slots;
980 }
981 count = (ring->ring_num_slots - n - 1);
982 } else {
983 n = ring->ring_tail - ring->ring_head;
984 if (n < 0) {
985 n += ring->ring_num_slots;
986 }
987 count = n;
988 }
989 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? 0 : count;
990 }
991
992 int
os_channel_advance_slot(channel_ring_t chrd,const channel_slot_t slot)993 os_channel_advance_slot(channel_ring_t chrd, const channel_slot_t slot)
994 {
995 struct __user_channel_ring *ring =
996 __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
997 slot_idx_t idx;
998 int err;
999
1000 idx = _SLOT_INDEX(chrd, slot);
1001 if (__probable(_slot_index_is_valid(ring, idx))) {
1002 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
1003 err = 0;
1004 } else {
1005 err = (_CHANNEL_RING_IS_DEFUNCT(chrd) ? ENXIO : EINVAL);
1006 }
1007 return err;
1008 }
1009
1010 channel_slot_t
os_channel_get_next_slot(const channel_ring_t chrd,const channel_slot_t slot0,slot_prop_t * prop)1011 os_channel_get_next_slot(const channel_ring_t chrd, const channel_slot_t slot0,
1012 slot_prop_t *prop)
1013 {
1014 const struct __user_channel_ring *ring = chrd->chrd_ring;
1015 const struct __slot_desc *slot;
1016 slot_idx_t idx;
1017
1018 if (__probable(slot0 != NULL)) {
1019 idx = _SLOT_INDEX(chrd, slot0);
1020 if (__probable(_slot_index_is_valid(ring, idx))) {
1021 idx = _CHANNEL_RING_NEXT(ring, idx);
1022 } else if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1023 /* slot is out of bounds */
1024 SK_ABORT_WITH_CAUSE("Index out of bounds in gns", idx);
1025 /* NOTREACHED */
1026 __builtin_unreachable();
1027 } else {
1028 /*
1029 * In case of a defunct, pretend as if we've
1030 * advanced to the last slot; this will result
1031 * in a NULL slot below.
1032 */
1033 idx = ring->ring_tail;
1034 }
1035 } else {
1036 idx = ring->ring_head;
1037 }
1038
1039 if (__probable(idx != ring->ring_tail)) {
1040 slot = &chrd->chrd_slot_desc[idx];
1041 } else {
1042 /* we just advanced to the last slot */
1043 slot = NULL;
1044 }
1045
1046 if (__probable(slot != NULL)) {
1047 uint16_t ring_kind = ring->ring_kind;
1048 struct __user_quantum *q;
1049 mach_vm_address_t baddr;
1050 uint16_t bdoff;
1051
1052 if (__improbable((ring_kind == CR_KIND_TX) &&
1053 (CHD_INFO(chrd->chrd_channel)->cinfo_ch_mode &
1054 CHMODE_USER_PACKET_POOL))) {
1055 if (SD_VALID_METADATA(SLOT_DESC_USD(slot))) {
1056 SK_ABORT_WITH_CAUSE("Tx slot has attached "
1057 "metadata", idx);
1058 /* NOTREACHED */
1059 __builtin_unreachable();
1060 }
1061 if (prop != NULL) {
1062 prop->sp_len = 0;
1063 prop->sp_flags = 0;
1064 prop->sp_buf_ptr = 0;
1065 prop->sp_mdata_ptr = 0;
1066 }
1067 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ?
1068 NULL : (channel_slot_t)slot;
1069 }
1070
1071 _SLOT_DESC_VERIFY(chrd, SLOT_DESC_USD(slot));
1072 q = _SLOT_METADATA(chrd, ring, idx);
1073 _METADATA_VERIFY(chrd, q);
1074
1075 baddr = _initialize_metadata_address(chrd, q, &bdoff);
1076 if (__improbable(baddr == 0)) {
1077 return NULL;
1078 }
1079 /* No multi-buflet support for slot based interface */
1080 if (__probable(prop != NULL)) {
1081 /* immutable: slot index */
1082 prop->sp_idx = idx;
1083 prop->sp_flags = 0;
1084 prop->sp_buf_ptr = baddr + bdoff;
1085 prop->sp_mdata_ptr = q;
1086 /* reset slot length if this is to be used for tx */
1087 prop->sp_len = (ring_kind == CR_KIND_TX) ?
1088 ring->ring_def_buf_size : q->qum_len;
1089 }
1090 }
1091
1092 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ?
1093 NULL : (channel_slot_t)slot;
1094 }
1095
1096 void
os_channel_set_slot_properties(const channel_ring_t chrd,const channel_slot_t slot,const slot_prop_t * prop)1097 os_channel_set_slot_properties(const channel_ring_t chrd,
1098 const channel_slot_t slot, const slot_prop_t *prop)
1099 {
1100 const struct __user_channel_ring *ring = chrd->chrd_ring;
1101 slot_idx_t idx = _SLOT_INDEX(chrd, slot);
1102
1103 if (__probable(_slot_index_is_valid(ring, idx))) {
1104 struct __user_quantum *q;
1105
1106 _METADATA_VERIFY(chrd, prop->sp_mdata_ptr);
1107 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
1108
1109 /*
1110 * In the event of a defunct, we'd be accessing zero-filled
1111 * memory; this is fine we ignore all changes made to the
1112 * region at that time.
1113 */
1114 q = _SLOT_METADATA(chrd, ring, idx);
1115 q->qum_len = prop->sp_len;
1116 switch (chrd->chrd_md_type) {
1117 case NEXUS_META_TYPE_PACKET: {
1118 struct __user_packet *p = (struct __user_packet *)q;
1119 /* No multi-buflet support for slot based interface */
1120 p->pkt_qum_buf.buf_dlen = prop->sp_len;
1121 p->pkt_qum_buf.buf_doff = 0;
1122 break;
1123 }
1124 default:
1125 q->qum_buf[0].buf_dlen = prop->sp_len;
1126 q->qum_buf[0].buf_doff = 0;
1127 break;
1128 }
1129 } else if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1130 /* slot is out of bounds */
1131 SK_ABORT_WITH_CAUSE("Index out of bounds in ssp", idx);
1132 /* NOTREACHED */
1133 __builtin_unreachable();
1134 }
1135 }
1136
1137 packet_t
os_channel_slot_get_packet(const channel_ring_t chrd,const channel_slot_t slot)1138 os_channel_slot_get_packet(const channel_ring_t chrd, const channel_slot_t slot)
1139 {
1140 const struct __user_channel_ring *ring = chrd->chrd_ring;
1141 struct __user_quantum *q = NULL;
1142
1143 if (__probable(slot != NULL)) {
1144 slot_idx_t idx = _SLOT_INDEX(chrd, slot);
1145 if (__improbable(!_slot_index_is_valid(ring, idx)) &&
1146 !_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1147 /* slot is out of bounds */
1148 SK_ABORT_WITH_CAUSE("Index out of bounds in sgp", idx);
1149 /* NOTREACHED */
1150 __builtin_unreachable();
1151 }
1152
1153 if (__probable(SD_VALID_METADATA(_SLOT_DESC(chrd, idx)))) {
1154 obj_idx_t midx;
1155 q = _SLOT_METADATA(chrd, ring, idx);
1156 _METADATA_VERIFY(chrd, q);
1157 /*
1158 * In the event of a defunct, we'd be accessing
1159 * zero-filed memory and end up with 0 for midx;
1160 * this is fine since we ignore all changes made
1161 * to the region at that time.
1162 */
1163 midx = METADATA_IDX(q);
1164 _SLOT_METADATA_IDX_VERIFY(chrd, q, midx);
1165 }
1166 }
1167
1168 return (q == NULL) ? 0 :
1169 SK_PTR_ENCODE(q, chrd->chrd_md_type, chrd->chrd_md_subtype);
1170 }
1171
1172 void *
os_channel_get_stats_region(const channel_t chd,const channel_stats_id_t id)1173 os_channel_get_stats_region(const channel_t chd, const channel_stats_id_t id)
1174 {
1175 void *sp = CHD_NX_STATS(chd);
1176 struct __nx_stats_fsw *nxs_fsw;
1177 void *ptr = NULL;
1178
1179 /* we currently deal only with flowswitch */
1180 if (sp == NULL ||
1181 CHD_SCHEMA(chd)->csm_stats_type != NEXUS_STATS_TYPE_FSW) {
1182 return NULL;
1183 }
1184
1185 nxs_fsw = sp;
1186
1187 switch (id) {
1188 case CHANNEL_STATS_ID_IP:
1189 ptr = &nxs_fsw->nxs_ipstat;
1190 break;
1191
1192 case CHANNEL_STATS_ID_IP6:
1193 ptr = &nxs_fsw->nxs_ip6stat;
1194 break;
1195
1196 case CHANNEL_STATS_ID_TCP:
1197 ptr = &nxs_fsw->nxs_tcpstat;
1198 break;
1199
1200 case CHANNEL_STATS_ID_UDP:
1201 ptr = &nxs_fsw->nxs_udpstat;
1202 break;
1203
1204 case CHANNEL_STATS_ID_QUIC:
1205 ptr = &nxs_fsw->nxs_quicstat;
1206 break;
1207
1208 default:
1209 ptr = NULL;
1210 break;
1211 }
1212
1213 return ptr;
1214 }
1215
1216 void *
os_channel_get_advisory_region(const channel_t chd)1217 os_channel_get_advisory_region(const channel_t chd)
1218 {
1219 struct __kern_nexus_adv_metadata *adv_md;
1220 /*
1221 * To be backward compatible this API will only return
1222 * the advisory region for flowswitch.
1223 */
1224 adv_md = CHD_NX_ADV_MD(chd);
1225 if (adv_md == NULL ||
1226 adv_md->knam_type != NEXUS_ADVISORY_TYPE_FLOWSWITCH) {
1227 return NULL;
1228 }
1229 return CHD_NX_ADV_FSW(adv_md);
1230 }
1231
1232 __attribute__((always_inline, visibility("hidden")))
1233 static inline int
_flowadv_id_equal(struct __flowadv_entry * fe,uuid_t id)1234 _flowadv_id_equal(struct __flowadv_entry *fe, uuid_t id)
1235 {
1236 /*
1237 * Anticipate a nicely (8-bytes) aligned UUID from
1238 * caller; the one in fae_id is always 8-byte aligned.
1239 */
1240 if (__probable(IS_P2ALIGNED(id, sizeof(uint64_t)))) {
1241 uint64_t *id_64 = (uint64_t *)(uintptr_t)id;
1242 return fe->fae_id_64[0] == id_64[0] &&
1243 fe->fae_id_64[1] == id_64[1];
1244 } else if (__probable(IS_P2ALIGNED(id, sizeof(uint32_t)))) {
1245 uint32_t *id_32 = (uint32_t *)(uintptr_t)id;
1246 return fe->fae_id_32[0] == id_32[0] &&
1247 fe->fae_id_32[1] == id_32[1] &&
1248 fe->fae_id_32[2] == id_32[2] &&
1249 fe->fae_id_32[3] == id_32[3];
1250 }
1251
1252 return UUID_COMPARE(fe->fae_id, id);
1253 }
1254
1255 int
os_channel_flow_admissible(const channel_ring_t chrd,uuid_t flow_id,const flowadv_idx_t flow_index)1256 os_channel_flow_admissible(const channel_ring_t chrd, uuid_t flow_id,
1257 const flowadv_idx_t flow_index)
1258 {
1259 const struct __user_channel_ring *ring = chrd->chrd_ring;
1260 const struct channel *chd = chrd->chrd_channel;
1261 struct __flowadv_entry *fe = CHD_NX_FLOWADV(chd);
1262
1263 /*
1264 * Currently, flow advisory is on a per-nexus port basis.
1265 * To anticipate for future requirements, we use the ring
1266 * as parameter instead, even though we use it only to
1267 * check if this is a TX ring for now.
1268 */
1269 if (__improbable(CHD_NX_FLOWADV(chd) == NULL)) {
1270 return ENXIO;
1271 } else if (__improbable(ring->ring_kind != CR_KIND_TX ||
1272 flow_index >= CHD_PARAMS(chd)->nxp_flowadv_max)) {
1273 return EINVAL;
1274 }
1275
1276 /*
1277 * Rather than checking if the UUID is all zeroes, check
1278 * against fae_flags since the presence of FLOWADV_VALID
1279 * means fae_id is non-zero. This avoids another round of
1280 * comparison against zeroes.
1281 */
1282 fe = &CHD_NX_FLOWADV(chd)[flow_index];
1283 if (__improbable(fe->fae_flags == 0 || !_flowadv_id_equal(fe, flow_id))) {
1284 return ENOENT;
1285 }
1286
1287 return __improbable((fe->fae_flags & FLOWADVF_SUSPENDED) != 0) ?
1288 ENOBUFS: 0;
1289 }
1290
1291 channel_attr_t
os_channel_attr_create(void)1292 os_channel_attr_create(void)
1293 {
1294 struct channel_attr *cha;
1295
1296 cha = malloc(sizeof(*cha));
1297 if (cha != NULL) {
1298 bzero(cha, sizeof(*cha));
1299 }
1300 return cha;
1301 }
1302
1303 channel_attr_t
os_channel_attr_clone(const channel_attr_t cha)1304 os_channel_attr_clone(const channel_attr_t cha)
1305 {
1306 struct channel_attr *ncha;
1307
1308 ncha = os_channel_attr_create();
1309 if (ncha != NULL && cha != NULL) {
1310 bcopy(cha, ncha, sizeof(*ncha));
1311 ncha->cha_key = NULL;
1312 ncha->cha_key_len = 0;
1313 if (cha->cha_key != NULL && cha->cha_key_len != 0 &&
1314 os_channel_attr_set_key(ncha, cha->cha_key,
1315 cha->cha_key_len) != 0) {
1316 os_channel_attr_destroy(ncha);
1317 ncha = NULL;
1318 }
1319 }
1320
1321 return ncha;
1322 }
1323
1324 int
os_channel_attr_set(const channel_attr_t cha,const channel_attr_type_t type,const uint64_t value)1325 os_channel_attr_set(const channel_attr_t cha, const channel_attr_type_t type,
1326 const uint64_t value)
1327 {
1328 int err = 0;
1329
1330 switch (type) {
1331 case CHANNEL_ATTR_TX_RINGS:
1332 case CHANNEL_ATTR_RX_RINGS:
1333 case CHANNEL_ATTR_TX_SLOTS:
1334 case CHANNEL_ATTR_RX_SLOTS:
1335 case CHANNEL_ATTR_SLOT_BUF_SIZE:
1336 case CHANNEL_ATTR_SLOT_META_SIZE:
1337 case CHANNEL_ATTR_NEXUS_EXTENSIONS:
1338 case CHANNEL_ATTR_NEXUS_MHINTS:
1339 case CHANNEL_ATTR_NEXUS_IFINDEX:
1340 case CHANNEL_ATTR_NEXUS_STATS_SIZE:
1341 case CHANNEL_ATTR_NEXUS_FLOWADV_MAX:
1342 case CHANNEL_ATTR_NEXUS_META_TYPE:
1343 case CHANNEL_ATTR_NEXUS_META_SUBTYPE:
1344 case CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD:
1345 case CHANNEL_ATTR_NEXUS_ADV_SIZE:
1346 case CHANNEL_ATTR_MAX_FRAGS:
1347 case CHANNEL_ATTR_NUM_BUFFERS:
1348 case CHANNEL_ATTR_LARGE_BUF_SIZE:
1349 err = ENOTSUP;
1350 break;
1351
1352 case CHANNEL_ATTR_EXCLUSIVE:
1353 cha->cha_exclusive = (uint32_t)value;
1354 break;
1355
1356 case CHANNEL_ATTR_NO_AUTO_SYNC:
1357 if (value == 0) {
1358 err = ENOTSUP;
1359 }
1360 break;
1361
1362 case CHANNEL_ATTR_MONITOR:
1363 switch (value) {
1364 case CHANNEL_MONITOR_OFF:
1365 case CHANNEL_MONITOR_NO_COPY:
1366 case CHANNEL_MONITOR_COPY:
1367 cha->cha_monitor = (uint32_t)value;
1368 goto done;
1369 }
1370 err = EINVAL;
1371 break;
1372
1373 case CHANNEL_ATTR_TX_LOWAT_UNIT:
1374 case CHANNEL_ATTR_RX_LOWAT_UNIT:
1375 switch (value) {
1376 case CHANNEL_THRESHOLD_UNIT_BYTES:
1377 case CHANNEL_THRESHOLD_UNIT_SLOTS:
1378 if (type == CHANNEL_ATTR_TX_LOWAT_UNIT) {
1379 cha->cha_tx_lowat.cet_unit =
1380 (channel_threshold_unit_t)value;
1381 } else {
1382 cha->cha_rx_lowat.cet_unit =
1383 (channel_threshold_unit_t)value;
1384 }
1385 goto done;
1386 }
1387 err = EINVAL;
1388 break;
1389
1390 case CHANNEL_ATTR_TX_LOWAT_VALUE:
1391 cha->cha_tx_lowat.cet_value = (uint32_t)value;
1392 break;
1393
1394 case CHANNEL_ATTR_RX_LOWAT_VALUE:
1395 cha->cha_rx_lowat.cet_value = (uint32_t)value;
1396 break;
1397
1398 case CHANNEL_ATTR_USER_PACKET_POOL:
1399 cha->cha_user_packet_pool = (value != 0);
1400 break;
1401
1402 case CHANNEL_ATTR_NEXUS_DEFUNCT_OK:
1403 cha->cha_nexus_defunct_ok = (value != 0);
1404 break;
1405
1406 case CHANNEL_ATTR_FILTER:
1407 cha->cha_filter = (uint32_t)value;
1408 break;
1409
1410 case CHANNEL_ATTR_EVENT_RING:
1411 cha->cha_enable_event_ring = (value != 0);
1412 break;
1413
1414 case CHANNEL_ATTR_LOW_LATENCY:
1415 cha->cha_low_latency = (value != 0);
1416 break;
1417
1418 default:
1419 err = EINVAL;
1420 break;
1421 }
1422 done:
1423 return err;
1424 }
1425
1426 int
os_channel_attr_set_key(const channel_attr_t cha,const void * key,const uint32_t key_len)1427 os_channel_attr_set_key(const channel_attr_t cha, const void *key,
1428 const uint32_t key_len)
1429 {
1430 int err = 0;
1431
1432 if ((key == NULL && key_len != 0) || (key != NULL && key_len == 0) ||
1433 (key_len != 0 && key_len > NEXUS_MAX_KEY_LEN)) {
1434 err = EINVAL;
1435 goto done;
1436 }
1437 cha->cha_key_len = 0;
1438 if (key_len == 0 && cha->cha_key != NULL) {
1439 free(cha->cha_key);
1440 cha->cha_key = NULL;
1441 } else if (key != NULL && key_len != 0) {
1442 if (cha->cha_key != NULL) {
1443 free(cha->cha_key);
1444 }
1445 if ((cha->cha_key = malloc(key_len)) == NULL) {
1446 err = ENOMEM;
1447 goto done;
1448 }
1449 cha->cha_key_len = key_len;
1450 bcopy(key, cha->cha_key, key_len);
1451 }
1452 done:
1453 return err;
1454 }
1455
1456 int
os_channel_attr_get(const channel_attr_t cha,const channel_attr_type_t type,uint64_t * value)1457 os_channel_attr_get(const channel_attr_t cha, const channel_attr_type_t type,
1458 uint64_t *value)
1459 {
1460 int err = 0;
1461
1462 switch (type) {
1463 case CHANNEL_ATTR_TX_RINGS:
1464 *value = cha->cha_tx_rings;
1465 break;
1466
1467 case CHANNEL_ATTR_RX_RINGS:
1468 *value = cha->cha_rx_rings;
1469 break;
1470
1471 case CHANNEL_ATTR_TX_SLOTS:
1472 *value = cha->cha_tx_slots;
1473 break;
1474
1475 case CHANNEL_ATTR_RX_SLOTS:
1476 *value = cha->cha_rx_slots;
1477 break;
1478
1479 case CHANNEL_ATTR_SLOT_BUF_SIZE:
1480 *value = cha->cha_buf_size;
1481 break;
1482
1483 case CHANNEL_ATTR_SLOT_META_SIZE:
1484 *value = cha->cha_meta_size;
1485 break;
1486
1487 case CHANNEL_ATTR_NEXUS_STATS_SIZE:
1488 *value = cha->cha_stats_size;
1489 break;
1490
1491 case CHANNEL_ATTR_NEXUS_FLOWADV_MAX:
1492 *value = cha->cha_flowadv_max;
1493 break;
1494
1495 case CHANNEL_ATTR_EXCLUSIVE:
1496 *value = cha->cha_exclusive;
1497 break;
1498
1499 case CHANNEL_ATTR_NO_AUTO_SYNC:
1500 *value = 1;
1501 break;
1502
1503 case CHANNEL_ATTR_MONITOR:
1504 *value = cha->cha_monitor;
1505 break;
1506
1507 case CHANNEL_ATTR_TX_LOWAT_UNIT:
1508 *value = cha->cha_tx_lowat.cet_unit;
1509 break;
1510
1511 case CHANNEL_ATTR_TX_LOWAT_VALUE:
1512 *value = cha->cha_tx_lowat.cet_value;
1513 break;
1514
1515 case CHANNEL_ATTR_RX_LOWAT_UNIT:
1516 *value = cha->cha_rx_lowat.cet_unit;
1517 break;
1518
1519 case CHANNEL_ATTR_RX_LOWAT_VALUE:
1520 *value = cha->cha_rx_lowat.cet_value;
1521 break;
1522
1523 case CHANNEL_ATTR_NEXUS_TYPE:
1524 *value = cha->cha_nexus_type;
1525 break;
1526
1527 case CHANNEL_ATTR_NEXUS_EXTENSIONS:
1528 *value = cha->cha_nexus_extensions;
1529 break;
1530
1531 case CHANNEL_ATTR_NEXUS_MHINTS:
1532 *value = cha->cha_nexus_mhints;
1533 break;
1534
1535 case CHANNEL_ATTR_NEXUS_IFINDEX:
1536 *value = cha->cha_nexus_ifindex;
1537 break;
1538
1539 case CHANNEL_ATTR_NEXUS_META_TYPE:
1540 *value = cha->cha_nexus_meta_type;
1541 break;
1542
1543 case CHANNEL_ATTR_NEXUS_META_SUBTYPE:
1544 *value = cha->cha_nexus_meta_subtype;
1545 break;
1546
1547 case CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD:
1548 *value = cha->cha_nexus_checksum_offload;
1549 break;
1550
1551 case CHANNEL_ATTR_USER_PACKET_POOL:
1552 *value = (cha->cha_user_packet_pool != 0);
1553 break;
1554
1555 case CHANNEL_ATTR_NEXUS_ADV_SIZE:
1556 *value = cha->cha_nexusadv_size;
1557 break;
1558
1559 case CHANNEL_ATTR_NEXUS_DEFUNCT_OK:
1560 *value = cha->cha_nexus_defunct_ok;
1561 break;
1562
1563 case CHANNEL_ATTR_EVENT_RING:
1564 *value = (cha->cha_enable_event_ring != 0);
1565 break;
1566
1567 case CHANNEL_ATTR_MAX_FRAGS:
1568 *value = cha->cha_max_frags;
1569 break;
1570
1571 case CHANNEL_ATTR_NUM_BUFFERS:
1572 *value = cha->cha_num_buffers;
1573 break;
1574
1575 case CHANNEL_ATTR_LOW_LATENCY:
1576 *value = (cha->cha_low_latency != 0);
1577 break;
1578
1579 case CHANNEL_ATTR_LARGE_BUF_SIZE:
1580 *value = cha->cha_large_buf_size;
1581 break;
1582
1583 default:
1584 err = EINVAL;
1585 break;
1586 }
1587
1588 return err;
1589 }
1590
1591 int
os_channel_attr_get_key(const channel_attr_t cha,void * key,uint32_t * key_len)1592 os_channel_attr_get_key(const channel_attr_t cha, void *key,
1593 uint32_t *key_len)
1594 {
1595 int err = 0;
1596
1597 if (key_len == NULL) {
1598 err = EINVAL;
1599 goto done;
1600 } else if (key == NULL || cha->cha_key == NULL) {
1601 *key_len = (cha->cha_key != NULL) ? cha->cha_key_len : 0;
1602 goto done;
1603 }
1604
1605 if (*key_len >= cha->cha_key_len) {
1606 bcopy(cha->cha_key, key, cha->cha_key_len);
1607 *key_len = cha->cha_key_len;
1608 } else {
1609 err = ENOMEM;
1610 }
1611 done:
1612 return err;
1613 }
1614
1615 __attribute__((visibility("hidden")))
1616 static void
os_channel_info2attr(struct channel * chd,channel_attr_t cha)1617 os_channel_info2attr(struct channel *chd, channel_attr_t cha)
1618 {
1619 struct ch_info *cinfo = CHD_INFO(chd);
1620 /* Save these first before we wipe out the attribute */
1621 uint32_t cha_key_len = cha->cha_key_len;
1622 void *cha_key = cha->cha_key;
1623 uint32_t caps;
1624
1625 _CASSERT(NEXUS_META_TYPE_INVALID == CHANNEL_NEXUS_META_TYPE_INVALID);
1626 _CASSERT(NEXUS_META_TYPE_QUANTUM == CHANNEL_NEXUS_META_TYPE_QUANTUM);
1627 _CASSERT(NEXUS_META_TYPE_PACKET == CHANNEL_NEXUS_META_TYPE_PACKET);
1628 _CASSERT(NEXUS_META_SUBTYPE_INVALID ==
1629 CHANNEL_NEXUS_META_SUBTYPE_INVALID);
1630 _CASSERT(NEXUS_META_SUBTYPE_PAYLOAD ==
1631 CHANNEL_NEXUS_META_SUBTYPE_PAYLOAD);
1632 _CASSERT(NEXUS_META_SUBTYPE_RAW == CHANNEL_NEXUS_META_SUBTYPE_RAW);
1633
1634 bzero(cha, sizeof(*cha));
1635 cha->cha_tx_rings = CHD_PARAMS(chd)->nxp_tx_rings;
1636 cha->cha_rx_rings = CHD_PARAMS(chd)->nxp_rx_rings;
1637 cha->cha_tx_slots = CHD_PARAMS(chd)->nxp_tx_slots;
1638 cha->cha_rx_slots = CHD_PARAMS(chd)->nxp_rx_slots;
1639 cha->cha_buf_size = CHD_PARAMS(chd)->nxp_buf_size;
1640 cha->cha_meta_size = CHD_PARAMS(chd)->nxp_meta_size;
1641 cha->cha_stats_size = CHD_PARAMS(chd)->nxp_stats_size;
1642 cha->cha_flowadv_max = CHD_PARAMS(chd)->nxp_flowadv_max;
1643 cha->cha_exclusive = !!(cinfo->cinfo_ch_mode & CHMODE_EXCLUSIVE);
1644 cha->cha_user_packet_pool = !!(cinfo->cinfo_ch_mode &
1645 CHMODE_USER_PACKET_POOL);
1646 cha->cha_nexus_defunct_ok = !!(cinfo->cinfo_ch_mode &
1647 CHMODE_DEFUNCT_OK);
1648 cha->cha_nexusadv_size = CHD_PARAMS(chd)->nxp_nexusadv_size;
1649 if (cinfo->cinfo_ch_mode & CHMODE_MONITOR) {
1650 cha->cha_monitor =
1651 (cinfo->cinfo_ch_mode & CHMODE_MONITOR_NO_COPY) ?
1652 CHANNEL_MONITOR_NO_COPY : CHANNEL_MONITOR_COPY;
1653 } else {
1654 cha->cha_monitor = CHANNEL_MONITOR_OFF;
1655 }
1656 cha->cha_key_len = cha_key_len;
1657 cha->cha_key = cha_key;
1658 cha->cha_tx_lowat = cinfo->cinfo_tx_lowat;
1659 cha->cha_rx_lowat = cinfo->cinfo_rx_lowat;
1660 cha->cha_nexus_type = CHD_PARAMS(chd)->nxp_type;
1661 cha->cha_nexus_extensions = CHD_PARAMS(chd)->nxp_extensions;
1662 cha->cha_nexus_mhints = CHD_PARAMS(chd)->nxp_mhints;
1663 cha->cha_nexus_ifindex = CHD_PARAMS(chd)->nxp_ifindex;
1664 cha->cha_nexus_meta_type = chd->chd_md_type;
1665 cha->cha_nexus_meta_subtype = chd->chd_md_subtype;
1666 cha->cha_enable_event_ring =
1667 (cinfo->cinfo_ch_mode & CHMODE_EVENT_RING) != 0;
1668 cha->cha_low_latency =
1669 (cinfo->cinfo_ch_mode & CHMODE_LOW_LATENCY) != 0;
1670
1671 caps = CHD_PARAMS(chd)->nxp_capabilities;
1672 if (caps & NXPCAP_CHECKSUM_PARTIAL) {
1673 cha->cha_nexus_checksum_offload =
1674 CHANNEL_NEXUS_CHECKSUM_PARTIAL;
1675 } else {
1676 cha->cha_nexus_checksum_offload = 0;
1677 }
1678 cha->cha_max_frags = CHD_PARAMS(chd)->nxp_max_frags;
1679 cha->cha_num_buffers = cinfo->cinfo_num_bufs;
1680 cha->cha_large_buf_size = CHD_PARAMS(chd)->nxp_large_buf_size;
1681 }
1682
1683 void
os_channel_attr_destroy(channel_attr_t cha)1684 os_channel_attr_destroy(channel_attr_t cha)
1685 {
1686 if (cha->cha_key != NULL) {
1687 free(cha->cha_key);
1688 cha->cha_key = NULL;
1689 }
1690 free(cha);
1691 }
1692
1693 int
os_channel_packet_alloc(const channel_t chd,packet_t * ph)1694 os_channel_packet_alloc(const channel_t chd, packet_t *ph)
1695 {
1696 struct __user_channel_ring *ring;
1697 struct channel_ring_desc *chrd;
1698 struct __user_quantum *q;
1699 slot_idx_t idx;
1700 mach_vm_address_t baddr;
1701 uint16_t bdoff;
1702 struct ch_info *ci = CHD_INFO(chd);
1703
1704 if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) {
1705 return ENOTSUP;
1706 }
1707
1708 chrd = &chd->chd_rings[chd->chd_alloc_ring_idx];
1709 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
1710 idx = ring->ring_head;
1711
1712 if (__improbable(idx == ring->ring_tail)) {
1713 /*
1714 * do a sync to get more packets;
1715 * since we are paying the cost of a syscall do a sync for
1716 * free ring as well.
1717 */
1718 int err;
1719 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP,
1720 (chd->chd_sync_flags & ~CHANNEL_SYNCF_ALLOC_BUF));
1721 if (__improbable(err != 0)) {
1722 if (!_CHANNEL_IS_DEFUNCT(chd)) {
1723 SK_ABORT_WITH_CAUSE("packet pool alloc "
1724 "sync failed", err);
1725 /* NOTREACHED */
1726 __builtin_unreachable();
1727 }
1728 return err;
1729 }
1730 }
1731
1732 if (__improbable(idx == ring->ring_tail)) {
1733 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ?
1734 ENXIO : ENOMEM;
1735 }
1736
1737 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
1738 q = _SLOT_METADATA(chrd, ring, idx);
1739 _METADATA_VERIFY(chrd, q);
1740
1741 *ph = SK_PTR_ENCODE(q, chrd->chrd_md_type, chrd->chrd_md_subtype);
1742 _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx));
1743
1744 /*
1745 * Initialize the metadata buffer address. In the event of a
1746 * defunct, we'd be accessing zero-filled memory; this is fine
1747 * since we ignore all changes made to region at that time.
1748 */
1749 baddr = _initialize_metadata_address(chrd, q, &bdoff);
1750 if (__improbable(baddr == 0)) {
1751 return ENXIO;
1752 }
1753 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
1754 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0;
1755 }
1756
1757 int
os_channel_packet_free(const channel_t chd,packet_t ph)1758 os_channel_packet_free(const channel_t chd, packet_t ph)
1759 {
1760 struct __user_channel_ring *ring;
1761 struct channel_ring_desc *chrd;
1762 slot_idx_t idx;
1763 obj_idx_t midx;
1764 struct ch_info *ci = CHD_INFO(chd);
1765
1766 if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) {
1767 return ENOTSUP;
1768 }
1769
1770 chrd = &chd->chd_rings[chd->chd_free_ring_idx];
1771 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
1772
1773 idx = ring->ring_head;
1774 if (__improbable(idx == ring->ring_tail)) {
1775 /*
1776 * do a sync to reclaim space in free ring;
1777 */
1778 int err;
1779 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP,
1780 CHANNEL_SYNCF_FREE);
1781 if (__improbable(err != 0) && !_CHANNEL_IS_DEFUNCT(chd)) {
1782 SK_ABORT_WITH_CAUSE("packet pool free "
1783 "sync failed", err);
1784 /* NOTREACHED */
1785 __builtin_unreachable();
1786 }
1787 }
1788
1789 if (__improbable(idx == ring->ring_tail) && !_CHANNEL_IS_DEFUNCT(chd)) {
1790 SK_ABORT("no free ring space");
1791 /* NOTREACHED */
1792 __builtin_unreachable();
1793 }
1794
1795 /*
1796 * In the event of a defunct, midx will be 0 and we'll end up
1797 * attaching it to the slot; this is fine since we ignore all
1798 * changes made to the slot descriptors at that time.
1799 */
1800 midx = METADATA_IDX(QUM_ADDR(ph));
1801 _SLOT_METADATA_IDX_VERIFY(chrd, QUM_ADDR(ph), midx);
1802 _SLOT_ATTACH_METADATA(_SLOT_DESC(chrd, idx), midx);
1803 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
1804
1805 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0;
1806 }
1807
1808 int
os_channel_slot_attach_packet(const channel_ring_t chrd,const channel_slot_t slot,packet_t ph)1809 os_channel_slot_attach_packet(const channel_ring_t chrd,
1810 const channel_slot_t slot, packet_t ph)
1811 {
1812 slot_idx_t idx;
1813 obj_idx_t midx;
1814
1815 if (__improbable((chrd->chrd_channel->chd_info->cinfo_ch_mode &
1816 CHMODE_USER_PACKET_POOL) == 0)) {
1817 return ENOTSUP;
1818 }
1819
1820 if (__improbable(!__packet_is_finalized(ph))) {
1821 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1822 SK_ABORT("packet not finalized");
1823 /* NOTREACHED */
1824 __builtin_unreachable();
1825 }
1826 goto done;
1827 }
1828
1829 idx = _SLOT_INDEX(chrd, slot);
1830 if (__improbable(!_slot_index_is_valid(chrd->chrd_ring, idx))) {
1831 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1832 SK_ABORT_WITH_CAUSE("Invalid slot", slot);
1833 /* NOTREACHED */
1834 __builtin_unreachable();
1835 }
1836 goto done;
1837 }
1838
1839 if (__improbable(SD_VALID_METADATA(SLOT_DESC_USD(slot)))) {
1840 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1841 SK_ABORT_WITH_CAUSE("Slot has attached packet", slot);
1842 /* NOTREACHED */
1843 __builtin_unreachable();
1844 }
1845 goto done;
1846 }
1847
1848 /*
1849 * In the event of a defunct, midx will be 0 and we'll end up
1850 * attaching it to the slot; this is fine since we ignore all
1851 * changes made to the slot descriptors at that time.
1852 */
1853 midx = METADATA_IDX(QUM_ADDR(ph));
1854 _SLOT_METADATA_IDX_VERIFY(chrd, QUM_ADDR(ph), midx);
1855 _SLOT_ATTACH_METADATA(SLOT_DESC_USD(slot), midx);
1856
1857 done:
1858 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0;
1859 }
1860
1861 int
os_channel_slot_detach_packet(const channel_ring_t chrd,const channel_slot_t slot,packet_t ph)1862 os_channel_slot_detach_packet(const channel_ring_t chrd,
1863 const channel_slot_t slot, packet_t ph)
1864 {
1865 slot_idx_t idx;
1866
1867 if (__improbable((chrd->chrd_channel->chd_info->cinfo_ch_mode &
1868 CHMODE_USER_PACKET_POOL) == 0)) {
1869 return ENOTSUP;
1870 }
1871
1872 idx = _SLOT_INDEX(chrd, slot);
1873 if (__improbable(!_slot_index_is_valid(chrd->chrd_ring, idx))) {
1874 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1875 SK_ABORT_WITH_CAUSE("Invalid slot", slot);
1876 /* NOTREACHED */
1877 __builtin_unreachable();
1878 }
1879 goto done;
1880 }
1881
1882 if (__improbable(!SD_VALID_METADATA(SLOT_DESC_USD(slot)))) {
1883 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1884 SK_ABORT_WITH_CAUSE("Slot has no attached packet",
1885 slot);
1886 /* NOTREACHED */
1887 __builtin_unreachable();
1888 }
1889 goto done;
1890 }
1891
1892 if (__improbable(ph != SK_PTR_ENCODE(_SLOT_METADATA(chrd,
1893 chrd->chrd_ring, idx), chrd->chrd_md_type,
1894 chrd->chrd_md_subtype))) {
1895 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1896 SK_ABORT("packet handle mismatch");
1897 /* NOTREACHED */
1898 __builtin_unreachable();
1899 }
1900 goto done;
1901 }
1902
1903 if (__improbable(!__packet_is_finalized(ph))) {
1904 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1905 SK_ABORT("packet not finalized");
1906 /* NOTREACHED */
1907 __builtin_unreachable();
1908 }
1909 goto done;
1910 }
1911
1912 /*
1913 * In the event of a defunct, we ignore any changes made to
1914 * the slot descriptors, and so doing this is harmless.
1915 */
1916 _SLOT_DETACH_METADATA(SLOT_DESC_USD(slot));
1917
1918 done:
1919 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0;
1920 }
1921
1922 __attribute__((visibility("hidden")))
1923 static inline int
os_channel_purge_packet_alloc_ring(const channel_t chd)1924 os_channel_purge_packet_alloc_ring(const channel_t chd)
1925 {
1926 struct __user_channel_ring *ring;
1927 struct channel_ring_desc *chrd;
1928 uint32_t curr_ws;
1929 slot_idx_t idx;
1930 packet_t ph;
1931 int npkts, err;
1932
1933 chrd = &chd->chd_rings[chd->chd_alloc_ring_idx];
1934 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
1935 idx = ring->ring_head;
1936
1937 /* calculate the number of packets in alloc pool */
1938 npkts = ring->ring_tail - idx;
1939 if (npkts < 0) {
1940 npkts += ring->ring_num_slots;
1941 }
1942
1943 curr_ws = ring->ring_alloc_ws;
1944 while ((uint32_t)npkts-- > curr_ws) {
1945 struct __user_quantum *q;
1946
1947 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
1948 q = _SLOT_METADATA(chrd, ring, idx);
1949 _METADATA_VERIFY(chrd, q);
1950
1951 ph = SK_PTR_ENCODE(q, chrd->chrd_md_type,
1952 chrd->chrd_md_subtype);
1953 _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx));
1954
1955 /*
1956 * Initialize the metadata buffer address. In the event of a
1957 * defunct, we'd be accessing zero-filled memory; this is fine
1958 * since we ignore all changes made to region at that time.
1959 */
1960 if (chrd->chrd_md_type == NEXUS_META_TYPE_PACKET) {
1961 struct __user_packet *p = (struct __user_packet *)q;
1962 uint16_t bcnt = p->pkt_bufs_cnt;
1963 uint16_t bmax = p->pkt_bufs_max;
1964
1965 if (__improbable((bcnt == 0) || (bmax == 0))) {
1966 if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) {
1967 SK_ABORT("pkt pool purge, bad bufcnt");
1968 /* NOTREACHED */
1969 __builtin_unreachable();
1970 } else {
1971 return ENXIO;
1972 }
1973 }
1974 /*
1975 * alloc ring will not have multi-buflet packets.
1976 */
1977 _PKT_BUFCNT_VERIFY(chrd, bcnt, 1);
1978 }
1979 *(mach_vm_address_t *) (uintptr_t)&q->qum_buf[0].buf_addr =
1980 _CHANNEL_RING_BUF(chrd, ring, &q->qum_buf[0]);
1981 idx = _CHANNEL_RING_NEXT(ring, idx);
1982 ring->ring_head = idx;
1983 err = os_channel_packet_free(chd, ph);
1984 if (__improbable(err != 0)) {
1985 if (!_CHANNEL_IS_DEFUNCT(chd)) {
1986 SK_ABORT_WITH_CAUSE("packet pool purge "
1987 "free failed", err);
1988 /* NOTREACHED */
1989 __builtin_unreachable();
1990 }
1991 return err;
1992 }
1993 }
1994
1995 return 0;
1996 }
1997
1998 __attribute__((visibility("hidden")))
1999 static inline int
os_channel_purge_buflet_alloc_ring(const channel_t chd)2000 os_channel_purge_buflet_alloc_ring(const channel_t chd)
2001 {
2002 struct __user_channel_ring *ring;
2003 struct channel_ring_desc *chrd;
2004 uint32_t curr_ws;
2005 slot_idx_t idx;
2006 int nbfts, err;
2007
2008 chrd = &chd->chd_rings[chd->chd_buf_alloc_ring_idx];
2009 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
2010 idx = ring->ring_head;
2011
2012 /* calculate the number of packets in alloc pool */
2013 nbfts = ring->ring_tail - idx;
2014 if (nbfts < 0) {
2015 nbfts += ring->ring_num_slots;
2016 }
2017
2018 curr_ws = ring->ring_alloc_ws;
2019 while ((uint32_t)nbfts-- > curr_ws) {
2020 struct __user_buflet *ubft;
2021 obj_idx_t nbft_idx;
2022
2023 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
2024 ubft = _SLOT_BFT_METADATA(chrd, ring, idx);
2025 _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx));
2026
2027 /*
2028 * Initialize the buflet metadata buffer address.
2029 */
2030 *(mach_vm_address_t *)(uintptr_t)&(ubft->buf_addr) =
2031 _CHANNEL_RING_BUF(chrd, ring, ubft);
2032 if (__improbable(ubft->buf_addr == 0)) {
2033 SK_ABORT_WITH_CAUSE("buflet with NULL buffer",
2034 ubft->buf_idx);
2035 /* NOTREACHED */
2036 __builtin_unreachable();
2037 }
2038
2039 nbft_idx = ubft->buf_nbft_idx;
2040 if (__improbable(nbft_idx != OBJ_IDX_NONE)) {
2041 if (_CHANNEL_IS_DEFUNCT(chd)) {
2042 return ENXIO;
2043 } else {
2044 SK_ABORT_WITH_CAUSE("buflet with invalid nidx",
2045 nbft_idx);
2046 /* NOTREACHED */
2047 __builtin_unreachable();
2048 }
2049 }
2050
2051 idx = _CHANNEL_RING_NEXT(ring, idx);
2052 ring->ring_head = idx;
2053 err = os_channel_buflet_free(chd, ubft);
2054 if (__improbable(err != 0)) {
2055 if (!_CHANNEL_IS_DEFUNCT(chd)) {
2056 SK_ABORT_WITH_CAUSE("buflet pool purge "
2057 "free failed", err);
2058 /* NOTREACHED */
2059 __builtin_unreachable();
2060 }
2061 return err;
2062 }
2063 }
2064
2065 return 0;
2066 }
2067
2068 int
os_channel_packet_pool_purge(const channel_t chd)2069 os_channel_packet_pool_purge(const channel_t chd)
2070 {
2071 struct ch_info *ci = CHD_INFO(chd);
2072 int err;
2073
2074 if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) {
2075 return ENOTSUP;
2076 }
2077
2078 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP,
2079 (chd->chd_sync_flags | ~CHANNEL_SYNCF_FREE | CHANNEL_SYNCF_PURGE));
2080 if (__improbable(err != 0)) {
2081 if (!_CHANNEL_IS_DEFUNCT(chd)) {
2082 SK_ABORT_WITH_CAUSE("packet pool purge sync failed",
2083 err);
2084 /* NOTREACHED */
2085 __builtin_unreachable();
2086 }
2087 return err;
2088 }
2089
2090 err = os_channel_purge_packet_alloc_ring(chd);
2091 if (__improbable(err != 0)) {
2092 return err;
2093 }
2094
2095 if (_num_allocator_rings(CHD_SCHEMA(chd)) > 2) {
2096 err = os_channel_purge_buflet_alloc_ring(chd);
2097 if (__improbable(err != 0)) {
2098 return err;
2099 }
2100 }
2101
2102 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, CHANNEL_SYNCF_FREE);
2103 if (__improbable(err != 0)) {
2104 if (!_CHANNEL_IS_DEFUNCT(chd)) {
2105 SK_ABORT_WITH_CAUSE("packet pool free sync failed",
2106 err);
2107 /* NOTREACHED */
2108 __builtin_unreachable();
2109 }
2110 return err;
2111 }
2112
2113 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0;
2114 }
2115
2116 int
os_channel_get_next_event_handle(const channel_t chd,os_channel_event_handle_t * ehandle,os_channel_event_type_t * etype,uint32_t * nevents)2117 os_channel_get_next_event_handle(const channel_t chd,
2118 os_channel_event_handle_t *ehandle, os_channel_event_type_t *etype,
2119 uint32_t *nevents)
2120 {
2121 struct __kern_channel_event_metadata *emd;
2122 struct __user_channel_ring *ring;
2123 struct channel_ring_desc *chrd;
2124 struct __user_quantum *qum;
2125 mach_vm_address_t baddr;
2126 uint16_t bdoff;
2127 slot_idx_t idx;
2128 struct __user_channel_schema *csm = CHD_SCHEMA(chd);
2129 struct ch_info *ci = CHD_INFO(chd);
2130
2131 if (__improbable((ehandle == NULL) || (etype == NULL) ||
2132 (nevents == NULL))) {
2133 return EINVAL;
2134 }
2135 if (__improbable((ci->cinfo_ch_mode & CHMODE_EVENT_RING) == 0)) {
2136 return ENOTSUP;
2137 }
2138 *ehandle = NULL;
2139 chrd = &chd->chd_rings[_num_tx_rings(ci) + _num_rx_rings(ci) +
2140 _num_allocator_rings(csm)];
2141 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
2142 idx = ring->ring_head;
2143
2144 if (__improbable(idx == ring->ring_tail)) {
2145 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ?
2146 ENXIO : ENODATA;
2147 }
2148 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
2149 qum = _SLOT_METADATA(chrd, ring, idx);
2150 _METADATA_VERIFY(chrd, qum);
2151 _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx));
2152
2153 baddr = _initialize_metadata_address(chrd, qum, &bdoff);
2154 if (__improbable(baddr == 0)) {
2155 return ENXIO;
2156 }
2157 *ehandle = SK_PTR_ENCODE(qum, chrd->chrd_md_type,
2158 chrd->chrd_md_subtype);
2159 emd = (void *)(baddr + bdoff);
2160 *etype = emd->emd_etype;
2161 *nevents = emd->emd_nevents;
2162 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
2163 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0;
2164 }
2165
2166 int
os_channel_event_free(const channel_t chd,os_channel_event_handle_t ehandle)2167 os_channel_event_free(const channel_t chd, os_channel_event_handle_t ehandle)
2168 {
2169 return os_channel_packet_free(chd, (packet_t)ehandle);
2170 }
2171
2172 int
os_channel_get_interface_advisory(const channel_t chd,struct ifnet_interface_advisory * advisory)2173 os_channel_get_interface_advisory(const channel_t chd,
2174 struct ifnet_interface_advisory *advisory)
2175 {
2176 struct __kern_netif_intf_advisory *intf_adv;
2177 struct __kern_nexus_adv_metadata *adv_md;
2178 nexus_advisory_type_t adv_type;
2179
2180 /*
2181 * Interface advisory is only supported for netif and flowswitch.
2182 */
2183 adv_md = CHD_NX_ADV_MD(chd);
2184 if (adv_md == NULL) {
2185 return ENOENT;
2186 }
2187 adv_type = adv_md->knam_type;
2188 if (__improbable(adv_type != NEXUS_ADVISORY_TYPE_NETIF &&
2189 adv_type != NEXUS_ADVISORY_TYPE_FLOWSWITCH)) {
2190 return _CHANNEL_IS_DEFUNCT(chd) ? ENXIO : ENOENT;
2191 }
2192 if (adv_type == NEXUS_ADVISORY_TYPE_NETIF) {
2193 intf_adv = &(CHD_NX_ADV_NETIF(adv_md))->__kern_intf_adv;
2194 } else {
2195 intf_adv = &(CHD_NX_ADV_FSW(adv_md))->_nxadv_intf_adv;
2196 }
2197 if (intf_adv->cksum != os_cpu_copy_in_cksum(&intf_adv->adv, advisory,
2198 sizeof(*advisory), 0)) {
2199 return _CHANNEL_IS_DEFUNCT(chd) ? ENXIO : EAGAIN;
2200 }
2201 return 0;
2202 }
2203
2204 int
os_channel_configure_interface_advisory(const channel_t chd,boolean_t enable)2205 os_channel_configure_interface_advisory(const channel_t chd, boolean_t enable)
2206 {
2207 uint32_t value = enable;
2208
2209 return __channel_set_opt(chd->chd_fd, CHOPT_IF_ADV_CONF,
2210 &value, sizeof(value));
2211 }
2212
2213 int
os_channel_buflet_alloc(const channel_t chd,buflet_t * bft)2214 os_channel_buflet_alloc(const channel_t chd, buflet_t *bft)
2215 {
2216 struct __user_channel_ring *ring;
2217 struct channel_ring_desc *chrd;
2218 struct __user_buflet *ubft;
2219 obj_idx_t nbft_idx;
2220 slot_idx_t idx;
2221 struct ch_info *ci = CHD_INFO(chd);
2222
2223 if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) {
2224 return ENOTSUP;
2225 }
2226
2227 if (__improbable(_num_allocator_rings(CHD_SCHEMA(chd)) < 4)) {
2228 return ENOTSUP;
2229 }
2230
2231 chrd = &chd->chd_rings[chd->chd_buf_alloc_ring_idx];
2232 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
2233 idx = ring->ring_head;
2234
2235 if (__improbable(idx == ring->ring_tail)) {
2236 /*
2237 * do a sync to get more buflets;
2238 */
2239 int err;
2240 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP,
2241 CHANNEL_SYNCF_ALLOC_BUF | CHANNEL_SYNCF_FREE);
2242 if (__improbable(err != 0)) {
2243 if (!_CHANNEL_IS_DEFUNCT(chd)) {
2244 SK_ABORT_WITH_CAUSE("buflet pool alloc "
2245 "sync failed", err);
2246 /* NOTREACHED */
2247 __builtin_unreachable();
2248 }
2249 return err;
2250 }
2251 }
2252
2253 if (__improbable(idx == ring->ring_tail)) {
2254 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ?
2255 ENXIO : ENOMEM;
2256 }
2257
2258 _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx));
2259 ubft = _SLOT_BFT_METADATA(chrd, ring, idx);
2260 _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx));
2261
2262 /*
2263 * Initialize the buflet metadata buffer address.
2264 */
2265 *(mach_vm_address_t *)(uintptr_t)&(ubft->buf_addr) =
2266 _CHANNEL_RING_BUF(chrd, ring, ubft);
2267 if (__improbable(ubft->buf_addr == 0)) {
2268 SK_ABORT_WITH_CAUSE("buflet alloc with NULL buffer",
2269 ubft->buf_idx);
2270 /* NOTREACHED */
2271 __builtin_unreachable();
2272 }
2273 nbft_idx = ubft->buf_nbft_idx;
2274 if (__improbable(nbft_idx != OBJ_IDX_NONE)) {
2275 if (_CHANNEL_IS_DEFUNCT(chd)) {
2276 return ENXIO;
2277 } else {
2278 SK_ABORT_WITH_CAUSE("buflet alloc with invalid nidx",
2279 nbft_idx);
2280 /* NOTREACHED */
2281 __builtin_unreachable();
2282 }
2283 }
2284 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
2285 *bft = ubft;
2286 return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0;
2287 }
2288
2289 int
os_channel_buflet_free(const channel_t chd,buflet_t ubft)2290 os_channel_buflet_free(const channel_t chd, buflet_t ubft)
2291 {
2292 struct __user_channel_ring *ring;
2293 struct channel_ring_desc *chrd;
2294 slot_idx_t idx;
2295 obj_idx_t midx;
2296 struct ch_info *ci = CHD_INFO(chd);
2297
2298 if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) {
2299 return ENOTSUP;
2300 }
2301
2302 if (__improbable(_num_allocator_rings(CHD_SCHEMA(chd)) < 4)) {
2303 return ENOTSUP;
2304 }
2305
2306 chrd = &chd->chd_rings[chd->chd_buf_free_ring_idx];
2307 ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring);
2308
2309 idx = ring->ring_head;
2310 if (__improbable(idx == ring->ring_tail)) {
2311 /*
2312 * do a sync to reclaim space in free ring;
2313 */
2314 int err;
2315 err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP,
2316 CHANNEL_SYNCF_FREE);
2317 if (__improbable(err != 0) && !_CHANNEL_IS_DEFUNCT(chd)) {
2318 SK_ABORT_WITH_CAUSE("buflet pool free "
2319 "sync failed", err);
2320 /* NOTREACHED */
2321 __builtin_unreachable();
2322 }
2323 }
2324
2325 if (__improbable(idx == ring->ring_tail) && !_CHANNEL_IS_DEFUNCT(chd)) {
2326 SK_ABORT("no ring space in buflet free ring");
2327 /* NOTREACHED */
2328 __builtin_unreachable();
2329 }
2330
2331 midx = _BFT_INDEX(chrd, ubft);
2332 _SLOT_BFT_METADATA_IDX_VERIFY(chrd, ubft, midx);
2333 _SLOT_ATTACH_METADATA(_SLOT_DESC(chrd, idx), midx);
2334 ring->ring_head = _CHANNEL_RING_NEXT(ring, idx);
2335
2336 return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0;
2337 }
2338