/* * Copyright (c) 2015-2023 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include #include #include #include #include #include #include #ifndef LIBSYSCALL_INTERFACE #error "LIBSYSCALL_INTERFACE not defined" #endif /* !LIBSYSCALL_INTERFACE */ /* * Defined here as we don't have Libc */ extern int __getpid(void); extern int __kill(int pid, int signum, int posix); extern int __exit(int) __attribute__((noreturn)); static ring_id_t _ring_id(struct ch_info *cinfo, const ring_id_type_t type); static void os_channel_info2attr(struct channel *chd, channel_attr_t cha); static int _flowadv_id_equal(struct __flowadv_entry *, uuid_t); /* * This is pretty much what an inlined memcmp() would do for UUID * comparison; since we don't have access to memcmp() here, we * manually handle it ourselves. */ #define UUID_COMPARE(a, b) \ (a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3] && \ a[4] == b[4] && a[5] == b[5] && a[6] == b[6] && a[7] == b[7] && \ a[8] == b[8] && a[9] == b[9] && a[10] == b[10] && a[11] == b[11] && \ a[12] == b[12] && a[13] == b[13] && a[14] == b[14] && a[15] == b[15]) #define _SLOT_INDEX(_chrd, _slot) \ ((slot_idx_t)((_slot - (_chrd)->chrd_slot_desc))) #define _SLOT_DESC(_chrd, _idx) \ (SLOT_DESC_USD(&(_chrd)->chrd_slot_desc[_idx])) #define _METADATA(_chrd, _ring, _midx) \ ((void *)((_chrd)->chrd_md_base_addr + \ ((_midx) * (_ring)->ring_md_size) + METADATA_PREAMBLE_SZ)) #define _SLOT_METADATA(_chrd, _ring, _idx) \ _METADATA(_chrd, _ring, _SLOT_DESC(_chrd, _idx)->sd_md_idx) #define _SLOT_METADATA_IDX_VERIFY(_chrd, _md, _midx) do { \ if (__improbable((_md) != _METADATA((_chrd), (_chrd)->chrd_ring, \ (_midx))) && !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \ SK_ABORT_WITH_CAUSE("bad packet handle", (_midx)); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _BFT_INDEX(_chrd, _bft) (_bft)->buf_bft_idx_reg #define _SLOT_BFT_METADATA(_chrd, _ring, _idx) \ _CHANNEL_RING_BFT(_chrd, _ring, _SLOT_DESC(_chrd, _idx)->sd_md_idx) #define _SLOT_BFT_METADATA_IDX_VERIFY(_chrd, _md, _midx) do { \ if (__improbable((mach_vm_address_t)(_md) != \ _CHANNEL_RING_BFT((_chrd), (_chrd)->chrd_ring, (_midx))) && \ !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \ SK_ABORT_WITH_CAUSE("bad buflet handle", (_midx)); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _SLOT_DESC_VERIFY(_chrd, _sdp) do { \ if (__improbable(!SD_VALID_METADATA(_sdp)) && \ !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \ SK_ABORT("Slot descriptor has no metadata"); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _METADATA_VERIFY(_chrd, _md) do { \ if (__improbable(METADATA_PREAMBLE(_md)->mdp_redzone != \ (((mach_vm_address_t)(_md) - (_chrd)->chrd_md_base_addr) ^ \ __os_ch_md_redzone_cookie)) && \ !_CHANNEL_RING_IS_DEFUNCT(_chrd)) { \ SK_ABORT_WITH_CAUSE("Metadata redzone corrupted", \ METADATA_PREAMBLE(_md)->mdp_redzone); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _PKT_BUFCNT_VERIFY(_chrd, _bcnt, _bmax) do { \ if (__improbable((_chrd)->chrd_max_bufs < (_bmax))) { \ SK_ABORT_WITH_CAUSE("Invalid max bufcnt", (_bmax)); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ if (__improbable((_bcnt) > (_bmax))) { \ SK_ABORT_WITH_CAUSE("Invalid bufcnt", (_bcnt)); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _ABORT_MSGSZ 1024 #define _SCHEMA_VER_VERIFY(_chd) do { \ /* ensure all stores are globally visible */ \ os_atomic_thread_fence(seq_cst); \ if (CHD_SCHEMA(_chd)->csm_ver != CSM_CURRENT_VERSION) { \ char *_msg = malloc(_ABORT_MSGSZ); \ uint32_t _ver = (uint32_t)CHD_SCHEMA(_chd)->csm_ver; \ /* we're stuck with %x and %s formatters */ \ (void) _mach_snprintf(_msg, _ABORT_MSGSZ, \ "Schema region version mismatch: 0x%x != 0x%x\n" \ "Kernel version: %s - did you forget to install " \ "a matching libsystem_kernel.dylib?\n" \ "Kernel UUID: %x%x%x%x-%x%x-%x%x-%x%x-%x%x%x%x%x%x", \ _ver, (uint32_t)CSM_CURRENT_VERSION, \ CHD_SCHEMA(_chd)->csm_kern_name, \ CHD_SCHEMA(_chd)->csm_kern_uuid[0], \ CHD_SCHEMA(_chd)->csm_kern_uuid[1], \ CHD_SCHEMA(_chd)->csm_kern_uuid[2], \ CHD_SCHEMA(_chd)->csm_kern_uuid[3], \ CHD_SCHEMA(_chd)->csm_kern_uuid[4], \ CHD_SCHEMA(_chd)->csm_kern_uuid[5], \ CHD_SCHEMA(_chd)->csm_kern_uuid[6], \ CHD_SCHEMA(_chd)->csm_kern_uuid[7], \ CHD_SCHEMA(_chd)->csm_kern_uuid[8], \ CHD_SCHEMA(_chd)->csm_kern_uuid[9], \ CHD_SCHEMA(_chd)->csm_kern_uuid[10], \ CHD_SCHEMA(_chd)->csm_kern_uuid[11], \ CHD_SCHEMA(_chd)->csm_kern_uuid[12], \ CHD_SCHEMA(_chd)->csm_kern_uuid[13], \ CHD_SCHEMA(_chd)->csm_kern_uuid[14], \ CHD_SCHEMA(_chd)->csm_kern_uuid[15]); \ SK_ABORT_DYNAMIC(_msg); \ /* NOTREACHED */ \ __builtin_unreachable(); \ } \ } while (0) #define _SLOT_ATTACH_METADATA(_usd, _md_idx) do { \ (_usd)->sd_md_idx = (_md_idx); \ (_usd)->sd_flags |= SD_IDX_VALID; \ } while (0) #define _SLOT_DETACH_METADATA(_usd) do { \ (_usd)->sd_md_idx = OBJ_IDX_NONE; \ (_usd)->sd_flags &= ~SD_IDX_VALID; \ } while (0) #define _CHANNEL_OFFSET(_type, _ptr, _offset) \ ((_type)(void *)((uintptr_t)(_ptr) + (_offset))) #define _CHANNEL_SCHEMA(_base, _off) \ _CHANNEL_OFFSET(struct __user_channel_schema *, _base, _off) #define _CHANNEL_RING_DEF_BUF(_chrd, _ring, _idx) \ ((_chrd)->chrd_def_buf_base_addr + \ ((_idx) * (_ring)->ring_def_buf_size)) #define _CHANNEL_RING_LARGE_BUF(_chrd, _ring, _idx) \ ((_chrd)->chrd_large_buf_base_addr + \ ((_idx) * (_ring)->ring_large_buf_size)) #define _CHANNEL_RING_BUF(_chrd, _ring, _bft) \ BUFLET_HAS_LARGE_BUF(_bft) ? \ _CHANNEL_RING_LARGE_BUF(_chrd, _ring, (_bft)->buf_idx) : \ _CHANNEL_RING_DEF_BUF(_chrd, _ring, (_bft)->buf_idx) #define _CHANNEL_RING_BFT(_chrd, _ring, _idx) \ ((_chrd)->chrd_bft_base_addr + ((_idx) * (_ring)->ring_bft_size)) #define _CHANNEL_RING_NEXT(_ring, _cur) \ (__improbable((_cur) + 1 == (_ring)->ring_num_slots) ? 0 : (_cur) + 1) #define _CHANNEL_RING_IS_DEFUNCT(_chrd) \ (!(*(_chrd)->chrd_csm_flags & CSM_ACTIVE)) #define _CHANNEL_IS_DEFUNCT(_chd) \ (!(CHD_SCHEMA(_chd)->csm_flags & CSM_ACTIVE)) #define _CH_PKT_GET_FIRST_BUFLET(_pkt, _bft, _chrd, _ring) do { \ if (__probable((_pkt)->pkt_qum_buf.buf_idx != OBJ_IDX_NONE)) { \ (_bft) = &(_pkt)->pkt_qum_buf; \ } else if ((_pkt)->pkt_qum_buf.buf_nbft_idx != OBJ_IDX_NONE) { \ (_bft) = _CHANNEL_RING_BFT(_chrd, _ring, \ (_pkt)->pkt_qum_buf.buf_nbft_idx); \ } else { \ (_bft) = NULL; \ } \ } while (0) /* * A per process copy of the channel metadata redzone cookie. */ __attribute__((visibility("hidden"))) static uint64_t __os_ch_md_redzone_cookie = 0; __attribute__((always_inline, visibility("hidden"))) static inline uint32_t _num_tx_rings(struct ch_info *ci) { ring_id_t first, last; first = _ring_id(ci, CHANNEL_FIRST_TX_RING); last = _ring_id(ci, CHANNEL_LAST_TX_RING); return (last - first) + 1; } __attribute__((always_inline, visibility("hidden"))) static inline uint32_t _num_rx_rings(struct ch_info *ci) { ring_id_t first, last; first = _ring_id(ci, CHANNEL_FIRST_RX_RING); last = _ring_id(ci, CHANNEL_LAST_RX_RING); return (last - first) + 1; } __attribute__((always_inline, visibility("hidden"))) static inline uint32_t _num_allocator_rings(const struct __user_channel_schema *csm) { return csm->csm_allocator_ring_pairs << 1; } __attribute__((visibility("hidden"))) static void os_channel_init_ring(struct channel_ring_desc *chrd, struct channel *chd, uint32_t ring_index) { struct __user_channel_schema *csm = CHD_SCHEMA(chd); struct __user_channel_ring *ring = NULL; struct __slot_desc *sd = NULL; nexus_meta_type_t md_type; nexus_meta_subtype_t md_subtype; ring = _CHANNEL_OFFSET(struct __user_channel_ring *, csm, csm->csm_ring_ofs[ring_index].ring_off); sd = _CHANNEL_OFFSET(struct __slot_desc *, csm, csm->csm_ring_ofs[ring_index].sd_off); md_type = csm->csm_md_type; md_subtype = csm->csm_md_subtype; if (ring == NULL || sd == NULL) { SK_ABORT("Channel schema not valid"); /* NOTREACHED */ __builtin_unreachable(); } else if (!(md_type == NEXUS_META_TYPE_QUANTUM || md_type == NEXUS_META_TYPE_PACKET)) { SK_ABORT_WITH_CAUSE("Metadata type unknown", md_type); /* NOTREACHED */ __builtin_unreachable(); } else if (!(md_subtype == NEXUS_META_SUBTYPE_PAYLOAD || md_subtype == NEXUS_META_SUBTYPE_RAW)) { SK_ABORT_WITH_CAUSE("Metadata subtype unknown", md_subtype); /* NOTREACHED */ __builtin_unreachable(); } chrd->chrd_slot_desc = sd; chrd->chrd_csm_flags = &chd->chd_schema->csm_flags; /* const overrides */ *(struct channel **)(uintptr_t)&chrd->chrd_channel = chd; *(struct __user_channel_ring **)(uintptr_t)&chrd->chrd_ring = ring; *(nexus_meta_type_t *)(uintptr_t)&chrd->chrd_md_type = md_type; *(nexus_meta_subtype_t *)(uintptr_t)&chrd->chrd_md_subtype = md_subtype; *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_shmem_base_addr = CHD_INFO(chd)->cinfo_mem_base; *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_def_buf_base_addr = (mach_vm_address_t)((uintptr_t)ring + ring->ring_def_buf_base); *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_md_base_addr = (mach_vm_address_t)((uintptr_t)ring + ring->ring_md_base); *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_sd_base_addr = (mach_vm_address_t)((uintptr_t)ring + ring->ring_sd_base); *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_bft_base_addr = (mach_vm_address_t)((uintptr_t)ring + ring->ring_bft_base); *(mach_vm_address_t *)(uintptr_t)&chrd->chrd_large_buf_base_addr = (mach_vm_address_t)((uintptr_t)ring + ring->ring_large_buf_base); *(uint32_t *)(uintptr_t)&chrd->chrd_max_bufs = CHD_PARAMS(chd)->nxp_max_frags; } __attribute__((always_inline, visibility("hidden"))) static inline mach_vm_address_t _initialize_metadata_address(const channel_ring_t chrd, struct __user_quantum *q, uint16_t *bdoff) { int i; struct __user_buflet *ubft0; const struct __user_channel_ring *ring = chrd->chrd_ring; switch (chrd->chrd_md_type) { case NEXUS_META_TYPE_PACKET: { struct __user_buflet *ubft, *pbft; struct __user_packet *p = (struct __user_packet *)q; uint16_t bcnt = p->pkt_bufs_cnt; uint16_t bmax = p->pkt_bufs_max; _CASSERT(sizeof(p->pkt_qum_buf.buf_addr) == sizeof(mach_vm_address_t)); /* * In the event of a defunct, we'd be accessing zero-filled * memory and end up with 0 for bcnt or bmax. */ if (__improbable((bcnt == 0) || (bmax == 0))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("bad bufcnt"); /* NOTREACHED */ __builtin_unreachable(); } return 0; } _PKT_BUFCNT_VERIFY(chrd, bcnt, bmax); _CH_PKT_GET_FIRST_BUFLET(p, ubft, chrd, ring); if (__improbable(ubft == NULL)) { SK_ABORT("bad packet: no buflet"); /* NOTREACHED */ __builtin_unreachable(); } /* * special handling for empty packet buflet. */ if (__improbable(p->pkt_qum_buf.buf_idx == OBJ_IDX_NONE)) { *__DECONST(mach_vm_address_t *, &p->pkt_qum_buf.buf_addr) = 0; *__DECONST(mach_vm_address_t *, &p->pkt_qum_buf.buf_nbft_addr) = (mach_vm_address_t)ubft; } ubft0 = ubft; for (i = 0; (i < bcnt) && (ubft != NULL); i++) { pbft = ubft; if (__probable(pbft->buf_idx != OBJ_IDX_NONE)) { *(mach_vm_address_t *)(uintptr_t) &(pbft->buf_addr) = _CHANNEL_RING_BUF(chrd, ring, pbft); } else { *(mach_vm_address_t *)(uintptr_t) &(pbft->buf_addr) = NULL; } if (pbft->buf_nbft_idx != OBJ_IDX_NONE) { ubft = _CHANNEL_RING_BFT(chrd, ring, pbft->buf_nbft_idx); } else { ubft = NULL; } *__DECONST(mach_vm_address_t *, &pbft->buf_nbft_addr) = (mach_vm_address_t)ubft; } if (__improbable(pbft->buf_nbft_idx != OBJ_IDX_NONE)) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("non terminating buflet chain"); /* NOTREACHED */ __builtin_unreachable(); } return 0; } if (__improbable(i != bcnt)) { SK_ABORT_WITH_CAUSE("invalid buflet count", bcnt); /* NOTREACHED */ __builtin_unreachable(); } break; } default: ubft0 = &q->qum_buf[0]; _CASSERT(sizeof(q->qum_buf[0].buf_addr) == sizeof(mach_vm_address_t)); /* immutable: compute pointers from the index */ *(mach_vm_address_t *)(uintptr_t)&ubft0->buf_addr = _CHANNEL_RING_BUF(chrd, ring, ubft0); break; } /* return address and offset of the first buffer */ *bdoff = ubft0->buf_doff; return ubft0->buf_addr; } /* * _slot_index_is_valid * - verify that the slot index is within valid bounds * - if the head is less than (or equal to) the tail (case A below) * head <= valid < tail * - if the head is greater than the tail (case B below) * valid < tail * or * head <= valid < num_slots * * case A: x x x x x x x H o o o o o T x x x x x x * case B: o o o o o T x x x x H o o o o o o o o o * * 'H' - head * 'T' - tail * 'x' - invalid * 'o' - valid */ __attribute__((always_inline, visibility("hidden"))) static inline int _slot_index_is_valid(const struct __user_channel_ring *ring, slot_idx_t idx) { int is_valid = 0; if (ring->ring_head <= ring->ring_tail) { if (__probable(idx >= ring->ring_head && idx < ring->ring_tail)) { is_valid = 1; } } else { if (__probable(idx < ring->ring_tail || (idx >= ring->ring_head && idx < ring->ring_num_slots))) { is_valid = 1; } } return is_valid; } channel_t os_channel_create_extended(const uuid_t uuid, const nexus_port_t port, const ring_dir_t dir, const ring_id_t ring, const channel_attr_t cha) { uint32_t num_tx_rings, num_rx_rings, num_allocator_rings; uint32_t ring_offset, ring_index, num_event_rings, num_large_buf_alloc_rings; struct __user_channel_schema *ucs; struct channel *chd = NULL; struct ch_info *ci = NULL; struct ch_init init; int i, fd = -1; int err = 0; size_t chd_sz; SK_ALIGN64_CASSERT(struct ch_info, cinfo_mem_map_size); switch (dir) { case CHANNEL_DIR_TX_RX: case CHANNEL_DIR_TX: case CHANNEL_DIR_RX: break; default: err = EINVAL; goto done; } ci = malloc(CHD_INFO_SIZE); if (ci == NULL) { err = errno = ENOMEM; goto done; } bzero(ci, CHD_INFO_SIZE); bzero(&init, sizeof(init)); init.ci_version = CHANNEL_INIT_CURRENT_VERSION; if (cha != NULL) { if (cha->cha_exclusive != 0) { init.ci_ch_mode |= CHMODE_EXCLUSIVE; } if (cha->cha_user_packet_pool != 0) { init.ci_ch_mode |= CHMODE_USER_PACKET_POOL; } if (cha->cha_nexus_defunct_ok != 0) { init.ci_ch_mode |= CHMODE_DEFUNCT_OK; } if (cha->cha_enable_event_ring != 0) { /* User packet pool is required for event rings */ if (cha->cha_user_packet_pool == 0) { err = EINVAL; goto done; } init.ci_ch_mode |= CHMODE_EVENT_RING; } if (cha->cha_monitor != 0) { if (dir == CHANNEL_DIR_TX_RX) { init.ci_ch_mode |= CHMODE_MONITOR; } else if (dir == CHANNEL_DIR_TX) { init.ci_ch_mode |= CHMODE_MONITOR_TX; } else if (dir == CHANNEL_DIR_RX) { init.ci_ch_mode |= CHMODE_MONITOR_RX; } if (cha->cha_monitor == CHANNEL_MONITOR_NO_COPY) { init.ci_ch_mode |= CHMODE_MONITOR_NO_COPY; } } if (cha->cha_filter != 0) { init.ci_ch_mode |= CHMODE_FILTER; } if (cha->cha_low_latency != 0) { init.ci_ch_mode |= CHMODE_LOW_LATENCY; } init.ci_key_len = cha->cha_key_len; init.ci_key = cha->cha_key; init.ci_tx_lowat = cha->cha_tx_lowat; init.ci_rx_lowat = cha->cha_rx_lowat; } init.ci_ch_ring_id = ring; init.ci_nx_port = port; bcopy(uuid, init.ci_nx_uuid, sizeof(uuid_t)); fd = __channel_open(&init, sizeof(init)); if (fd == -1) { err = errno; goto done; } err = __channel_get_info(fd, ci, CHD_INFO_SIZE); if (err != 0) { err = errno; goto done; } ucs = _CHANNEL_SCHEMA(ci->cinfo_mem_base, ci->cinfo_schema_offset); num_tx_rings = _num_tx_rings(ci); /* # of channel tx rings */ num_rx_rings = _num_rx_rings(ci); /* # of channel rx rings */ num_allocator_rings = _num_allocator_rings(ucs); num_event_rings = ucs->csm_num_event_rings; num_large_buf_alloc_rings = ucs->csm_large_buf_alloc_rings; /* * if the user requested packet allocation mode for channel, then * check that channel was opened in packet allocation mode and * allocator rings were created. */ if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) && ((num_allocator_rings < 2) || !(ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL))) { err = errno = ENXIO; goto done; } if ((init.ci_ch_mode & CHMODE_EVENT_RING) && ((num_event_rings == 0) || !(ci->cinfo_ch_mode & CHMODE_EVENT_RING))) { err = errno = ENXIO; goto done; } chd_sz = CHD_SIZE(num_tx_rings + num_rx_rings + num_allocator_rings + num_event_rings + num_large_buf_alloc_rings); chd = malloc(chd_sz); if (chd == NULL) { err = errno = ENOMEM; goto done; } bzero(chd, chd_sz); chd->chd_fd = fd; chd->chd_guard = init.ci_guard; /* claim ch_info (will be freed along with the channel itself) */ CHD_INFO(chd) = ci; ci = NULL; /* const override */ *(struct __user_channel_schema **)(uintptr_t)&chd->chd_schema = ucs; /* make sure we're running on the right kernel */ _SCHEMA_VER_VERIFY(chd); *(nexus_meta_type_t *)&chd->chd_md_type = CHD_SCHEMA(chd)->csm_md_type; *(nexus_meta_subtype_t *)&chd->chd_md_subtype = CHD_SCHEMA(chd)->csm_md_subtype; if (CHD_SCHEMA(chd)->csm_stats_ofs != 0) { *(void **)(uintptr_t)&chd->chd_nx_stats = _CHANNEL_OFFSET(void *, CHD_INFO(chd)->cinfo_mem_base, CHD_SCHEMA(chd)->csm_stats_ofs); } if (CHD_SCHEMA(chd)->csm_flowadv_ofs != 0) { *(struct __flowadv_entry **)(uintptr_t)&chd->chd_nx_flowadv = _CHANNEL_OFFSET(struct __flowadv_entry *, CHD_INFO(chd)->cinfo_mem_base, CHD_SCHEMA(chd)->csm_flowadv_ofs); } if (CHD_SCHEMA(chd)->csm_nexusadv_ofs != 0) { struct __kern_nexus_adv_metadata *adv_md; *(struct __kern_nexus_adv_metadata **) (uintptr_t)&chd->chd_nx_adv = _CHANNEL_OFFSET(struct __kern_nexus_adv_metadata *, CHD_INFO(chd)->cinfo_mem_base, CHD_SCHEMA(chd)->csm_nexusadv_ofs); adv_md = CHD_NX_ADV_MD(chd); if (adv_md->knam_version != NX_ADVISORY_MD_CURRENT_VERSION && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("nexus advisory metadata version" " mismatch", NX_ADVISORY_MD_CURRENT_VERSION); /* NOTREACHED */ __builtin_unreachable(); } if (chd->chd_nx_adv->knam_type == NEXUS_ADVISORY_TYPE_NETIF) { struct netif_nexus_advisory *netif_adv; netif_adv = CHD_NX_ADV_NETIF(adv_md); if (netif_adv->nna_version != NX_NETIF_ADVISORY_CURRENT_VERSION && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("nexus advisory " "version mismatch for netif", NX_NETIF_ADVISORY_CURRENT_VERSION); /* NOTREACHED */ __builtin_unreachable(); } } else if (chd->chd_nx_adv->knam_type == NEXUS_ADVISORY_TYPE_FLOWSWITCH) { struct sk_nexusadv *fsw_adv; fsw_adv = CHD_NX_ADV_FSW(adv_md); if (fsw_adv->nxadv_ver != NX_FLOWSWITCH_ADVISORY_CURRENT_VERSION && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("nexus advisory " "version mismatch for flowswitch", NX_FLOWSWITCH_ADVISORY_CURRENT_VERSION); /* NOTREACHED */ __builtin_unreachable(); } } else if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("nexus advisory metadata type" " unknown", NX_ADVISORY_MD_CURRENT_VERSION); /* NOTREACHED */ __builtin_unreachable(); } } if (cha != NULL) { os_channel_info2attr(chd, cha); } ring_offset = 0; for (i = 0; i < num_tx_rings; i++) { ring_index = ring_offset + i; os_channel_init_ring(&chd->chd_rings[ring_index], chd, ring_index); } ring_offset += num_tx_rings; for (i = 0; i < num_rx_rings; i++) { ring_index = ring_offset + i; os_channel_init_ring(&chd->chd_rings[ring_index], chd, ring_index); } ring_offset += num_rx_rings; for (i = 0; i < num_allocator_rings; i++) { ring_index = ring_offset + i; os_channel_init_ring(&chd->chd_rings[ring_index], chd, ring_index); } ring_offset += num_allocator_rings; for (i = 0; i < num_event_rings; i++) { ring_index = ring_offset + i; os_channel_init_ring(&chd->chd_rings[ring_index], chd, ring_index); } ring_offset += num_event_rings; for (i = 0; i < num_large_buf_alloc_rings; i++) { ring_index = ring_offset + i; os_channel_init_ring(&chd->chd_rings[ring_index], chd, ring_index); } if (init.ci_ch_mode & CHMODE_USER_PACKET_POOL) { chd->chd_sync_flags = CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE; *__DECONST(uint8_t *, &chd->chd_alloc_ring_idx) = num_tx_rings + num_rx_rings; if (num_allocator_rings > 2) { chd->chd_sync_flags |= CHANNEL_SYNCF_ALLOC_BUF; *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) = chd->chd_alloc_ring_idx + 1; *__DECONST(uint8_t *, &chd->chd_free_ring_idx) = chd->chd_buf_alloc_ring_idx + 1; *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) = chd->chd_free_ring_idx + 1; } else { *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_free_ring_idx) = chd->chd_alloc_ring_idx + 1; } if (num_large_buf_alloc_rings > 0) { *__DECONST(uint8_t *, &chd->chd_large_buf_alloc_ring_idx) = num_tx_rings + num_rx_rings + num_allocator_rings + num_event_rings; } else { *__DECONST(uint8_t *, &chd->chd_large_buf_alloc_ring_idx) = CHD_RING_IDX_NONE; } } else { *__DECONST(uint8_t *, &chd->chd_alloc_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_free_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_buf_alloc_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_buf_free_ring_idx) = CHD_RING_IDX_NONE; *__DECONST(uint8_t *, &chd->chd_large_buf_alloc_ring_idx) = CHD_RING_IDX_NONE; } if (__os_ch_md_redzone_cookie == 0) { __os_ch_md_redzone_cookie = CHD_SCHEMA(chd)->csm_md_redzone_cookie; } /* ensure all stores are globally visible */ os_atomic_thread_fence(seq_cst); done: if (err != 0) { if (fd != -1) { (void) guarded_close_np(fd, &init.ci_guard); } if (chd != NULL) { if (CHD_INFO(chd) != NULL) { free(CHD_INFO(chd)); CHD_INFO(chd) = NULL; } free(chd); chd = NULL; } if (ci != NULL) { free(ci); ci = NULL; } errno = err; } return chd; } channel_t os_channel_create(const uuid_t uuid, const nexus_port_t port) { return os_channel_create_extended(uuid, port, CHANNEL_DIR_TX_RX, CHANNEL_RING_ID_ANY, NULL); } int os_channel_get_fd(const channel_t chd) { return chd->chd_fd; } int os_channel_read_attr(const channel_t chd, channel_attr_t cha) { int err; if ((err = __channel_get_info(chd->chd_fd, CHD_INFO(chd), CHD_INFO_SIZE)) == 0) { os_channel_info2attr(chd, cha); } return err; } int os_channel_write_attr(const channel_t chd, channel_attr_t cha) { int err = 0; if (CHD_INFO(chd)->cinfo_tx_lowat.cet_unit != cha->cha_tx_lowat.cet_unit || CHD_INFO(chd)->cinfo_tx_lowat.cet_value != cha->cha_tx_lowat.cet_value) { if ((err = __channel_set_opt(chd->chd_fd, CHOPT_TX_LOWAT_THRESH, &cha->cha_tx_lowat, sizeof(cha->cha_tx_lowat))) != 0) { goto done; } /* update local copy */ CHD_INFO(chd)->cinfo_tx_lowat = cha->cha_tx_lowat; } if (CHD_INFO(chd)->cinfo_rx_lowat.cet_unit != cha->cha_rx_lowat.cet_unit || CHD_INFO(chd)->cinfo_rx_lowat.cet_value != cha->cha_rx_lowat.cet_value) { if ((err = __channel_set_opt(chd->chd_fd, CHOPT_RX_LOWAT_THRESH, &cha->cha_rx_lowat, sizeof(cha->cha_rx_lowat))) != 0) { goto done; } /* update local copy */ CHD_INFO(chd)->cinfo_rx_lowat = cha->cha_rx_lowat; } done: return err; } int os_channel_read_nexus_extension_info(const channel_t chd, nexus_type_t *nt, uint64_t *ext) { struct nxprov_params *nxp; nxp = &CHD_INFO(chd)->cinfo_nxprov_params; if (nt != NULL) { *nt = nxp->nxp_type; } if (ext != NULL) { *ext = (uint64_t)nxp->nxp_extensions; } return 0; } int os_channel_sync(const channel_t chd, const sync_mode_t mode) { if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX)) { return EINVAL; } return __channel_sync(chd->chd_fd, mode, (mode == CHANNEL_SYNC_TX) ? chd->chd_sync_flags : (chd->chd_sync_flags & ~(CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_ALLOC_BUF))); } void os_channel_destroy(channel_t chd) { if (chd->chd_fd != -1) { (void) guarded_close_np(chd->chd_fd, &chd->chd_guard); } if (CHD_INFO(chd) != NULL) { free(CHD_INFO(chd)); CHD_INFO(chd) = NULL; } free(chd); } int os_channel_is_defunct(channel_t chd) { return _CHANNEL_IS_DEFUNCT(chd); } __attribute__((always_inline, visibility("hidden"))) static inline ring_id_t _ring_id(struct ch_info *cinfo, const ring_id_type_t type) { ring_id_t rid = CHANNEL_RING_ID_ANY; /* make it crash */ switch (type) { case CHANNEL_FIRST_TX_RING: rid = cinfo->cinfo_first_tx_ring; break; case CHANNEL_LAST_TX_RING: rid = cinfo->cinfo_last_tx_ring; break; case CHANNEL_FIRST_RX_RING: rid = cinfo->cinfo_first_rx_ring; break; case CHANNEL_LAST_RX_RING: rid = cinfo->cinfo_last_rx_ring; break; } return rid; } ring_id_t os_channel_ring_id(const channel_t chd, const ring_id_type_t type) { return _ring_id(CHD_INFO(chd), type); } channel_ring_t os_channel_tx_ring(const channel_t chd, const ring_id_t rid) { struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_ring_id != CHANNEL_RING_ID_ANY && ci->cinfo_ch_ring_id != rid) || rid < _ring_id(ci, CHANNEL_FIRST_TX_RING) || rid > _ring_id(ci, CHANNEL_LAST_TX_RING))) { return NULL; } return &chd->chd_rings[rid - _ring_id(ci, CHANNEL_FIRST_TX_RING)]; } channel_ring_t os_channel_rx_ring(const channel_t chd, const ring_id_t rid) { struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_ring_id != CHANNEL_RING_ID_ANY && ci->cinfo_ch_ring_id != rid) || rid < _ring_id(ci, CHANNEL_FIRST_RX_RING) || rid > _ring_id(ci, CHANNEL_LAST_RX_RING))) { return NULL; } return &chd->chd_rings[_num_tx_rings(ci) + /* add tx rings */ (rid - _ring_id(ci, CHANNEL_FIRST_RX_RING))]; } /* * Return 1 if we have pending transmissions in the tx ring. When everything * is complete ring->ring_head == ring->ring_khead. */ int os_channel_pending(const channel_ring_t chrd) { struct __user_channel_ring *ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); return ring->ring_head != ring->ring_khead; } uint64_t os_channel_ring_sync_time(const channel_ring_t chrd) { return chrd->chrd_ring->ring_sync_time; } uint64_t os_channel_ring_notify_time(const channel_ring_t chrd) { return chrd->chrd_ring->ring_notify_time; } uint32_t os_channel_available_slot_count(const channel_ring_t chrd) { const struct __user_channel_ring *ring = chrd->chrd_ring; uint32_t count; int n; if (ring->ring_kind == CR_KIND_TX) { n = ring->ring_head - ring->ring_khead; if (n < 0) { n += ring->ring_num_slots; } count = (ring->ring_num_slots - n - 1); } else { n = ring->ring_tail - ring->ring_head; if (n < 0) { n += ring->ring_num_slots; } count = n; } return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? 0 : count; } int os_channel_advance_slot(channel_ring_t chrd, const channel_slot_t slot) { struct __user_channel_ring *ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); slot_idx_t idx; int err; idx = _SLOT_INDEX(chrd, slot); if (__probable(_slot_index_is_valid(ring, idx))) { ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); err = 0; } else { err = (_CHANNEL_RING_IS_DEFUNCT(chrd) ? ENXIO : EINVAL); } return err; } channel_slot_t os_channel_get_next_slot(const channel_ring_t chrd, const channel_slot_t slot0, slot_prop_t *prop) { const struct __user_channel_ring *ring = chrd->chrd_ring; const struct __slot_desc *slot; slot_idx_t idx; if (__probable(slot0 != NULL)) { idx = _SLOT_INDEX(chrd, slot0); if (__probable(_slot_index_is_valid(ring, idx))) { idx = _CHANNEL_RING_NEXT(ring, idx); } else if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { /* slot is out of bounds */ SK_ABORT_WITH_CAUSE("Index out of bounds in gns", idx); /* NOTREACHED */ __builtin_unreachable(); } else { /* * In case of a defunct, pretend as if we've * advanced to the last slot; this will result * in a NULL slot below. */ idx = ring->ring_tail; } } else { idx = ring->ring_head; } if (__probable(idx != ring->ring_tail)) { slot = &chrd->chrd_slot_desc[idx]; } else { /* we just advanced to the last slot */ slot = NULL; } if (__probable(slot != NULL)) { uint16_t ring_kind = ring->ring_kind; struct __user_quantum *q; mach_vm_address_t baddr; uint16_t bdoff; if (__improbable((ring_kind == CR_KIND_TX) && (CHD_INFO(chrd->chrd_channel)->cinfo_ch_mode & CHMODE_USER_PACKET_POOL))) { if (SD_VALID_METADATA(SLOT_DESC_USD(slot))) { SK_ABORT_WITH_CAUSE("Tx slot has attached " "metadata", idx); /* NOTREACHED */ __builtin_unreachable(); } if (prop != NULL) { prop->sp_len = 0; prop->sp_flags = 0; prop->sp_buf_ptr = 0; prop->sp_mdata_ptr = 0; } return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? NULL : (channel_slot_t)slot; } _SLOT_DESC_VERIFY(chrd, SLOT_DESC_USD(slot)); q = _SLOT_METADATA(chrd, ring, idx); _METADATA_VERIFY(chrd, q); baddr = _initialize_metadata_address(chrd, q, &bdoff); if (__improbable(baddr == 0)) { return NULL; } /* No multi-buflet support for slot based interface */ if (__probable(prop != NULL)) { /* immutable: slot index */ prop->sp_idx = idx; prop->sp_flags = 0; prop->sp_buf_ptr = baddr + bdoff; prop->sp_mdata_ptr = q; /* reset slot length if this is to be used for tx */ prop->sp_len = (ring_kind == CR_KIND_TX) ? ring->ring_def_buf_size : q->qum_len; } } return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? NULL : (channel_slot_t)slot; } void os_channel_set_slot_properties(const channel_ring_t chrd, const channel_slot_t slot, const slot_prop_t *prop) { const struct __user_channel_ring *ring = chrd->chrd_ring; slot_idx_t idx = _SLOT_INDEX(chrd, slot); if (__probable(_slot_index_is_valid(ring, idx))) { struct __user_quantum *q; _METADATA_VERIFY(chrd, prop->sp_mdata_ptr); _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); /* * In the event of a defunct, we'd be accessing zero-filled * memory; this is fine we ignore all changes made to the * region at that time. */ q = _SLOT_METADATA(chrd, ring, idx); q->qum_len = prop->sp_len; switch (chrd->chrd_md_type) { case NEXUS_META_TYPE_PACKET: { struct __user_packet *p = (struct __user_packet *)q; /* No multi-buflet support for slot based interface */ p->pkt_qum_buf.buf_dlen = prop->sp_len; p->pkt_qum_buf.buf_doff = 0; break; } default: q->qum_buf[0].buf_dlen = prop->sp_len; q->qum_buf[0].buf_doff = 0; break; } } else if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { /* slot is out of bounds */ SK_ABORT_WITH_CAUSE("Index out of bounds in ssp", idx); /* NOTREACHED */ __builtin_unreachable(); } } packet_t os_channel_slot_get_packet(const channel_ring_t chrd, const channel_slot_t slot) { const struct __user_channel_ring *ring = chrd->chrd_ring; struct __user_quantum *q = NULL; if (__probable(slot != NULL)) { slot_idx_t idx = _SLOT_INDEX(chrd, slot); if (__improbable(!_slot_index_is_valid(ring, idx)) && !_CHANNEL_RING_IS_DEFUNCT(chrd)) { /* slot is out of bounds */ SK_ABORT_WITH_CAUSE("Index out of bounds in sgp", idx); /* NOTREACHED */ __builtin_unreachable(); } if (__probable(SD_VALID_METADATA(_SLOT_DESC(chrd, idx)))) { obj_idx_t midx; q = _SLOT_METADATA(chrd, ring, idx); _METADATA_VERIFY(chrd, q); /* * In the event of a defunct, we'd be accessing * zero-filed memory and end up with 0 for midx; * this is fine since we ignore all changes made * to the region at that time. */ midx = METADATA_IDX(q); _SLOT_METADATA_IDX_VERIFY(chrd, q, midx); } } return (q == NULL) ? 0 : SK_PTR_ENCODE(q, chrd->chrd_md_type, chrd->chrd_md_subtype); } void * os_channel_get_stats_region(const channel_t chd, const channel_stats_id_t id) { void *sp = CHD_NX_STATS(chd); struct __nx_stats_fsw *nxs_fsw; void *ptr = NULL; /* we currently deal only with flowswitch */ if (sp == NULL || CHD_SCHEMA(chd)->csm_stats_type != NEXUS_STATS_TYPE_FSW) { return NULL; } nxs_fsw = sp; switch (id) { case CHANNEL_STATS_ID_IP: ptr = &nxs_fsw->nxs_ipstat; break; case CHANNEL_STATS_ID_IP6: ptr = &nxs_fsw->nxs_ip6stat; break; case CHANNEL_STATS_ID_TCP: ptr = &nxs_fsw->nxs_tcpstat; break; case CHANNEL_STATS_ID_UDP: ptr = &nxs_fsw->nxs_udpstat; break; case CHANNEL_STATS_ID_QUIC: ptr = &nxs_fsw->nxs_quicstat; break; default: ptr = NULL; break; } return ptr; } void * os_channel_get_advisory_region(const channel_t chd) { struct __kern_nexus_adv_metadata *adv_md; /* * To be backward compatible this API will only return * the advisory region for flowswitch. */ adv_md = CHD_NX_ADV_MD(chd); if (adv_md == NULL || adv_md->knam_type != NEXUS_ADVISORY_TYPE_FLOWSWITCH) { return NULL; } return CHD_NX_ADV_FSW(adv_md); } __attribute__((always_inline, visibility("hidden"))) static inline int _flowadv_id_equal(struct __flowadv_entry *fe, uuid_t id) { /* * Anticipate a nicely (8-bytes) aligned UUID from * caller; the one in fae_id is always 8-byte aligned. */ if (__probable(IS_P2ALIGNED(id, sizeof(uint64_t)))) { uint64_t *id_64 = (uint64_t *)(uintptr_t)id; return fe->fae_id_64[0] == id_64[0] && fe->fae_id_64[1] == id_64[1]; } else if (__probable(IS_P2ALIGNED(id, sizeof(uint32_t)))) { uint32_t *id_32 = (uint32_t *)(uintptr_t)id; return fe->fae_id_32[0] == id_32[0] && fe->fae_id_32[1] == id_32[1] && fe->fae_id_32[2] == id_32[2] && fe->fae_id_32[3] == id_32[3]; } return UUID_COMPARE(fe->fae_id, id); } int os_channel_flow_admissible(const channel_ring_t chrd, uuid_t flow_id, const flowadv_idx_t flow_index) { const struct __user_channel_ring *ring = chrd->chrd_ring; const struct channel *chd = chrd->chrd_channel; struct __flowadv_entry *fe = CHD_NX_FLOWADV(chd); /* * Currently, flow advisory is on a per-nexus port basis. * To anticipate for future requirements, we use the ring * as parameter instead, even though we use it only to * check if this is a TX ring for now. */ if (__improbable(CHD_NX_FLOWADV(chd) == NULL)) { return ENXIO; } else if (__improbable(ring->ring_kind != CR_KIND_TX || flow_index >= CHD_PARAMS(chd)->nxp_flowadv_max)) { return EINVAL; } /* * Rather than checking if the UUID is all zeroes, check * against fae_flags since the presence of FLOWADV_VALID * means fae_id is non-zero. This avoids another round of * comparison against zeroes. */ fe = &CHD_NX_FLOWADV(chd)[flow_index]; if (__improbable(fe->fae_flags == 0 || !_flowadv_id_equal(fe, flow_id))) { return ENOENT; } return __improbable((fe->fae_flags & FLOWADVF_SUSPENDED) != 0) ? ENOBUFS: 0; } int os_channel_flow_adv_get_ce_count(const channel_ring_t chrd, uuid_t flow_id, const flowadv_idx_t flow_index, uint32_t *ce_cnt, uint32_t *pkt_cnt) { const struct __user_channel_ring *ring = chrd->chrd_ring; const struct channel *chd = chrd->chrd_channel; struct __flowadv_entry *fe = CHD_NX_FLOWADV(chd); /* * Currently, flow advisory is on a per-nexus port basis. * To anticipate for future requirements, we use the ring * as parameter instead, even though we use it only to * check if this is a TX ring for now. */ if (__improbable(CHD_NX_FLOWADV(chd) == NULL)) { return ENXIO; } else if (__improbable(ring->ring_kind != CR_KIND_TX || flow_index >= CHD_PARAMS(chd)->nxp_flowadv_max)) { return EINVAL; } /* * Rather than checking if the UUID is all zeroes, check * against fae_flags since the presence of FLOWADV_VALID * means fae_id is non-zero. This avoids another round of * comparison against zeroes. */ fe = &CHD_NX_FLOWADV(chd)[flow_index]; if (__improbable(fe->fae_flags == 0 || !_flowadv_id_equal(fe, flow_id))) { return ENOENT; } *ce_cnt = fe->fae_ce_cnt; *pkt_cnt = fe->fae_pkt_cnt; return 0; } channel_attr_t os_channel_attr_create(void) { struct channel_attr *cha; cha = malloc(sizeof(*cha)); if (cha != NULL) { bzero(cha, sizeof(*cha)); } return cha; } channel_attr_t os_channel_attr_clone(const channel_attr_t cha) { struct channel_attr *ncha; ncha = os_channel_attr_create(); if (ncha != NULL && cha != NULL) { bcopy(cha, ncha, sizeof(*ncha)); ncha->cha_key = NULL; ncha->cha_key_len = 0; if (cha->cha_key != NULL && cha->cha_key_len != 0 && os_channel_attr_set_key(ncha, cha->cha_key, cha->cha_key_len) != 0) { os_channel_attr_destroy(ncha); ncha = NULL; } } return ncha; } int os_channel_attr_set(const channel_attr_t cha, const channel_attr_type_t type, const uint64_t value) { int err = 0; switch (type) { case CHANNEL_ATTR_TX_RINGS: case CHANNEL_ATTR_RX_RINGS: case CHANNEL_ATTR_TX_SLOTS: case CHANNEL_ATTR_RX_SLOTS: case CHANNEL_ATTR_SLOT_BUF_SIZE: case CHANNEL_ATTR_SLOT_META_SIZE: case CHANNEL_ATTR_NEXUS_EXTENSIONS: case CHANNEL_ATTR_NEXUS_MHINTS: case CHANNEL_ATTR_NEXUS_IFINDEX: case CHANNEL_ATTR_NEXUS_STATS_SIZE: case CHANNEL_ATTR_NEXUS_FLOWADV_MAX: case CHANNEL_ATTR_NEXUS_META_TYPE: case CHANNEL_ATTR_NEXUS_META_SUBTYPE: case CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD: case CHANNEL_ATTR_NEXUS_ADV_SIZE: case CHANNEL_ATTR_MAX_FRAGS: case CHANNEL_ATTR_NUM_BUFFERS: case CHANNEL_ATTR_LARGE_BUF_SIZE: err = ENOTSUP; break; case CHANNEL_ATTR_EXCLUSIVE: cha->cha_exclusive = (uint32_t)value; break; case CHANNEL_ATTR_NO_AUTO_SYNC: if (value == 0) { err = ENOTSUP; } break; case CHANNEL_ATTR_MONITOR: switch (value) { case CHANNEL_MONITOR_OFF: case CHANNEL_MONITOR_NO_COPY: case CHANNEL_MONITOR_COPY: cha->cha_monitor = (uint32_t)value; goto done; } err = EINVAL; break; case CHANNEL_ATTR_TX_LOWAT_UNIT: case CHANNEL_ATTR_RX_LOWAT_UNIT: switch (value) { case CHANNEL_THRESHOLD_UNIT_BYTES: case CHANNEL_THRESHOLD_UNIT_SLOTS: if (type == CHANNEL_ATTR_TX_LOWAT_UNIT) { cha->cha_tx_lowat.cet_unit = (channel_threshold_unit_t)value; } else { cha->cha_rx_lowat.cet_unit = (channel_threshold_unit_t)value; } goto done; } err = EINVAL; break; case CHANNEL_ATTR_TX_LOWAT_VALUE: cha->cha_tx_lowat.cet_value = (uint32_t)value; break; case CHANNEL_ATTR_RX_LOWAT_VALUE: cha->cha_rx_lowat.cet_value = (uint32_t)value; break; case CHANNEL_ATTR_USER_PACKET_POOL: cha->cha_user_packet_pool = (value != 0); break; case CHANNEL_ATTR_NEXUS_DEFUNCT_OK: cha->cha_nexus_defunct_ok = (value != 0); break; case CHANNEL_ATTR_FILTER: cha->cha_filter = (uint32_t)value; break; case CHANNEL_ATTR_EVENT_RING: cha->cha_enable_event_ring = (value != 0); break; case CHANNEL_ATTR_LOW_LATENCY: cha->cha_low_latency = (value != 0); break; default: err = EINVAL; break; } done: return err; } int os_channel_attr_set_key(const channel_attr_t cha, const void *key, const uint32_t key_len) { int err = 0; if ((key == NULL && key_len != 0) || (key != NULL && key_len == 0) || (key_len != 0 && key_len > NEXUS_MAX_KEY_LEN)) { err = EINVAL; goto done; } cha->cha_key_len = 0; if (key_len == 0 && cha->cha_key != NULL) { free(cha->cha_key); cha->cha_key = NULL; } else if (key != NULL && key_len != 0) { if (cha->cha_key != NULL) { free(cha->cha_key); } if ((cha->cha_key = malloc(key_len)) == NULL) { err = ENOMEM; goto done; } cha->cha_key_len = key_len; bcopy(key, cha->cha_key, key_len); } done: return err; } int os_channel_attr_get(const channel_attr_t cha, const channel_attr_type_t type, uint64_t *value) { int err = 0; switch (type) { case CHANNEL_ATTR_TX_RINGS: *value = cha->cha_tx_rings; break; case CHANNEL_ATTR_RX_RINGS: *value = cha->cha_rx_rings; break; case CHANNEL_ATTR_TX_SLOTS: *value = cha->cha_tx_slots; break; case CHANNEL_ATTR_RX_SLOTS: *value = cha->cha_rx_slots; break; case CHANNEL_ATTR_SLOT_BUF_SIZE: *value = cha->cha_buf_size; break; case CHANNEL_ATTR_SLOT_META_SIZE: *value = cha->cha_meta_size; break; case CHANNEL_ATTR_NEXUS_STATS_SIZE: *value = cha->cha_stats_size; break; case CHANNEL_ATTR_NEXUS_FLOWADV_MAX: *value = cha->cha_flowadv_max; break; case CHANNEL_ATTR_EXCLUSIVE: *value = cha->cha_exclusive; break; case CHANNEL_ATTR_NO_AUTO_SYNC: *value = 1; break; case CHANNEL_ATTR_MONITOR: *value = cha->cha_monitor; break; case CHANNEL_ATTR_TX_LOWAT_UNIT: *value = cha->cha_tx_lowat.cet_unit; break; case CHANNEL_ATTR_TX_LOWAT_VALUE: *value = cha->cha_tx_lowat.cet_value; break; case CHANNEL_ATTR_RX_LOWAT_UNIT: *value = cha->cha_rx_lowat.cet_unit; break; case CHANNEL_ATTR_RX_LOWAT_VALUE: *value = cha->cha_rx_lowat.cet_value; break; case CHANNEL_ATTR_NEXUS_TYPE: *value = cha->cha_nexus_type; break; case CHANNEL_ATTR_NEXUS_EXTENSIONS: *value = cha->cha_nexus_extensions; break; case CHANNEL_ATTR_NEXUS_MHINTS: *value = cha->cha_nexus_mhints; break; case CHANNEL_ATTR_NEXUS_IFINDEX: *value = cha->cha_nexus_ifindex; break; case CHANNEL_ATTR_NEXUS_META_TYPE: *value = cha->cha_nexus_meta_type; break; case CHANNEL_ATTR_NEXUS_META_SUBTYPE: *value = cha->cha_nexus_meta_subtype; break; case CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD: *value = cha->cha_nexus_checksum_offload; break; case CHANNEL_ATTR_USER_PACKET_POOL: *value = (cha->cha_user_packet_pool != 0); break; case CHANNEL_ATTR_NEXUS_ADV_SIZE: *value = cha->cha_nexusadv_size; break; case CHANNEL_ATTR_NEXUS_DEFUNCT_OK: *value = cha->cha_nexus_defunct_ok; break; case CHANNEL_ATTR_EVENT_RING: *value = (cha->cha_enable_event_ring != 0); break; case CHANNEL_ATTR_MAX_FRAGS: *value = cha->cha_max_frags; break; case CHANNEL_ATTR_NUM_BUFFERS: *value = cha->cha_num_buffers; break; case CHANNEL_ATTR_LOW_LATENCY: *value = (cha->cha_low_latency != 0); break; case CHANNEL_ATTR_LARGE_BUF_SIZE: *value = cha->cha_large_buf_size; break; default: err = EINVAL; break; } return err; } int os_channel_attr_get_key(const channel_attr_t cha, void *key, uint32_t *key_len) { int err = 0; if (key_len == NULL) { err = EINVAL; goto done; } else if (key == NULL || cha->cha_key == NULL) { *key_len = (cha->cha_key != NULL) ? cha->cha_key_len : 0; goto done; } if (*key_len >= cha->cha_key_len) { bcopy(cha->cha_key, key, cha->cha_key_len); *key_len = cha->cha_key_len; } else { err = ENOMEM; } done: return err; } __attribute__((visibility("hidden"))) static void os_channel_info2attr(struct channel *chd, channel_attr_t cha) { struct ch_info *cinfo = CHD_INFO(chd); /* Save these first before we wipe out the attribute */ uint32_t cha_key_len = cha->cha_key_len; void *cha_key = cha->cha_key; uint32_t caps; _CASSERT(NEXUS_META_TYPE_INVALID == CHANNEL_NEXUS_META_TYPE_INVALID); _CASSERT(NEXUS_META_TYPE_QUANTUM == CHANNEL_NEXUS_META_TYPE_QUANTUM); _CASSERT(NEXUS_META_TYPE_PACKET == CHANNEL_NEXUS_META_TYPE_PACKET); _CASSERT(NEXUS_META_SUBTYPE_INVALID == CHANNEL_NEXUS_META_SUBTYPE_INVALID); _CASSERT(NEXUS_META_SUBTYPE_PAYLOAD == CHANNEL_NEXUS_META_SUBTYPE_PAYLOAD); _CASSERT(NEXUS_META_SUBTYPE_RAW == CHANNEL_NEXUS_META_SUBTYPE_RAW); bzero(cha, sizeof(*cha)); cha->cha_tx_rings = CHD_PARAMS(chd)->nxp_tx_rings; cha->cha_rx_rings = CHD_PARAMS(chd)->nxp_rx_rings; cha->cha_tx_slots = CHD_PARAMS(chd)->nxp_tx_slots; cha->cha_rx_slots = CHD_PARAMS(chd)->nxp_rx_slots; cha->cha_buf_size = CHD_PARAMS(chd)->nxp_buf_size; cha->cha_meta_size = CHD_PARAMS(chd)->nxp_meta_size; cha->cha_stats_size = CHD_PARAMS(chd)->nxp_stats_size; cha->cha_flowadv_max = CHD_PARAMS(chd)->nxp_flowadv_max; cha->cha_exclusive = !!(cinfo->cinfo_ch_mode & CHMODE_EXCLUSIVE); cha->cha_user_packet_pool = !!(cinfo->cinfo_ch_mode & CHMODE_USER_PACKET_POOL); cha->cha_nexus_defunct_ok = !!(cinfo->cinfo_ch_mode & CHMODE_DEFUNCT_OK); cha->cha_nexusadv_size = CHD_PARAMS(chd)->nxp_nexusadv_size; if (cinfo->cinfo_ch_mode & CHMODE_MONITOR) { cha->cha_monitor = (cinfo->cinfo_ch_mode & CHMODE_MONITOR_NO_COPY) ? CHANNEL_MONITOR_NO_COPY : CHANNEL_MONITOR_COPY; } else { cha->cha_monitor = CHANNEL_MONITOR_OFF; } cha->cha_key_len = cha_key_len; cha->cha_key = cha_key; cha->cha_tx_lowat = cinfo->cinfo_tx_lowat; cha->cha_rx_lowat = cinfo->cinfo_rx_lowat; cha->cha_nexus_type = CHD_PARAMS(chd)->nxp_type; cha->cha_nexus_extensions = CHD_PARAMS(chd)->nxp_extensions; cha->cha_nexus_mhints = CHD_PARAMS(chd)->nxp_mhints; cha->cha_nexus_ifindex = CHD_PARAMS(chd)->nxp_ifindex; cha->cha_nexus_meta_type = chd->chd_md_type; cha->cha_nexus_meta_subtype = chd->chd_md_subtype; cha->cha_enable_event_ring = (cinfo->cinfo_ch_mode & CHMODE_EVENT_RING) != 0; cha->cha_low_latency = (cinfo->cinfo_ch_mode & CHMODE_LOW_LATENCY) != 0; caps = CHD_PARAMS(chd)->nxp_capabilities; if (caps & NXPCAP_CHECKSUM_PARTIAL) { cha->cha_nexus_checksum_offload = CHANNEL_NEXUS_CHECKSUM_PARTIAL; } else { cha->cha_nexus_checksum_offload = 0; } cha->cha_max_frags = CHD_PARAMS(chd)->nxp_max_frags; cha->cha_num_buffers = cinfo->cinfo_num_bufs; cha->cha_large_buf_size = CHD_PARAMS(chd)->nxp_large_buf_size; } void os_channel_attr_destroy(channel_attr_t cha) { if (cha->cha_key != NULL) { free(cha->cha_key); cha->cha_key = NULL; } free(cha); } static int os_channel_packet_alloc_common(const channel_t chd, packet_t *ph, bool large) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; struct __user_quantum *q; slot_idx_t idx; mach_vm_address_t baddr; uint16_t bdoff; struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } if (__improbable(large && chd->chd_large_buf_alloc_ring_idx == CHD_RING_IDX_NONE)) { return ENOTSUP; } chrd = &chd->chd_rings[large ? chd->chd_large_buf_alloc_ring_idx : chd->chd_alloc_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; if (__improbable(idx == ring->ring_tail)) { /* * do a sync to get more packets; * since we are paying the cost of a syscall do a sync for * free ring as well. */ int err; sync_flags_t flags; if (large) { flags = (chd->chd_sync_flags & ~(CHANNEL_SYNCF_ALLOC_BUF | CHANNEL_SYNCF_ALLOC)) | CHANNEL_SYNCF_LARGE_ALLOC; } else { flags = chd->chd_sync_flags & ~CHANNEL_SYNCF_ALLOC_BUF; } err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, flags); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("packet pool alloc " "sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } } if (__improbable(idx == ring->ring_tail)) { return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : ENOMEM; } _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); q = _SLOT_METADATA(chrd, ring, idx); _METADATA_VERIFY(chrd, q); *ph = SK_PTR_ENCODE(q, chrd->chrd_md_type, chrd->chrd_md_subtype); _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx)); /* * Initialize the metadata buffer address. In the event of a * defunct, we'd be accessing zero-filled memory; this is fine * since we ignore all changes made to region at that time. */ baddr = _initialize_metadata_address(chrd, q, &bdoff); if (__improbable(baddr == 0)) { return ENXIO; } ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0; } int os_channel_packet_alloc(const channel_t chd, packet_t *ph) { return os_channel_packet_alloc_common(chd, ph, false); } int os_channel_large_packet_alloc(const channel_t chd, packet_t *ph) { return os_channel_packet_alloc_common(chd, ph, true); } int os_channel_packet_free(const channel_t chd, packet_t ph) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; slot_idx_t idx; obj_idx_t midx; struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } chrd = &chd->chd_rings[chd->chd_free_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; if (__improbable(idx == ring->ring_tail)) { /* * do a sync to reclaim space in free ring; */ int err; err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, CHANNEL_SYNCF_FREE); if (__improbable(err != 0) && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("packet pool free " "sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } } if (__improbable(idx == ring->ring_tail) && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT("no free ring space"); /* NOTREACHED */ __builtin_unreachable(); } /* * In the event of a defunct, midx will be 0 and we'll end up * attaching it to the slot; this is fine since we ignore all * changes made to the slot descriptors at that time. */ midx = METADATA_IDX(QUM_ADDR(ph)); _SLOT_METADATA_IDX_VERIFY(chrd, QUM_ADDR(ph), midx); _SLOT_ATTACH_METADATA(_SLOT_DESC(chrd, idx), midx); ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0; } int os_channel_slot_attach_packet(const channel_ring_t chrd, const channel_slot_t slot, packet_t ph) { slot_idx_t idx; obj_idx_t midx; if (__improbable((chrd->chrd_channel->chd_info->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } if (__improbable(!__packet_is_finalized(ph))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("packet not finalized"); /* NOTREACHED */ __builtin_unreachable(); } goto done; } idx = _SLOT_INDEX(chrd, slot); if (__improbable(!_slot_index_is_valid(chrd->chrd_ring, idx))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT_WITH_CAUSE("Invalid slot", slot); /* NOTREACHED */ __builtin_unreachable(); } goto done; } if (__improbable(SD_VALID_METADATA(SLOT_DESC_USD(slot)))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT_WITH_CAUSE("Slot has attached packet", slot); /* NOTREACHED */ __builtin_unreachable(); } goto done; } /* * In the event of a defunct, midx will be 0 and we'll end up * attaching it to the slot; this is fine since we ignore all * changes made to the slot descriptors at that time. */ midx = METADATA_IDX(QUM_ADDR(ph)); _SLOT_METADATA_IDX_VERIFY(chrd, QUM_ADDR(ph), midx); _SLOT_ATTACH_METADATA(SLOT_DESC_USD(slot), midx); done: return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0; } int os_channel_slot_detach_packet(const channel_ring_t chrd, const channel_slot_t slot, packet_t ph) { slot_idx_t idx; if (__improbable((chrd->chrd_channel->chd_info->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } idx = _SLOT_INDEX(chrd, slot); if (__improbable(!_slot_index_is_valid(chrd->chrd_ring, idx))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT_WITH_CAUSE("Invalid slot", slot); /* NOTREACHED */ __builtin_unreachable(); } goto done; } if (__improbable(!SD_VALID_METADATA(SLOT_DESC_USD(slot)))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT_WITH_CAUSE("Slot has no attached packet", slot); /* NOTREACHED */ __builtin_unreachable(); } goto done; } if (__improbable(ph != SK_PTR_ENCODE(_SLOT_METADATA(chrd, chrd->chrd_ring, idx), chrd->chrd_md_type, chrd->chrd_md_subtype))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("packet handle mismatch"); /* NOTREACHED */ __builtin_unreachable(); } goto done; } if (__improbable(!__packet_is_finalized(ph))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("packet not finalized"); /* NOTREACHED */ __builtin_unreachable(); } goto done; } /* * In the event of a defunct, we ignore any changes made to * the slot descriptors, and so doing this is harmless. */ _SLOT_DETACH_METADATA(SLOT_DESC_USD(slot)); done: return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0; } __attribute__((visibility("hidden"))) static inline int os_channel_purge_packet_alloc_ring_common(const channel_t chd, bool large) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; uint32_t curr_ws; slot_idx_t idx; packet_t ph; int npkts, err; chrd = &chd->chd_rings[large ? chd->chd_large_buf_alloc_ring_idx : chd->chd_alloc_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; /* calculate the number of packets in alloc pool */ npkts = ring->ring_tail - idx; if (npkts < 0) { npkts += ring->ring_num_slots; } curr_ws = ring->ring_alloc_ws; while ((uint32_t)npkts-- > curr_ws) { struct __user_quantum *q; _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); q = _SLOT_METADATA(chrd, ring, idx); _METADATA_VERIFY(chrd, q); ph = SK_PTR_ENCODE(q, chrd->chrd_md_type, chrd->chrd_md_subtype); _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx)); /* * Initialize the metadata buffer address. In the event of a * defunct, we'd be accessing zero-filled memory; this is fine * since we ignore all changes made to region at that time. */ if (chrd->chrd_md_type == NEXUS_META_TYPE_PACKET) { struct __user_packet *p = (struct __user_packet *)q; uint16_t bcnt = p->pkt_bufs_cnt; uint16_t bmax = p->pkt_bufs_max; if (__improbable((bcnt == 0) || (bmax == 0))) { if (!_CHANNEL_RING_IS_DEFUNCT(chrd)) { SK_ABORT("pkt pool purge, bad bufcnt"); /* NOTREACHED */ __builtin_unreachable(); } else { return ENXIO; } } /* * alloc ring will not have multi-buflet packets. */ _PKT_BUFCNT_VERIFY(chrd, bcnt, 1); } *(mach_vm_address_t *) (uintptr_t)&q->qum_buf[0].buf_addr = _CHANNEL_RING_BUF(chrd, ring, &q->qum_buf[0]); idx = _CHANNEL_RING_NEXT(ring, idx); ring->ring_head = idx; err = os_channel_packet_free(chd, ph); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("packet pool purge " "free failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } } return 0; } __attribute__((visibility("hidden"))) static inline int os_channel_purge_packet_alloc_ring(const channel_t chd) { return os_channel_purge_packet_alloc_ring_common(chd, false); } __attribute__((visibility("hidden"))) static inline int os_channel_purge_large_packet_alloc_ring(const channel_t chd) { return os_channel_purge_packet_alloc_ring_common(chd, true); } __attribute__((visibility("hidden"))) static inline int os_channel_purge_buflet_alloc_ring(const channel_t chd) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; uint32_t curr_ws; slot_idx_t idx; int nbfts, err; chrd = &chd->chd_rings[chd->chd_buf_alloc_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; /* calculate the number of packets in alloc pool */ nbfts = ring->ring_tail - idx; if (nbfts < 0) { nbfts += ring->ring_num_slots; } curr_ws = ring->ring_alloc_ws; while ((uint32_t)nbfts-- > curr_ws) { struct __user_buflet *ubft; obj_idx_t nbft_idx; _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); ubft = _SLOT_BFT_METADATA(chrd, ring, idx); _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx)); /* * Initialize the buflet metadata buffer address. */ *(mach_vm_address_t *)(uintptr_t)&(ubft->buf_addr) = _CHANNEL_RING_BUF(chrd, ring, ubft); if (__improbable(ubft->buf_addr == 0)) { SK_ABORT_WITH_CAUSE("buflet with NULL buffer", ubft->buf_idx); /* NOTREACHED */ __builtin_unreachable(); } nbft_idx = ubft->buf_nbft_idx; if (__improbable(nbft_idx != OBJ_IDX_NONE)) { if (_CHANNEL_IS_DEFUNCT(chd)) { return ENXIO; } else { SK_ABORT_WITH_CAUSE("buflet with invalid nidx", nbft_idx); /* NOTREACHED */ __builtin_unreachable(); } } idx = _CHANNEL_RING_NEXT(ring, idx); ring->ring_head = idx; err = os_channel_buflet_free(chd, ubft); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("buflet pool purge " "free failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } } return 0; } int os_channel_packet_pool_purge(const channel_t chd) { struct ch_info *ci = CHD_INFO(chd); int err; if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, ((chd->chd_sync_flags & ~CHANNEL_SYNCF_FREE) | CHANNEL_SYNCF_PURGE)); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("packet pool purge sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } err = os_channel_purge_packet_alloc_ring(chd); if (__improbable(err != 0)) { return err; } if (chd->chd_large_buf_alloc_ring_idx != CHD_RING_IDX_NONE) { err = os_channel_purge_large_packet_alloc_ring(chd); if (__improbable(err != 0)) { return err; } } if (_num_allocator_rings(CHD_SCHEMA(chd)) > 2) { err = os_channel_purge_buflet_alloc_ring(chd); if (__improbable(err != 0)) { return err; } } err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, CHANNEL_SYNCF_FREE); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("packet pool free sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0; } int os_channel_get_next_event_handle(const channel_t chd, os_channel_event_handle_t *ehandle, os_channel_event_type_t *etype, uint32_t *nevents) { struct __kern_channel_event_metadata *emd; struct __user_channel_ring *ring; struct channel_ring_desc *chrd; struct __user_quantum *qum; mach_vm_address_t baddr; uint16_t bdoff; slot_idx_t idx; struct __user_channel_schema *csm = CHD_SCHEMA(chd); struct ch_info *ci = CHD_INFO(chd); if (__improbable((ehandle == NULL) || (etype == NULL) || (nevents == NULL))) { return EINVAL; } if (__improbable((ci->cinfo_ch_mode & CHMODE_EVENT_RING) == 0)) { return ENOTSUP; } *ehandle = NULL; chrd = &chd->chd_rings[_num_tx_rings(ci) + _num_rx_rings(ci) + _num_allocator_rings(csm)]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; if (__improbable(idx == ring->ring_tail)) { return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : ENODATA; } _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); qum = _SLOT_METADATA(chrd, ring, idx); _METADATA_VERIFY(chrd, qum); _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx)); baddr = _initialize_metadata_address(chrd, qum, &bdoff); if (__improbable(baddr == 0)) { return ENXIO; } *ehandle = SK_PTR_ENCODE(qum, chrd->chrd_md_type, chrd->chrd_md_subtype); emd = (void *)(baddr + bdoff); *etype = emd->emd_etype; *nevents = emd->emd_nevents; ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0; } int os_channel_event_free(const channel_t chd, os_channel_event_handle_t ehandle) { return os_channel_packet_free(chd, (packet_t)ehandle); } int os_channel_get_interface_advisory(const channel_t chd, struct ifnet_interface_advisory *advisory) { struct __kern_netif_intf_advisory *intf_adv; struct __kern_nexus_adv_metadata *adv_md; nexus_advisory_type_t adv_type; /* * Interface advisory is only supported for netif and flowswitch. */ adv_md = CHD_NX_ADV_MD(chd); if (adv_md == NULL) { return ENOENT; } adv_type = adv_md->knam_type; if (__improbable(adv_type != NEXUS_ADVISORY_TYPE_NETIF && adv_type != NEXUS_ADVISORY_TYPE_FLOWSWITCH)) { return _CHANNEL_IS_DEFUNCT(chd) ? ENXIO : ENOENT; } if (adv_type == NEXUS_ADVISORY_TYPE_NETIF) { intf_adv = &(CHD_NX_ADV_NETIF(adv_md))->__kern_intf_adv; } else { intf_adv = &(CHD_NX_ADV_FSW(adv_md))->_nxadv_intf_adv; } if (intf_adv->cksum != os_cpu_copy_in_cksum(&intf_adv->adv, advisory, sizeof(*advisory), 0)) { return _CHANNEL_IS_DEFUNCT(chd) ? ENXIO : EAGAIN; } return 0; } int os_channel_configure_interface_advisory(const channel_t chd, boolean_t enable) { uint32_t value = enable; return __channel_set_opt(chd->chd_fd, CHOPT_IF_ADV_CONF, &value, sizeof(value)); } int os_channel_buflet_alloc(const channel_t chd, buflet_t *bft) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; struct __user_buflet *ubft; obj_idx_t nbft_idx; slot_idx_t idx; struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } if (__improbable(_num_allocator_rings(CHD_SCHEMA(chd)) < 4)) { return ENOTSUP; } chrd = &chd->chd_rings[chd->chd_buf_alloc_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; if (__improbable(idx == ring->ring_tail)) { /* * do a sync to get more buflets; */ int err; err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, CHANNEL_SYNCF_ALLOC_BUF | CHANNEL_SYNCF_FREE); if (__improbable(err != 0)) { if (!_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("buflet pool alloc " "sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } return err; } } if (__improbable(idx == ring->ring_tail)) { return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : ENOMEM; } _SLOT_DESC_VERIFY(chrd, _SLOT_DESC(chrd, idx)); ubft = _SLOT_BFT_METADATA(chrd, ring, idx); _SLOT_DETACH_METADATA(_SLOT_DESC(chrd, idx)); /* * Initialize the buflet metadata buffer address. */ *(mach_vm_address_t *)(uintptr_t)&(ubft->buf_addr) = _CHANNEL_RING_BUF(chrd, ring, ubft); if (__improbable(ubft->buf_addr == 0)) { SK_ABORT_WITH_CAUSE("buflet alloc with NULL buffer", ubft->buf_idx); /* NOTREACHED */ __builtin_unreachable(); } nbft_idx = ubft->buf_nbft_idx; if (__improbable(nbft_idx != OBJ_IDX_NONE)) { if (_CHANNEL_IS_DEFUNCT(chd)) { return ENXIO; } else { SK_ABORT_WITH_CAUSE("buflet alloc with invalid nidx", nbft_idx); /* NOTREACHED */ __builtin_unreachable(); } } ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); *bft = ubft; return __improbable(_CHANNEL_IS_DEFUNCT(chd)) ? ENXIO : 0; } int os_channel_buflet_free(const channel_t chd, buflet_t ubft) { struct __user_channel_ring *ring; struct channel_ring_desc *chrd; slot_idx_t idx; obj_idx_t midx; struct ch_info *ci = CHD_INFO(chd); if (__improbable((ci->cinfo_ch_mode & CHMODE_USER_PACKET_POOL) == 0)) { return ENOTSUP; } if (__improbable(_num_allocator_rings(CHD_SCHEMA(chd)) < 4)) { return ENOTSUP; } chrd = &chd->chd_rings[chd->chd_buf_free_ring_idx]; ring = __DECONST(struct __user_channel_ring *, chrd->chrd_ring); idx = ring->ring_head; if (__improbable(idx == ring->ring_tail)) { /* * do a sync to reclaim space in free ring; */ int err; err = __channel_sync(chd->chd_fd, CHANNEL_SYNC_UPP, CHANNEL_SYNCF_FREE); if (__improbable(err != 0) && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT_WITH_CAUSE("buflet pool free " "sync failed", err); /* NOTREACHED */ __builtin_unreachable(); } } if (__improbable(idx == ring->ring_tail) && !_CHANNEL_IS_DEFUNCT(chd)) { SK_ABORT("no ring space in buflet free ring"); /* NOTREACHED */ __builtin_unreachable(); } midx = _BFT_INDEX(chrd, ubft); _SLOT_BFT_METADATA_IDX_VERIFY(chrd, ubft, midx); _SLOT_ATTACH_METADATA(_SLOT_DESC(chrd, idx), midx); ring->ring_head = _CHANNEL_RING_NEXT(ring, idx); return __improbable(_CHANNEL_RING_IS_DEFUNCT(chrd)) ? ENXIO : 0; }