1 /* 2 * Copyright (c) 2016-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _SKYWALK_PACKET_PBUFPOOLVAR_H_ 30 #define _SKYWALK_PACKET_PBUFPOOLVAR_H_ 31 32 #ifdef BSD_KERNEL_PRIVATE 33 #include <skywalk/core/skywalk_var.h> 34 35 struct __kern_quantum; 36 struct __kern_packet; 37 38 /* 39 * User packet pool hash bucket. Packets allocated by user space are 40 * kept in the hash table. This allows the kernel to validate whether 41 * or not a given packet object is valid or is already-freed, and thus 42 * take the appropriate measure during internalize. 43 */ 44 struct kern_pbufpool_u_bkt { 45 SLIST_HEAD(, __kern_quantum) upp_head; 46 }; 47 48 struct kern_pbufpool_u_bft_bkt { 49 SLIST_HEAD(, __kern_buflet_ext) upp_head; 50 }; 51 52 struct kern_pbufpool { 53 decl_lck_mtx_data(, pp_lock); 54 uint32_t pp_refcnt; 55 uint32_t pp_flags; 56 uint16_t pp_buflet_size; 57 uint16_t pp_max_frags; 58 59 /* 60 * Caches 61 */ 62 struct skmem_cache *pp_buf_cache; 63 struct skmem_cache *pp_kmd_cache; 64 struct skmem_cache *pp_kbft_cache; 65 66 /* 67 * Regions 68 */ 69 struct skmem_region *pp_buf_region; 70 struct skmem_region *pp_kmd_region; 71 struct skmem_region *pp_umd_region; 72 struct skmem_region *pp_ubft_region; 73 struct skmem_region *pp_kbft_region; 74 75 /* 76 * User packet pool: packet metadata hash table 77 */ 78 struct kern_pbufpool_u_bkt *pp_u_hash_table; 79 uint64_t pp_u_bufinuse; 80 81 /* 82 * User packet pool: buflet hash table 83 */ 84 struct kern_pbufpool_u_bft_bkt *pp_u_bft_hash_table; 85 uint64_t pp_u_bftinuse; 86 87 void *pp_ctx; 88 pbuf_ctx_retain_fn_t pp_ctx_retain; 89 pbuf_ctx_release_fn_t pp_ctx_release; 90 nexus_meta_type_t pp_md_type; 91 nexus_meta_subtype_t pp_md_subtype; 92 uint32_t pp_midx_start; 93 uint32_t pp_bidx_start; 94 pbufpool_name_t pp_name; 95 pbuf_seg_ctor_fn_t pp_pbuf_seg_ctor; 96 pbuf_seg_dtor_fn_t pp_pbuf_seg_dtor; 97 }; 98 99 /* valid values for pp_flags */ 100 #define PPF_EXTERNAL 0x1 /* externally configured */ 101 #define PPF_CLOSED 0x2 /* closed; awaiting final destruction */ 102 #define PPF_MONOLITHIC 0x4 /* non slab-based buffer region */ 103 /* buflet is truncated and may not contain the full payload */ 104 #define PPF_TRUNCATED_BUF 0x8 105 #define PPF_KERNEL 0x10 /* kernel only, no user region(s) */ 106 #define PPF_BUFFER_ON_DEMAND 0x20 /* attach buffers to packet on demand */ 107 #define PPF_BATCH 0x40 /* capable of batch alloc/free */ 108 #define PPF_DYNAMIC 0x80 /* capable of magazine resizing */ 109 110 #define PP_KERNEL_ONLY(_pp) \ 111 (((_pp)->pp_flags & PPF_KERNEL) != 0) 112 113 #define PP_HAS_TRUNCATED_BUF(_pp) \ 114 (((_pp)->pp_flags & PPF_TRUNCATED_BUF) != 0) 115 116 #define PP_HAS_BUFFER_ON_DEMAND(_pp) \ 117 (((_pp)->pp_flags & PPF_BUFFER_ON_DEMAND) != 0) 118 119 #define PP_BATCH_CAPABLE(_pp) \ 120 (((_pp)->pp_flags & PPF_BATCH) != 0) 121 122 #define PP_DYNAMIC(_pp) \ 123 (((_pp)->pp_flags & PPF_DYNAMIC) != 0) 124 125 #define PP_LOCK(_pp) \ 126 lck_mtx_lock(&_pp->pp_lock) 127 #define PP_LOCK_ASSERT_HELD(_pp) \ 128 LCK_MTX_ASSERT(&_pp->pp_lock, LCK_MTX_ASSERT_OWNED) 129 #define PP_LOCK_ASSERT_NOTHELD(_pp) \ 130 LCK_MTX_ASSERT(&_pp->pp_lock, LCK_MTX_ASSERT_NOTOWNED) 131 #define PP_UNLOCK(_pp) \ 132 lck_mtx_unlock(&_pp->pp_lock) 133 134 __BEGIN_DECLS 135 extern int pp_init(void); 136 extern void pp_fini(void); 137 extern void pp_close(struct kern_pbufpool *); 138 139 /* create flags for pp_create() */ 140 #define PPCREATEF_EXTERNAL 0x1 /* externally requested */ 141 #define PPCREATEF_KERNEL_ONLY 0x2 /* kernel-only */ 142 #define PPCREATEF_TRUNCATED_BUF 0x4 /* compat-only (buf is short) */ 143 #define PPCREATEF_ONDEMAND_BUF 0x8 /* buf alloc/free is decoupled */ 144 #define PPCREATEF_DYNAMIC 0x10 /* dynamic per-CPU magazines */ 145 146 extern struct kern_pbufpool *pp_create(const char *name, 147 struct skmem_region_params *buf_srp, struct skmem_region_params *kmd_srp, 148 struct skmem_region_params *kbft_srp, struct skmem_region_params *ubft_srp, 149 struct skmem_region_params *umd_srp, pbuf_seg_ctor_fn_t buf_seg_ctor, 150 pbuf_seg_dtor_fn_t buf_seg_dtor, const void *ctx, 151 pbuf_ctx_retain_fn_t ctx_retain, pbuf_ctx_release_fn_t ctx_release, 152 uint32_t ppcreatef); 153 extern void pp_destroy(struct kern_pbufpool *); 154 155 extern int pp_init_upp(struct kern_pbufpool *, boolean_t); 156 extern void pp_insert_upp(struct kern_pbufpool *, struct __kern_quantum *, 157 pid_t); 158 extern void pp_insert_upp_locked(struct kern_pbufpool *, 159 struct __kern_quantum *, pid_t); 160 extern void pp_insert_upp_batch(struct kern_pbufpool *pp, pid_t pid, 161 uint64_t *array, uint32_t num); 162 extern struct __kern_quantum *pp_remove_upp(struct kern_pbufpool *, obj_idx_t, 163 int *); 164 extern struct __kern_quantum *pp_remove_upp_locked(struct kern_pbufpool *, 165 obj_idx_t, int *); 166 extern struct __kern_quantum *pp_find_upp(struct kern_pbufpool *, obj_idx_t); 167 extern void pp_purge_upp(struct kern_pbufpool *, pid_t); 168 extern struct __kern_buflet *pp_remove_upp_bft(struct kern_pbufpool *, 169 obj_idx_t, int *); 170 extern void pp_insert_upp_bft(struct kern_pbufpool *, struct __kern_buflet *, 171 pid_t); 172 extern boolean_t pp_isempty_upp(struct kern_pbufpool *); 173 174 extern void pp_retain_locked(struct kern_pbufpool *); 175 extern void pp_retain(struct kern_pbufpool *); 176 extern boolean_t pp_release_locked(struct kern_pbufpool *); 177 extern boolean_t pp_release(struct kern_pbufpool *); 178 179 extern void pp_regions_params_adjust(struct skmem_region_params *, 180 struct skmem_region_params *, struct skmem_region_params *, 181 struct skmem_region_params *, struct skmem_region_params *, 182 nexus_meta_type_t, nexus_meta_subtype_t, uint32_t, uint16_t, 183 uint32_t, uint32_t); 184 185 extern uint64_t pp_alloc_packet(struct kern_pbufpool *, uint16_t, uint32_t); 186 extern uint64_t pp_alloc_packet_by_size(struct kern_pbufpool *, uint32_t, 187 uint32_t); 188 extern int pp_alloc_packet_batch(struct kern_pbufpool *, uint16_t, uint64_t *, 189 uint32_t *, boolean_t, alloc_cb_func_t, const void *, uint32_t); 190 extern int pp_alloc_pktq(struct kern_pbufpool *, uint16_t, struct pktq *, 191 uint32_t, alloc_cb_func_t, const void *, uint32_t); 192 extern void pp_free_packet(struct kern_pbufpool *, uint64_t); 193 extern void pp_free_packet_batch(struct kern_pbufpool *, uint64_t *, uint32_t); 194 extern void pp_free_packet_single(struct __kern_packet *); 195 extern void pp_free_packet_chain(struct __kern_packet *, int *); 196 extern void pp_free_pktq(struct pktq *); 197 extern errno_t pp_alloc_buffer(const kern_pbufpool_t, mach_vm_address_t *, 198 kern_segment_t *, kern_obj_idx_seg_t *, uint32_t); 199 extern void pp_free_buffer(const kern_pbufpool_t, mach_vm_address_t); 200 extern errno_t pp_alloc_buflet(struct kern_pbufpool *pp, kern_buflet_t *kbft, 201 uint32_t skmflag); 202 extern errno_t pp_alloc_buflet_batch(struct kern_pbufpool *pp, uint64_t *array, 203 uint32_t *size, uint32_t skmflag); 204 extern void pp_free_buflet(const kern_pbufpool_t, kern_buflet_t); 205 206 extern void pp_reap_caches(boolean_t); 207 __END_DECLS 208 #endif /* BSD_KERNEL_PRIVATE */ 209 #endif /* !_SKYWALK_PACKET_PBUFPOOLVAR_H_ */ 210