1 /*
2 * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_MEM_SKMEMARENAVAR_H
30 #define _SKYWALK_MEM_SKMEMARENAVAR_H
31
32 #ifdef BSD_KERNEL_PRIVATE
33 #include <skywalk/core/skywalk_var.h>
34
35 /*
36 * Arena types.
37 */
38 typedef enum {
39 SKMEM_ARENA_TYPE_NEXUS, /* skmem_arena_nexus */
40 SKMEM_ARENA_TYPE_NECP, /* skmem_arena_necp */
41 SKMEM_ARENA_TYPE_SYSTEM, /* skmem_arena_system */
42 } skmem_arena_type_t;
43
44 struct skmem_arena_mmap_info;
45
46 /*
47 * Structure common to all arena types.
48 */
49 struct skmem_arena {
50 decl_lck_mtx_data(, ar_lock); /* arena lock */
51 uint32_t ar_refcnt; /* reference count */
52
53 /*
54 * Arena properties.
55 */
56 TAILQ_ENTRY(skmem_arena) ar_link; /* skmem_region linkage */
57 char ar_name[64]; /* arena name */
58 skmem_arena_type_t ar_type; /* arena type */
59 uint32_t ar_flags; /* ARF_* */
60 size_t ar_zsize; /* zone object size */
61 struct zone *ar_zone; /* owning zone */
62 IOSKArenaRef ar_ar; /* backing IOSKArena */
63
64 /*
65 * Regions.
66 */
67 struct skmem_region *ar_regions[SKMEM_REGIONS]; /* arena regions */
68
69 /*
70 * ar_mapsize gets set the first time the arena is mapped to a task;
71 * it is an estimate since we don't update it on subsequent mappings.
72 * We use it only for statistics purposes.
73 */
74 mach_vm_size_t ar_mapsize; /* estimated mmap size */
75 uint32_t ar_mapcnt; /* # of active mmap on arena */
76 uint32_t ar_maprdrcnt; /* # of redirected mmap */
77 SLIST_HEAD(, skmem_arena_mmap_info) ar_map_head; /* list of mmap info */
78 };
79
80 /* valid values for ar_flags */
81 #define ARF_ACTIVE 0x1 /* arena is active */
82 #define ARF_DEFUNCT (1U << 31) /* arena is defunct */
83
84 #define ARF_BITS "\020\01ACTIVE\040DEFUNCT"
85
86 #define AR_LOCK(_ar) \
87 lck_mtx_lock(&(_ar)->ar_lock)
88 #define AR_LOCK_ASSERT_HELD(_ar) \
89 LCK_MTX_ASSERT(&(_ar)->ar_lock, LCK_MTX_ASSERT_OWNED)
90 #define AR_LOCK_ASSERT_NOTHELD(_ar) \
91 LCK_MTX_ASSERT(&(_ar)->ar_lock, LCK_MTX_ASSERT_NOTOWNED)
92 #define AR_UNLOCK(_ar) \
93 lck_mtx_unlock(&(_ar)->ar_lock)
94
95 #define AR_MEM_TOTAL(_ar, _id) \
96 ((_ar)->ar_regions[_id]->skr_memtotal)
97 #define AR_MEM_INUSE(_ar, _id) \
98 ((_ar)->ar_regions[_id]->skr_meminuse)
99 #define AR_MEM_WIRED_INUSE(_ar, _id) \
100 ((_ar)->ar_regions[_id]->skr_w_meminuse)
101 #define AR_MEM_SEGSIZE(_ar, _id) \
102 ((_ar)->ar_regions[_id]->skr_seg_size)
103 #define AR_MEM_SEGCNT(_ar, _id) \
104 ((_ar)->ar_regions[_id]->skr_seg_max_cnt)
105 #define AR_MEM_OBJCNT_R(_ar, _id) \
106 ((_ar)->ar_regions[_id]->skr_r_obj_cnt)
107 #define AR_MEM_OBJCNT_C(_ar, _id) \
108 ((_ar)->ar_regions[_id]->skr_c_obj_cnt)
109 #define AR_MEM_OBJSIZE_R(_ar, _id) \
110 ((_ar)->ar_regions[_id]->skr_r_obj_size)
111 #define AR_MEM_OBJSIZE_C(_ar, _id) \
112 ((_ar)->ar_regions[_id]->skr_c_obj_size)
113
114 /*
115 * Arena Task Map Information.
116 */
117 struct skmem_arena_mmap_info {
118 SLIST_ENTRY(skmem_arena_mmap_info) ami_link;
119 struct skmem_arena *ami_arena; /* backing arena */
120 IOSKMapperRef ami_mapref; /* IOSKMapper handle */
121 task_t ami_maptask; /* task where it's mapped to */
122 mach_vm_address_t ami_mapaddr; /* start address in task */
123 mach_vm_size_t ami_mapsize; /* size of memory map */
124 boolean_t ami_redirect; /* map is redirected */
125 };
126
127 /*
128 * Nexus Adapter Arena.
129 */
130 struct skmem_arena_nexus {
131 struct skmem_arena arn_cmn; /* common arena struct */
132
133 struct kern_pbufpool *arn_rx_pp; /* rx ppool handle */
134 struct kern_pbufpool *arn_tx_pp; /* tx ppool handle */
135 uint32_t arn_mode; /* mode flags */
136 nexus_meta_type_t arn_md_type; /* mdata regions type */
137 nexus_meta_subtype_t arn_md_subtype; /* mdata regions subtype */
138 /*
139 * For arenas used by adapters with external ring, slot callbacks or
140 * invocations via KPIs accessing kernel slot descriptors, we need to
141 * make sure the ksd region is kept intact during defunct. A non-zero
142 * value indicates that we leave ksd region alone, until the time when
143 * the arena is torn down for good.
144 */
145 int arn_ksd_nodefunct;
146
147 /*
148 * Caches.
149 */
150 struct skmem_cache *arn_schema_cache; /* schema object cache */
151 struct skmem_cache *arn_ring_cache; /* ring object cache */
152 struct skmem_cache *arn_txaksd_cache; /* tx/alloc slots cache */
153 struct skmem_cache *arn_rxfksd_cache; /* rx/free slots cache */
154
155 /*
156 * Statistics.
157 *
158 * This may be NULL if the arena was created without a statistics
159 * region. Otherwise, this value contains the segment address of
160 * the object that we allocate from that region. An arena contains
161 * at most one monolithic stats region.
162 */
163 void *arn_stats_obj; /* adapter stats object */
164
165 /*
166 * Flow advisory.
167 *
168 * This may be NULL if the arena was created without a flow advisory
169 * region. Otherwise, this value contains the segment address of
170 * the object that we allocate from that region. An arena contains
171 * at most one monolithic flow advisory region.
172 */
173 struct __flowadv_entry *arn_flowadv_obj;
174
175 /*
176 * Nexus advisory.
177 *
178 * This may be NULL if the arena was created without a nexus advisory
179 * region. Otherwise, this value contains the segment address of
180 * the object that we allocate from that region. An arena contains
181 * at most one monolithic nexus advisory region, that is nexus-wide.
182 */
183 void *arn_nexusadv_obj;
184 };
185
186 /* valid flags for arn_mode */
187 #define AR_NEXUS_MODE_EXTERNAL_PPOOL 0x1 /* external packet pool */
188
189 /*
190 * Given an arena, return its nexus variant (if applicable).
191 */
192 __attribute__((always_inline))
193 static inline struct skmem_arena_nexus *
skmem_arena_nexus(struct skmem_arena * ar)194 skmem_arena_nexus(struct skmem_arena *ar)
195 {
196 if (__improbable(ar->ar_type != SKMEM_ARENA_TYPE_NEXUS)) {
197 return NULL;
198 }
199
200 return (struct skmem_arena_nexus *)ar;
201 }
202
203 /*
204 * NECP Arena.
205 */
206 struct skmem_arena_necp {
207 struct skmem_arena arc_cmn; /* common arena struct */
208
209 /*
210 * Caches.
211 */
212 /* stats cache (kernel master mirrored with slave ustats) */
213 struct skmem_cache *arc_kstats_cache;
214 };
215
216 /*
217 * System Arena.
218 */
219 struct skmem_arena_system {
220 struct skmem_arena ars_cmn; /* common arena struct */
221
222 /*
223 * sysctls.
224 *
225 * This value contains the kernel virtual address of the system-wide
226 * sysctls object. This object is persistent, i.e. it does not get
227 * allocated or freed along with the arena.
228 */
229 void *ars_sysctls_obj;
230 size_t ars_sysctls_objsize;
231 };
232
233 struct kern_nexus_advisory;
234
235 __BEGIN_DECLS
236
237 #define SKMEM_PP_FLAG_KERNEL_ONLY 0x01
238 #define SKMEM_PP_FLAG_TRUNCATED_BUF 0x02
239 #define SKMEM_PP_FLAG_RAW_BFLT 0x04
240
241 extern struct skmem_arena *skmem_arena_create_for_nexus(
242 const struct nexus_adapter *, struct skmem_region_params[SKMEM_REGIONS],
243 struct kern_pbufpool **, struct kern_pbufpool **, uint32_t,
244 struct kern_nexus_advisory *, int *);
245 extern void skmem_arena_nexus_sd_set_noidle(struct skmem_arena_nexus *, int);
246 extern boolean_t skmem_arena_nexus_sd_idle(struct skmem_arena_nexus *);
247
248 extern struct skmem_arena *skmem_arena_create_for_necp(const char *,
249 struct skmem_region_params *, struct skmem_region_params *, int *);
250 extern struct skmem_arena_necp *skmem_arena_necp(struct skmem_arena *);
251
252 extern struct skmem_arena *skmem_arena_create_for_system(const char *, int *);
253 extern struct skmem_arena_system *skmem_arena_system(struct skmem_arena *);
254 extern void *skmem_arena_system_sysctls_obj_addr(struct skmem_arena *);
255 extern size_t skmem_arena_system_sysctls_obj_size(struct skmem_arena *);
256
257 extern void skmem_arena_retain(struct skmem_arena *);
258 extern boolean_t skmem_arena_release(struct skmem_arena *);
259 extern int skmem_arena_mmap(struct skmem_arena *, struct proc *,
260 struct skmem_arena_mmap_info *);
261 extern void skmem_arena_munmap(struct skmem_arena *,
262 struct skmem_arena_mmap_info *);
263 extern void skmem_arena_munmap_channel(struct skmem_arena *,
264 struct kern_channel *);
265 extern int skmem_arena_mredirect(struct skmem_arena *,
266 struct skmem_arena_mmap_info *, struct proc *, boolean_t *);
267 extern int skmem_arena_defunct(struct skmem_arena *);
268 extern void skmem_arena_get_stats(struct skmem_arena *, uint64_t *,
269 uint64_t *);
270 extern mach_vm_offset_t skmem_arena_get_region_offset(struct skmem_arena *,
271 skmem_region_id_t);
272 extern void skmem_arena_reap(struct skmem_arena *, boolean_t);
273 __END_DECLS
274 #endif /* BSD_KERNEL_PRIVATE */
275 #endif /* _SKYWALK_MEM_SKMEMARENAVAR_H */
276