1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* BEGIN CSTYLED */
30 /*
31 * A region represents a collection of one or more similarly-sized memory
32 * segments, each of which is a contiguous range of integers. A segment
33 * is either allocated or free, and is treated as disjoint from all other
34 * segments. That is, the contiguity applies only at the segment level,
35 * and a region with multiple segments is not contiguous at the region level.
36 * A segment always belongs to the segment freelist, or the allocated-address
37 * hash chain, as described below.
38 *
39 * The optional SKMEM_REGION_CR_NOREDIRECT flag indicates that the region
40 * stays intact even after a defunct. Otherwise, the segments belonging
41 * to the region will be freed at defunct time, and the span covered by
42 * the region will be redirected to zero-filled anonymous memory.
43 *
44 * Memory for a region is always created as pageable and purgeable. It is
45 * the client's responsibility to prepare (wire) it, and optionally insert
46 * it to the IOMMU, at segment construction time. When the segment is
47 * freed, the client is responsible for removing it from IOMMU (if needed),
48 * and complete (unwire) it.
49 *
50 * When the region is created with SKMEM_REGION_CR_PERSISTENT, the memory
51 * is immediately wired upon allocation (segment removed from freelist).
52 * It gets unwired when memory is discarded (segment inserted to freelist).
53 *
54 * The chronological life cycle of a segment is as such:
55 *
56 * SKSEG_STATE_DETACHED
57 * SKSEG_STATE_{MAPPED,MAPPED_WIRED}
58 * [segment allocated, useable by client]
59 * ...
60 * [client frees segment]
61 * SKSEG_STATE_{MAPPED,MAPPED_WIRED}
62 * [reclaim]
63 * SKSEG_STATE_DETACHED
64 *
65 * The region can also be marked as user-mappable (SKMEM_REGION_CR_MMAPOK);
66 * this allows it to be further marked with SKMEM_REGION_CR_UREADONLY to
67 * prevent modifications by the user task. Only user-mappable regions will
68 * be considered for inclusion during skmem_arena_mmap().
69 *
70 * Every skmem allocator has a region as its slab supplier. Each slab is
71 * exactly a segment. The allocator uses skmem_region_{alloc,free}() to
72 * create and destroy slabs.
73 *
74 * A region may be mirrored by another region; the latter acts as the master
75 * controller for both regions. Mirrored (slave) regions cannot be used
76 * directly by the skmem allocator. Region mirroring technique is used for
77 * managing shadow objects {umd,kmd} and {usd,ksd}, where an object in one
78 * region has the same size and lifetime as its shadow counterpart.
79 *
80 * CREATION/DESTRUCTION:
81 *
82 * At creation time, all segments are allocated and are immediately inserted
83 * into the freelist. Allocating a purgeable segment has very little cost,
84 * as it is not backed by physical memory until it is accessed. Immediate
85 * insertion into the freelist causes the mapping to be further torn down.
86 *
87 * At destruction time, the freelist is emptied, and each segment is then
88 * destroyed. The system will assert if it detects there are outstanding
89 * segments not yet returned to the region (not freed by the client.)
90 *
91 * ALLOCATION:
92 *
93 * Allocating involves searching the freelist for a segment; if found, the
94 * segment is removed from the freelist and is inserted into the allocated-
95 * address hash chain. The address of the memory object represented by
96 * the segment is used as hash key. The use of allocated-address hash chain
97 * is needed since we return the address of the memory object, and not the
98 * segment's itself, to the client.
99 *
100 * DEALLOCATION:
101 *
102 * Freeing a memory object causes the chain to be searched for a matching
103 * segment. The system will assert if a segment cannot be found, since
104 * that indicates that the memory object address is invalid. Once found,
105 * the segment is removed from the allocated-address hash chain, and is
106 * inserted to the freelist.
107 *
108 * Segment allocation and deallocation can be expensive. Because of this,
109 * we expect that most clients will utilize the skmem_cache slab allocator
110 * as the frontend instead.
111 */
112 /* END CSTYLED */
113
114 #include <skywalk/os_skywalk_private.h>
115 #define _FN_KPRINTF /* don't redefine kprintf() */
116 #include <pexpert/pexpert.h> /* for PE_parse_boot_argn */
117
118 static void skmem_region_destroy(struct skmem_region *skr);
119 static void skmem_region_depopulate(struct skmem_region *);
120 static int sksegment_cmp(const struct sksegment *, const struct sksegment *);
121 static struct sksegment *sksegment_create(struct skmem_region *, uint32_t);
122 static void sksegment_destroy(struct skmem_region *, struct sksegment *);
123 static void sksegment_freelist_insert(struct skmem_region *,
124 struct sksegment *, boolean_t);
125 static struct sksegment *sksegment_freelist_remove(struct skmem_region *,
126 struct sksegment *, uint32_t, boolean_t);
127 static struct sksegment *sksegment_freelist_grow(struct skmem_region *);
128 static struct sksegment *sksegment_alloc_with_idx(struct skmem_region *,
129 uint32_t);
130 static void *skmem_region_alloc_common(struct skmem_region *,
131 struct sksegment *);
132 static void *skmem_region_mirror_alloc(struct skmem_region *,
133 struct sksegment *, struct sksegment **);
134 static void skmem_region_applyall(void (*)(struct skmem_region *));
135 static void skmem_region_update(struct skmem_region *);
136 static void skmem_region_update_func(thread_call_param_t, thread_call_param_t);
137 static inline void skmem_region_retain_locked(struct skmem_region *);
138 static inline boolean_t skmem_region_release_locked(struct skmem_region *);
139 static int skmem_region_mib_get_sysctl SYSCTL_HANDLER_ARGS;
140
141 RB_PROTOTYPE_PREV(segtfreehead, sksegment, sg_node, sksegment_cmp);
142 RB_GENERATE_PREV(segtfreehead, sksegment, sg_node, sksegment_cmp);
143
144 SYSCTL_PROC(_kern_skywalk_stats, OID_AUTO, region,
145 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
146 0, 0, skmem_region_mib_get_sysctl, "S,sk_stats_region",
147 "Skywalk region statistics");
148
149 static LCK_ATTR_DECLARE(skmem_region_lock_attr, 0, 0);
150 static LCK_GRP_DECLARE(skmem_region_lock_grp, "skmem_region");
151 static LCK_MTX_DECLARE_ATTR(skmem_region_lock, &skmem_region_lock_grp,
152 &skmem_region_lock_attr);
153
154 /* protected by skmem_region_lock */
155 static TAILQ_HEAD(, skmem_region) skmem_region_head;
156
157 static thread_call_t skmem_region_update_tc;
158
159 #define SKMEM_REGION_UPDATE_INTERVAL 13 /* 13 seconds */
160 static uint32_t skmem_region_update_interval = SKMEM_REGION_UPDATE_INTERVAL;
161
162 #define SKMEM_WDT_MAXTIME 30 /* # of secs before watchdog */
163 #define SKMEM_WDT_PURGE 3 /* retry purge threshold */
164
165 #if (DEVELOPMENT || DEBUG)
166 /* Mean Time Between Failures (ms) */
167 static volatile uint64_t skmem_region_mtbf;
168
169 static int skmem_region_mtbf_sysctl(struct sysctl_oid *, void *, int,
170 struct sysctl_req *);
171
172 SYSCTL_PROC(_kern_skywalk_mem, OID_AUTO, region_mtbf,
173 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
174 skmem_region_mtbf_sysctl, "Q", "Region MTBF (ms)");
175
176 SYSCTL_UINT(_kern_skywalk_mem, OID_AUTO, region_update_interval,
177 CTLFLAG_RW | CTLFLAG_LOCKED, &skmem_region_update_interval,
178 SKMEM_REGION_UPDATE_INTERVAL, "Region update interval (sec)");
179 #endif /* (DEVELOPMENT || DEBUG) */
180
181 #define SKMEM_REGION_LOCK() \
182 lck_mtx_lock(&skmem_region_lock)
183 #define SKMEM_REGION_LOCK_ASSERT_HELD() \
184 LCK_MTX_ASSERT(&skmem_region_lock, LCK_MTX_ASSERT_OWNED)
185 #define SKMEM_REGION_LOCK_ASSERT_NOTHELD() \
186 LCK_MTX_ASSERT(&skmem_region_lock, LCK_MTX_ASSERT_NOTOWNED)
187 #define SKMEM_REGION_UNLOCK() \
188 lck_mtx_unlock(&skmem_region_lock)
189
190 /*
191 * Hash table bounds. Start with the initial value, and rescale up to
192 * the specified limit. Ideally we don't need a limit, but in practice
193 * this helps guard against runaways. These values should be revisited
194 * in future and be adjusted as needed.
195 */
196 #define SKMEM_REGION_HASH_INITIAL 32 /* initial hash table size */
197 #define SKMEM_REGION_HASH_LIMIT 4096 /* hash table size limit */
198
199 #define SKMEM_REGION_HASH_INDEX(_a, _s, _m) \
200 (((_a) + ((_a) >> (_s)) + ((_a) >> ((_s) << 1))) & (_m))
201 #define SKMEM_REGION_HASH(_skr, _addr) \
202 (&(_skr)->skr_hash_table[SKMEM_REGION_HASH_INDEX((uintptr_t)_addr, \
203 (_skr)->skr_hash_shift, (_skr)->skr_hash_mask)])
204
205 static SKMEM_TYPE_DEFINE(skr_zone, struct skmem_region);
206
207 static unsigned int sg_size; /* size of zone element */
208 static struct skmem_cache *skmem_sg_cache; /* cache for sksegment */
209
210 static uint32_t skmem_seg_size = SKMEM_SEG_SIZE;
211 static uint32_t skmem_md_seg_size = SKMEM_MD_SEG_SIZE;
212 static uint32_t skmem_drv_buf_seg_size = SKMEM_DRV_BUF_SEG_SIZE;
213 static uint32_t skmem_drv_buf_seg_eff_size = SKMEM_DRV_BUF_SEG_SIZE;
214 uint32_t skmem_usr_buf_seg_size = SKMEM_USR_BUF_SEG_SIZE;
215
216 #define SKMEM_TAG_SEGMENT_BMAP "com.apple.skywalk.segment.bmap"
217 static SKMEM_TAG_DEFINE(skmem_tag_segment_bmap, SKMEM_TAG_SEGMENT_BMAP);
218
219 #define SKMEM_TAG_SEGMENT_HASH "com.apple.skywalk.segment.hash"
220 static SKMEM_TAG_DEFINE(skmem_tag_segment_hash, SKMEM_TAG_SEGMENT_HASH);
221
222 #define SKMEM_TAG_REGION_MIB "com.apple.skywalk.region.mib"
223 static SKMEM_TAG_DEFINE(skmem_tag_region_mib, SKMEM_TAG_REGION_MIB);
224
225 #define BMAPSZ 64
226
227 /* 64-bit mask with range */
228 #define BMASK64(_beg, _end) \
229 ((((uint64_t)-1) >> ((BMAPSZ - 1) - (_end))) & ~((1ULL << (_beg)) - 1))
230
231 static int __skmem_region_inited = 0;
232
233 void
skmem_region_init(void)234 skmem_region_init(void)
235 {
236 boolean_t randomize_seg_size;
237
238 _CASSERT(sizeof(bitmap_t) == sizeof(uint64_t));
239 _CASSERT(BMAPSZ == (sizeof(bitmap_t) << 3));
240 _CASSERT((SKMEM_SEG_SIZE % SKMEM_PAGE_SIZE) == 0);
241 _CASSERT(SKMEM_REGION_HASH_LIMIT >= SKMEM_REGION_HASH_INITIAL);
242 ASSERT(!__skmem_region_inited);
243
244 /* enforce the ordering here */
245 _CASSERT(SKMEM_REGION_GUARD_HEAD == 0);
246 _CASSERT(SKMEM_REGION_SCHEMA == 1);
247 _CASSERT(SKMEM_REGION_RING == 2);
248 _CASSERT(SKMEM_REGION_BUF_DEF == 3);
249 _CASSERT(SKMEM_REGION_BUF_LARGE == 4);
250 _CASSERT(SKMEM_REGION_RXBUF_DEF == 5);
251 _CASSERT(SKMEM_REGION_RXBUF_LARGE == 6);
252 _CASSERT(SKMEM_REGION_TXBUF_DEF == 7);
253 _CASSERT(SKMEM_REGION_TXBUF_LARGE == 8);
254 _CASSERT(SKMEM_REGION_UMD == 9);
255 _CASSERT(SKMEM_REGION_TXAUSD == 10);
256 _CASSERT(SKMEM_REGION_RXFUSD == 11);
257 _CASSERT(SKMEM_REGION_UBFT == 12);
258 _CASSERT(SKMEM_REGION_USTATS == 13);
259 _CASSERT(SKMEM_REGION_FLOWADV == 14);
260 _CASSERT(SKMEM_REGION_NEXUSADV == 15);
261 _CASSERT(SKMEM_REGION_SYSCTLS == 16);
262 _CASSERT(SKMEM_REGION_GUARD_TAIL == 17);
263 _CASSERT(SKMEM_REGION_KMD == 18);
264 _CASSERT(SKMEM_REGION_RXKMD == 19);
265 _CASSERT(SKMEM_REGION_TXKMD == 20);
266 _CASSERT(SKMEM_REGION_KBFT == 21);
267 _CASSERT(SKMEM_REGION_RXKBFT == 22);
268 _CASSERT(SKMEM_REGION_TXKBFT == 23);
269 _CASSERT(SKMEM_REGION_TXAKSD == 24);
270 _CASSERT(SKMEM_REGION_RXFKSD == 25);
271 _CASSERT(SKMEM_REGION_KSTATS == 26);
272 _CASSERT(SKMEM_REGION_INTRINSIC == 27);
273
274 _CASSERT(SREG_GUARD_HEAD == SKMEM_REGION_GUARD_HEAD);
275 _CASSERT(SREG_SCHEMA == SKMEM_REGION_SCHEMA);
276 _CASSERT(SREG_RING == SKMEM_REGION_RING);
277 _CASSERT(SREG_BUF_DEF == SKMEM_REGION_BUF_DEF);
278 _CASSERT(SREG_BUF_LARGE == SKMEM_REGION_BUF_LARGE);
279 _CASSERT(SREG_RXBUF_DEF == SKMEM_REGION_RXBUF_DEF);
280 _CASSERT(SREG_RXBUF_LARGE == SKMEM_REGION_RXBUF_LARGE);
281 _CASSERT(SREG_TXBUF_DEF == SKMEM_REGION_TXBUF_DEF);
282 _CASSERT(SREG_TXBUF_LARGE == SKMEM_REGION_TXBUF_LARGE);
283 _CASSERT(SREG_UMD == SKMEM_REGION_UMD);
284 _CASSERT(SREG_TXAUSD == SKMEM_REGION_TXAUSD);
285 _CASSERT(SREG_RXFUSD == SKMEM_REGION_RXFUSD);
286 _CASSERT(SREG_UBFT == SKMEM_REGION_UBFT);
287 _CASSERT(SREG_USTATS == SKMEM_REGION_USTATS);
288 _CASSERT(SREG_FLOWADV == SKMEM_REGION_FLOWADV);
289 _CASSERT(SREG_NEXUSADV == SKMEM_REGION_NEXUSADV);
290 _CASSERT(SREG_SYSCTLS == SKMEM_REGION_SYSCTLS);
291 _CASSERT(SREG_GUARD_TAIL == SKMEM_REGION_GUARD_TAIL);
292 _CASSERT(SREG_KMD == SKMEM_REGION_KMD);
293 _CASSERT(SREG_RXKMD == SKMEM_REGION_RXKMD);
294 _CASSERT(SREG_TXKMD == SKMEM_REGION_TXKMD);
295 _CASSERT(SREG_KBFT == SKMEM_REGION_KBFT);
296 _CASSERT(SREG_RXKBFT == SKMEM_REGION_RXKBFT);
297 _CASSERT(SREG_TXKBFT == SKMEM_REGION_TXKBFT);
298 _CASSERT(SREG_TXAKSD == SKMEM_REGION_TXAKSD);
299 _CASSERT(SREG_RXFKSD == SKMEM_REGION_RXFKSD);
300 _CASSERT(SREG_KSTATS == SKMEM_REGION_KSTATS);
301
302 _CASSERT(SKR_MODE_NOREDIRECT == SREG_MODE_NOREDIRECT);
303 _CASSERT(SKR_MODE_MMAPOK == SREG_MODE_MMAPOK);
304 _CASSERT(SKR_MODE_UREADONLY == SREG_MODE_UREADONLY);
305 _CASSERT(SKR_MODE_KREADONLY == SREG_MODE_KREADONLY);
306 _CASSERT(SKR_MODE_PERSISTENT == SREG_MODE_PERSISTENT);
307 _CASSERT(SKR_MODE_MONOLITHIC == SREG_MODE_MONOLITHIC);
308 _CASSERT(SKR_MODE_NOMAGAZINES == SREG_MODE_NOMAGAZINES);
309 _CASSERT(SKR_MODE_NOCACHE == SREG_MODE_NOCACHE);
310 _CASSERT(SKR_MODE_IODIR_IN == SREG_MODE_IODIR_IN);
311 _CASSERT(SKR_MODE_IODIR_OUT == SREG_MODE_IODIR_OUT);
312 _CASSERT(SKR_MODE_GUARD == SREG_MODE_GUARD);
313 _CASSERT(SKR_MODE_SEGPHYSCONTIG == SREG_MODE_SEGPHYSCONTIG);
314 _CASSERT(SKR_MODE_SHAREOK == SREG_MODE_SHAREOK);
315 _CASSERT(SKR_MODE_PUREDATA == SREG_MODE_PUREDATA);
316 _CASSERT(SKR_MODE_PSEUDO == SREG_MODE_PSEUDO);
317 _CASSERT(SKR_MODE_THREADSAFE == SREG_MODE_THREADSAFE);
318 _CASSERT(SKR_MODE_SLAB == SREG_MODE_SLAB);
319 _CASSERT(SKR_MODE_MIRRORED == SREG_MODE_MIRRORED);
320
321 (void) PE_parse_boot_argn("skmem_seg_size", &skmem_seg_size,
322 sizeof(skmem_seg_size));
323 if (skmem_seg_size < SKMEM_MIN_SEG_SIZE) {
324 skmem_seg_size = SKMEM_MIN_SEG_SIZE;
325 }
326 skmem_seg_size = (uint32_t)P2ROUNDUP(skmem_seg_size,
327 SKMEM_MIN_SEG_SIZE);
328 VERIFY(skmem_seg_size != 0 && (skmem_seg_size % SKMEM_PAGE_SIZE) == 0);
329
330 (void) PE_parse_boot_argn("skmem_md_seg_size", &skmem_md_seg_size,
331 sizeof(skmem_md_seg_size));
332 if (skmem_md_seg_size < skmem_seg_size) {
333 skmem_md_seg_size = skmem_seg_size;
334 }
335 skmem_md_seg_size = (uint32_t)P2ROUNDUP(skmem_md_seg_size,
336 SKMEM_MIN_SEG_SIZE);
337 VERIFY((skmem_md_seg_size % SKMEM_PAGE_SIZE) == 0);
338
339 /*
340 * If set via boot-args, honor it and don't randomize.
341 */
342 randomize_seg_size = !PE_parse_boot_argn("skmem_drv_buf_seg_size",
343 &skmem_drv_buf_seg_size, sizeof(skmem_drv_buf_seg_size));
344 if (skmem_drv_buf_seg_size < skmem_seg_size) {
345 skmem_drv_buf_seg_size = skmem_seg_size;
346 }
347 skmem_drv_buf_seg_size = skmem_drv_buf_seg_eff_size =
348 (uint32_t)P2ROUNDUP(skmem_drv_buf_seg_size, SKMEM_MIN_SEG_SIZE);
349 VERIFY((skmem_drv_buf_seg_size % SKMEM_PAGE_SIZE) == 0);
350
351 /*
352 * Randomize the driver buffer segment size; here we choose
353 * a SKMEM_MIN_SEG_SIZE multiplier to bump up the value to.
354 * Set this as the effective driver buffer segment size.
355 */
356 if (randomize_seg_size) {
357 uint32_t sm;
358 read_frandom(&sm, sizeof(sm));
359 skmem_drv_buf_seg_eff_size +=
360 (SKMEM_MIN_SEG_SIZE * (sm % SKMEM_DRV_BUF_SEG_MULTIPLIER));
361 VERIFY((skmem_drv_buf_seg_eff_size % SKMEM_MIN_SEG_SIZE) == 0);
362 }
363 VERIFY(skmem_drv_buf_seg_eff_size >= skmem_drv_buf_seg_size);
364
365 (void) PE_parse_boot_argn("skmem_usr_buf_seg_size",
366 &skmem_usr_buf_seg_size, sizeof(skmem_usr_buf_seg_size));
367 if (skmem_usr_buf_seg_size < skmem_seg_size) {
368 skmem_usr_buf_seg_size = skmem_seg_size;
369 }
370 skmem_usr_buf_seg_size = (uint32_t)P2ROUNDUP(skmem_usr_buf_seg_size,
371 SKMEM_MIN_SEG_SIZE);
372 VERIFY((skmem_usr_buf_seg_size % SKMEM_PAGE_SIZE) == 0);
373
374 SK_ERR("seg_size %u, md_seg_size %u, drv_buf_seg_size %u [eff %u], "
375 "usr_buf_seg_size %u", skmem_seg_size, skmem_md_seg_size,
376 skmem_drv_buf_seg_size, skmem_drv_buf_seg_eff_size,
377 skmem_usr_buf_seg_size);
378
379 TAILQ_INIT(&skmem_region_head);
380
381 skmem_region_update_tc =
382 thread_call_allocate_with_options(skmem_region_update_func,
383 NULL, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE);
384 if (skmem_region_update_tc == NULL) {
385 panic("%s: thread_call_allocate failed", __func__);
386 /* NOTREACHED */
387 __builtin_unreachable();
388 }
389
390 sg_size = sizeof(struct sksegment);
391 skmem_sg_cache = skmem_cache_create("sg", sg_size,
392 sizeof(uint64_t), NULL, NULL, NULL, NULL, NULL, 0);
393
394 /* and start the periodic region update machinery */
395 skmem_dispatch(skmem_region_update_tc, NULL,
396 (skmem_region_update_interval * NSEC_PER_SEC));
397
398 __skmem_region_inited = 1;
399 }
400
401 void
skmem_region_fini(void)402 skmem_region_fini(void)
403 {
404 if (__skmem_region_inited) {
405 ASSERT(TAILQ_EMPTY(&skmem_region_head));
406
407 if (skmem_region_update_tc != NULL) {
408 (void) thread_call_cancel_wait(skmem_region_update_tc);
409 (void) thread_call_free(skmem_region_update_tc);
410 skmem_region_update_tc = NULL;
411 }
412
413 if (skmem_sg_cache != NULL) {
414 skmem_cache_destroy(skmem_sg_cache);
415 skmem_sg_cache = NULL;
416 }
417
418 __skmem_region_inited = 0;
419 }
420 }
421
422 /*
423 * Reap internal caches.
424 */
425 void
skmem_region_reap_caches(boolean_t purge)426 skmem_region_reap_caches(boolean_t purge)
427 {
428 skmem_cache_reap_now(skmem_sg_cache, purge);
429 }
430
431 /*
432 * Configure and compute the parameters of a region.
433 */
434 void
skmem_region_params_config(struct skmem_region_params * srp)435 skmem_region_params_config(struct skmem_region_params *srp)
436 {
437 uint32_t cache_line_size = skmem_cpu_cache_line_size();
438 size_t seglim, segsize, segcnt;
439 size_t objsize, objcnt;
440
441 ASSERT(srp->srp_id < SKMEM_REGIONS);
442
443 /*
444 * If magazines layer is disabled system-wide, override
445 * the region parameter here. This will effectively reduce
446 * the number of requested objects computed below. Note that
447 * the region may have already been configured to exclude
448 * magazines in the default skmem_regions[] array.
449 */
450 if (!skmem_allow_magazines()) {
451 srp->srp_cflags |= SKMEM_REGION_CR_NOMAGAZINES;
452 }
453
454 objsize = srp->srp_r_obj_size;
455 ASSERT(objsize != 0);
456 objcnt = srp->srp_r_obj_cnt;
457 ASSERT(objcnt != 0);
458
459 if (srp->srp_cflags & SKMEM_REGION_CR_PSEUDO) {
460 size_t align = srp->srp_align;
461
462 VERIFY(align != 0 && (align % SKMEM_CACHE_ALIGN) == 0);
463 VERIFY(powerof2(align));
464 objsize = MAX(objsize, sizeof(uint64_t));
465 #if KASAN
466 /*
467 * When KASAN is enabled, the zone allocator adjusts the
468 * element size to include the redzone regions, in which
469 * case we assume that the elements won't start on the
470 * alignment boundary and thus need to do some fix-ups.
471 * These include increasing the effective object size
472 * which adds at least 16 bytes to the original size.
473 */
474 objsize += sizeof(uint64_t) + align;
475 #endif /* KASAN */
476 objsize = P2ROUNDUP(objsize, align);
477
478 segsize = objsize;
479 srp->srp_r_seg_size = (uint32_t)segsize;
480 segcnt = objcnt;
481 goto done;
482 } else {
483 /* objects are always aligned at CPU cache line size */
484 srp->srp_align = cache_line_size;
485 }
486
487 /*
488 * Start with default segment size for the region, and compute the
489 * effective segment size (to nearest SKMEM_MIN_SEG_SIZE). If the
490 * object size is greater, then we adjust the segment size to next
491 * multiple of the effective size larger than the object size.
492 */
493 if (srp->srp_r_seg_size == 0) {
494 switch (srp->srp_id) {
495 case SKMEM_REGION_UMD:
496 case SKMEM_REGION_KMD:
497 case SKMEM_REGION_RXKMD:
498 case SKMEM_REGION_TXKMD:
499 srp->srp_r_seg_size = skmem_md_seg_size;
500 break;
501
502 case SKMEM_REGION_BUF_DEF:
503 case SKMEM_REGION_RXBUF_DEF:
504 case SKMEM_REGION_TXBUF_DEF:
505 /*
506 * Use the effective driver buffer segment size,
507 * since it reflects any randomization done at
508 * skmem_region_init() time.
509 */
510 srp->srp_r_seg_size = skmem_drv_buf_seg_eff_size;
511 break;
512
513 default:
514 srp->srp_r_seg_size = skmem_seg_size;
515 break;
516 }
517 } else {
518 srp->srp_r_seg_size = (uint32_t)P2ROUNDUP(srp->srp_r_seg_size,
519 SKMEM_MIN_SEG_SIZE);
520 }
521
522 seglim = srp->srp_r_seg_size;
523 VERIFY(seglim != 0 && (seglim % SKMEM_PAGE_SIZE) == 0);
524
525 SK_DF(SK_VERB_MEM, "%s: seglim %zu objsize %zu objcnt %zu",
526 srp->srp_name, seglim, objsize, objcnt);
527
528 /*
529 * Make sure object size is multiple of CPU cache line
530 * size, and that we can evenly divide the segment size.
531 */
532 if (!((objsize < cache_line_size) && (objsize < seglim) &&
533 ((cache_line_size % objsize) == 0) && ((seglim % objsize) == 0))) {
534 objsize = P2ROUNDUP(objsize, cache_line_size);
535 while (objsize < seglim && (seglim % objsize) != 0) {
536 SK_DF(SK_VERB_MEM, "%s: objsize %zu -> %zu",
537 srp->srp_name, objsize, objsize + cache_line_size);
538 objsize += cache_line_size;
539 }
540 }
541
542 /* segment must be larger than object */
543 while (objsize > seglim) {
544 SK_DF(SK_VERB_MEM, "%s: seglim %zu -> %zu", srp->srp_name,
545 seglim, seglim + SKMEM_MIN_SEG_SIZE);
546 seglim += SKMEM_MIN_SEG_SIZE;
547 }
548
549 /*
550 * Take into account worst-case per-CPU cached
551 * objects if this region is configured for it.
552 */
553 if (!(srp->srp_cflags & SKMEM_REGION_CR_NOMAGAZINES)) {
554 uint32_t magazine_max_objs =
555 skmem_cache_magazine_max((uint32_t)objsize);
556 SK_DF(SK_VERB_MEM, "%s: objcnt %zu -> %zu", srp->srp_name,
557 objcnt, objcnt + magazine_max_objs);
558 objcnt += magazine_max_objs;
559 }
560
561 SK_DF(SK_VERB_MEM, "%s: seglim %zu objsize %zu "
562 "objcnt %zu", srp->srp_name, seglim, objsize, objcnt);
563
564 segsize = P2ROUNDUP(objsize * objcnt, SKMEM_MIN_SEG_SIZE);
565 if (seglim > segsize) {
566 /*
567 * If the segment limit is larger than what we need,
568 * avoid memory wastage by shrinking it.
569 */
570 while (seglim > segsize && seglim > SKMEM_MIN_SEG_SIZE) {
571 VERIFY(seglim >= SKMEM_MIN_SEG_SIZE);
572 SK_DF(SK_VERB_MEM,
573 "%s: segsize %zu (%zu*%zu) seglim [-] %zu -> %zu",
574 srp->srp_name, segsize, objsize, objcnt, seglim,
575 P2ROUNDUP(seglim - SKMEM_MIN_SEG_SIZE,
576 SKMEM_MIN_SEG_SIZE));
577 seglim = P2ROUNDUP(seglim - SKMEM_MIN_SEG_SIZE,
578 SKMEM_MIN_SEG_SIZE);
579 }
580
581 /* adjust segment size */
582 segsize = seglim;
583 } else if (seglim < segsize) {
584 size_t oseglim = seglim;
585 /*
586 * If the segment limit is less than the segment size,
587 * see if increasing it slightly (up to 1.5x the segment
588 * size) would allow us to avoid allocating too many
589 * extra objects (due to excessive segment count).
590 */
591 while (seglim < segsize && (segsize % seglim) != 0) {
592 SK_DF(SK_VERB_MEM,
593 "%s: segsize %zu (%zu*%zu) seglim [+] %zu -> %zu",
594 srp->srp_name, segsize, objsize, objcnt, seglim,
595 (seglim + SKMEM_MIN_SEG_SIZE));
596 seglim += SKMEM_MIN_SEG_SIZE;
597 if (seglim >= (oseglim + (oseglim >> 1))) {
598 break;
599 }
600 }
601
602 /* can't use P2ROUNDUP since seglim may not be power of 2 */
603 segsize = SK_ROUNDUP(segsize, seglim);
604 }
605 ASSERT(segsize != 0 && (segsize % seglim) == 0);
606
607 SK_DF(SK_VERB_MEM, "%s: segsize %zu seglim %zu",
608 srp->srp_name, segsize, seglim);
609
610 /* compute segment count, and recompute segment size */
611 if (srp->srp_cflags & SKMEM_REGION_CR_MONOLITHIC) {
612 segcnt = 1;
613 } else {
614 /*
615 * The adjustments above were done in increments of
616 * SKMEM_MIN_SEG_SIZE. If the object size is greater
617 * than that, ensure that the segment size is a multiple
618 * of the object size.
619 */
620 if (objsize > SKMEM_MIN_SEG_SIZE) {
621 ASSERT(seglim >= objsize);
622 if ((seglim % objsize) != 0) {
623 seglim += (seglim - objsize);
624 }
625 /* recompute segsize; see SK_ROUNDUP comment above */
626 segsize = SK_ROUNDUP(segsize, seglim);
627 }
628
629 segcnt = MAX(1, (segsize / seglim));
630 segsize /= segcnt;
631 }
632
633 SK_DF(SK_VERB_MEM, "%s: segcnt %zu segsize %zu",
634 srp->srp_name, segcnt, segsize);
635
636 /* recompute object count to avoid wastage */
637 objcnt = (segsize * segcnt) / objsize;
638 ASSERT(objcnt != 0);
639 done:
640 srp->srp_c_obj_size = (uint32_t)objsize;
641 srp->srp_c_obj_cnt = (uint32_t)objcnt;
642 srp->srp_c_seg_size = (uint32_t)segsize;
643 srp->srp_seg_cnt = (uint32_t)segcnt;
644
645 SK_DF(SK_VERB_MEM, "%s: objsize %zu objcnt %zu segcnt %zu segsize %zu",
646 srp->srp_name, objsize, objcnt, segcnt, segsize);
647
648 #if SK_LOG
649 if (__improbable(sk_verbose != 0)) {
650 char label[32];
651 (void) snprintf(label, sizeof(label), "REGION_%s:",
652 skmem_region_id2name(srp->srp_id));
653 SK_D("%-16s o:[%4u x %6u -> %4u x %6u]", label,
654 (uint32_t)srp->srp_r_obj_cnt,
655 (uint32_t)srp->srp_r_obj_size,
656 (uint32_t)srp->srp_c_obj_cnt,
657 (uint32_t)srp->srp_c_obj_size);
658 }
659 #endif /* SK_LOG */
660 }
661
662 /*
663 * Create a region.
664 */
665 struct skmem_region *
skmem_region_create(const char * name,struct skmem_region_params * srp,sksegment_ctor_fn_t ctor,sksegment_dtor_fn_t dtor,void * private)666 skmem_region_create(const char *name, struct skmem_region_params *srp,
667 sksegment_ctor_fn_t ctor, sksegment_dtor_fn_t dtor, void *private)
668 {
669 boolean_t pseudo = (srp->srp_cflags & SKMEM_REGION_CR_PSEUDO);
670 uint32_t cflags = srp->srp_cflags;
671 struct skmem_region *skr;
672 uint32_t i;
673
674 ASSERT(srp->srp_id < SKMEM_REGIONS);
675 ASSERT(srp->srp_c_seg_size != 0 &&
676 (pseudo || (srp->srp_c_seg_size % SKMEM_PAGE_SIZE) == 0));
677 ASSERT(srp->srp_seg_cnt != 0);
678 ASSERT(srp->srp_c_obj_cnt == 1 ||
679 (srp->srp_c_seg_size % srp->srp_c_obj_size) == 0);
680 ASSERT(srp->srp_c_obj_size <= srp->srp_c_seg_size);
681
682 skr = zalloc_flags(skr_zone, Z_WAITOK | Z_ZERO);
683 skr->skr_params.srp_r_seg_size = srp->srp_r_seg_size;
684 skr->skr_seg_size = srp->srp_c_seg_size;
685 skr->skr_size = (srp->srp_c_seg_size * srp->srp_seg_cnt);
686 skr->skr_seg_objs = (srp->srp_c_seg_size / srp->srp_c_obj_size);
687
688 if (!pseudo) {
689 skr->skr_seg_max_cnt = srp->srp_seg_cnt;
690
691 /* set alignment to CPU cache line size */
692 skr->skr_params.srp_align = skmem_cpu_cache_line_size();
693
694 /* allocate the allocated-address hash chain */
695 skr->skr_hash_initial = SKMEM_REGION_HASH_INITIAL;
696 skr->skr_hash_limit = SKMEM_REGION_HASH_LIMIT;
697 skr->skr_hash_table = sk_alloc_type_array(struct sksegment_bkt,
698 skr->skr_hash_initial, Z_WAITOK | Z_NOFAIL,
699 skmem_tag_segment_hash);
700 skr->skr_hash_mask = (skr->skr_hash_initial - 1);
701 skr->skr_hash_shift = flsll(srp->srp_c_seg_size) - 1;
702
703 for (i = 0; i < (skr->skr_hash_mask + 1); i++) {
704 TAILQ_INIT(&skr->skr_hash_table[i].sgb_head);
705 }
706 } else {
707 /* this upper bound doesn't apply */
708 skr->skr_seg_max_cnt = 0;
709
710 /* pick up value set by skmem_regions_params_config() */
711 skr->skr_params.srp_align = srp->srp_align;
712 }
713
714 skr->skr_r_obj_size = srp->srp_r_obj_size;
715 skr->skr_r_obj_cnt = srp->srp_r_obj_cnt;
716 skr->skr_c_obj_size = srp->srp_c_obj_size;
717 skr->skr_c_obj_cnt = srp->srp_c_obj_cnt;
718
719 skr->skr_params.srp_md_type = srp->srp_md_type;
720 skr->skr_params.srp_md_subtype = srp->srp_md_subtype;
721 skr->skr_params.srp_max_frags = srp->srp_max_frags;
722
723 skr->skr_seg_ctor = ctor;
724 skr->skr_seg_dtor = dtor;
725 skr->skr_private = private;
726
727 lck_mtx_init(&skr->skr_lock, &skmem_region_lock_grp,
728 &skmem_region_lock_attr);
729
730 TAILQ_INIT(&skr->skr_seg_free);
731 RB_INIT(&skr->skr_seg_tfree);
732
733 skr->skr_id = srp->srp_id;
734 uuid_generate_random(skr->skr_uuid);
735 (void) snprintf(skr->skr_name, sizeof(skr->skr_name),
736 "%s.%s.%s", SKMEM_REGION_PREFIX, srp->srp_name, name);
737
738 SK_DF(SK_VERB_MEM_REGION, "\"%s\": skr 0x%llx ",
739 skr->skr_name, SK_KVA(skr));
740
741 /* sanity check */
742 ASSERT(!(cflags & SKMEM_REGION_CR_GUARD) ||
743 !(cflags & (SKMEM_REGION_CR_KREADONLY | SKMEM_REGION_CR_UREADONLY |
744 SKMEM_REGION_CR_PERSISTENT | SKMEM_REGION_CR_SHAREOK |
745 SKMEM_REGION_CR_IODIR_IN | SKMEM_REGION_CR_IODIR_OUT |
746 SKMEM_REGION_CR_PUREDATA)));
747
748 skr->skr_cflags = cflags;
749 if (cflags & SKMEM_REGION_CR_NOREDIRECT) {
750 skr->skr_mode |= SKR_MODE_NOREDIRECT;
751 }
752 if (cflags & SKMEM_REGION_CR_MMAPOK) {
753 skr->skr_mode |= SKR_MODE_MMAPOK;
754 }
755 if ((cflags & SKMEM_REGION_CR_MMAPOK) &&
756 (cflags & SKMEM_REGION_CR_UREADONLY)) {
757 skr->skr_mode |= SKR_MODE_UREADONLY;
758 }
759 if (cflags & SKMEM_REGION_CR_KREADONLY) {
760 skr->skr_mode |= SKR_MODE_KREADONLY;
761 }
762 if (cflags & SKMEM_REGION_CR_PERSISTENT) {
763 skr->skr_mode |= SKR_MODE_PERSISTENT;
764 }
765 if (cflags & SKMEM_REGION_CR_MONOLITHIC) {
766 skr->skr_mode |= SKR_MODE_MONOLITHIC;
767 }
768 if (cflags & SKMEM_REGION_CR_NOMAGAZINES) {
769 skr->skr_mode |= SKR_MODE_NOMAGAZINES;
770 }
771 if (cflags & SKMEM_REGION_CR_NOCACHE) {
772 skr->skr_mode |= SKR_MODE_NOCACHE;
773 }
774 if (cflags & SKMEM_REGION_CR_SEGPHYSCONTIG) {
775 skr->skr_mode |= SKR_MODE_SEGPHYSCONTIG;
776 }
777 if (cflags & SKMEM_REGION_CR_SHAREOK) {
778 skr->skr_mode |= SKR_MODE_SHAREOK;
779 }
780 if (cflags & SKMEM_REGION_CR_IODIR_IN) {
781 skr->skr_mode |= SKR_MODE_IODIR_IN;
782 }
783 if (cflags & SKMEM_REGION_CR_IODIR_OUT) {
784 skr->skr_mode |= SKR_MODE_IODIR_OUT;
785 }
786 if (cflags & SKMEM_REGION_CR_GUARD) {
787 skr->skr_mode |= SKR_MODE_GUARD;
788 }
789 if (cflags & SKMEM_REGION_CR_PUREDATA) {
790 skr->skr_mode |= SKR_MODE_PUREDATA;
791 }
792 if (cflags & SKMEM_REGION_CR_PSEUDO) {
793 skr->skr_mode |= SKR_MODE_PSEUDO;
794 }
795 if (cflags & SKMEM_REGION_CR_THREADSAFE) {
796 skr->skr_mode |= SKR_MODE_THREADSAFE;
797 }
798
799 #if XNU_TARGET_OS_OSX
800 /*
801 * Mark all regions as persistent except for the guard and Intrinsic
802 * regions.
803 * This is to ensure that kernel threads won't be faulting-in while
804 * accessing these memory regions. We have observed various kinds of
805 * kernel panics due to kernel threads faulting on non-wired memory
806 * access when the VM subsystem is not in a state to swap-in the page.
807 */
808 if (!((skr->skr_mode & SKR_MODE_PSEUDO) ||
809 (skr->skr_mode & SKR_MODE_GUARD))) {
810 skr->skr_mode |= SKR_MODE_PERSISTENT;
811 }
812 #endif /* XNU_TARGET_OS_OSX */
813
814 /* SKR_MODE_UREADONLY only takes effect for user task mapping */
815 skr->skr_bufspec.user_writable = !(skr->skr_mode & SKR_MODE_UREADONLY);
816 skr->skr_bufspec.kernel_writable = !(skr->skr_mode & SKR_MODE_KREADONLY);
817 skr->skr_bufspec.purgeable = TRUE;
818 skr->skr_bufspec.inhibitCache = !!(skr->skr_mode & SKR_MODE_NOCACHE);
819 skr->skr_bufspec.physcontig = (skr->skr_mode & SKR_MODE_SEGPHYSCONTIG);
820 skr->skr_bufspec.iodir_in = !!(skr->skr_mode & SKR_MODE_IODIR_IN);
821 skr->skr_bufspec.iodir_out = !!(skr->skr_mode & SKR_MODE_IODIR_OUT);
822 skr->skr_bufspec.puredata = !!(skr->skr_mode & SKR_MODE_PUREDATA);
823 skr->skr_bufspec.threadSafe = !!(skr->skr_mode & SKR_MODE_THREADSAFE);
824 skr->skr_regspec.noRedirect = !!(skr->skr_mode & SKR_MODE_NOREDIRECT);
825
826 /* allocate segment bitmaps */
827 if (!(skr->skr_mode & SKR_MODE_PSEUDO)) {
828 ASSERT(skr->skr_seg_max_cnt != 0);
829 skr->skr_seg_bmap_len = BITMAP_LEN(skr->skr_seg_max_cnt);
830 skr->skr_seg_bmap = sk_alloc_data(BITMAP_SIZE(skr->skr_seg_max_cnt),
831 Z_WAITOK | Z_NOFAIL, skmem_tag_segment_bmap);
832 ASSERT(BITMAP_SIZE(skr->skr_seg_max_cnt) ==
833 (skr->skr_seg_bmap_len * sizeof(*skr->skr_seg_bmap)));
834
835 /* mark all bitmaps as free (bit set) */
836 bitmap_full(skr->skr_seg_bmap, skr->skr_seg_max_cnt);
837 }
838
839 /*
840 * Populate the freelist by allocating all segments for the
841 * region, which will be mapped but not faulted-in, and then
842 * immediately insert each to the freelist. That will in
843 * turn unmap the segment's memory object.
844 */
845 SKR_LOCK(skr);
846 if (skr->skr_mode & SKR_MODE_PSEUDO) {
847 char zone_name[64];
848 (void) snprintf(zone_name, sizeof(zone_name), "%s.reg.%s",
849 SKMEM_ZONE_PREFIX, name);
850 skr->skr_zreg = zone_create(zone_name, skr->skr_c_obj_size,
851 ZC_ZFREE_CLEARMEM | ZC_DESTRUCTIBLE);
852 } else {
853 /* create a backing IOSKRegion object */
854 if ((skr->skr_reg = IOSKRegionCreate(&skr->skr_regspec,
855 (IOSKSize)skr->skr_seg_size,
856 (IOSKCount)skr->skr_seg_max_cnt)) == NULL) {
857 SK_ERR("\%s\": [%u * %u] cflags 0x%b skr_reg failed",
858 skr->skr_name, (uint32_t)skr->skr_seg_size,
859 (uint32_t)skr->skr_seg_max_cnt, skr->skr_cflags,
860 SKMEM_REGION_CR_BITS);
861 goto failed;
862 }
863 }
864
865 ASSERT(skr->skr_seg_objs != 0);
866
867 ++skr->skr_refcnt; /* for caller */
868 SKR_UNLOCK(skr);
869
870 SKMEM_REGION_LOCK();
871 TAILQ_INSERT_TAIL(&skmem_region_head, skr, skr_link);
872 SKMEM_REGION_UNLOCK();
873
874 SK_DF(SK_VERB_MEM_REGION,
875 " [TOTAL] seg (%u*%u) obj (%u*%u) cflags 0x%b",
876 (uint32_t)skr->skr_seg_size, (uint32_t)skr->skr_seg_max_cnt,
877 (uint32_t)skr->skr_c_obj_size, (uint32_t)skr->skr_c_obj_cnt,
878 skr->skr_cflags, SKMEM_REGION_CR_BITS);
879
880 return skr;
881
882 failed:
883 SKR_LOCK_ASSERT_HELD(skr);
884 skmem_region_destroy(skr);
885
886 return NULL;
887 }
888
889 /*
890 * Destroy a region.
891 */
892 static void
skmem_region_destroy(struct skmem_region * skr)893 skmem_region_destroy(struct skmem_region *skr)
894 {
895 struct skmem_region *mskr;
896
897 SKR_LOCK_ASSERT_HELD(skr);
898
899 SK_DF(SK_VERB_MEM_REGION, "\"%s\": skr 0x%llx",
900 skr->skr_name, SK_KVA(skr));
901
902 /*
903 * Panic if we detect there are unfreed segments; the caller
904 * destroying this region is responsible for ensuring that all
905 * allocated segments have been freed prior to getting here.
906 */
907 ASSERT(skr->skr_refcnt == 0);
908 if (skr->skr_seginuse != 0) {
909 panic("%s: '%s' (%p) not empty (%u unfreed)",
910 __func__, skr->skr_name, (void *)skr, skr->skr_seginuse);
911 /* NOTREACHED */
912 __builtin_unreachable();
913 }
914
915 if (skr->skr_link.tqe_next != NULL || skr->skr_link.tqe_prev != NULL) {
916 SKR_UNLOCK(skr);
917 SKMEM_REGION_LOCK();
918 TAILQ_REMOVE(&skmem_region_head, skr, skr_link);
919 SKMEM_REGION_UNLOCK();
920 SKR_LOCK(skr);
921 ASSERT(skr->skr_refcnt == 0);
922 }
923
924 /*
925 * Undo what's done earlier at region creation time.
926 */
927 skmem_region_depopulate(skr);
928 ASSERT(TAILQ_EMPTY(&skr->skr_seg_free));
929 ASSERT(RB_EMPTY(&skr->skr_seg_tfree));
930 ASSERT(skr->skr_seg_free_cnt == 0);
931
932 if (skr->skr_reg != NULL) {
933 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
934 IOSKRegionDestroy(skr->skr_reg);
935 skr->skr_reg = NULL;
936 }
937
938 if (skr->skr_zreg != NULL) {
939 ASSERT(skr->skr_mode & SKR_MODE_PSEUDO);
940 zdestroy(skr->skr_zreg);
941 skr->skr_zreg = NULL;
942 }
943
944 if (skr->skr_seg_bmap != NULL) {
945 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
946 #if (DEBUG || DEVELOPMENT)
947 ASSERT(skr->skr_seg_bmap_len != 0);
948 /* must have been set to vacant (bit set) by now */
949 assert(bitmap_is_full(skr->skr_seg_bmap, skr->skr_seg_max_cnt));
950 #endif /* DEBUG || DEVELOPMENT */
951
952 sk_free_data(skr->skr_seg_bmap, BITMAP_SIZE(skr->skr_seg_max_cnt));
953 skr->skr_seg_bmap = NULL;
954 skr->skr_seg_bmap_len = 0;
955 }
956 ASSERT(skr->skr_seg_bmap_len == 0);
957
958 if (skr->skr_hash_table != NULL) {
959 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
960 #if (DEBUG || DEVELOPMENT)
961 for (uint32_t i = 0; i < (skr->skr_hash_mask + 1); i++) {
962 ASSERT(TAILQ_EMPTY(&skr->skr_hash_table[i].sgb_head));
963 }
964 #endif /* DEBUG || DEVELOPMENT */
965
966 sk_free_type_array(struct sksegment_bkt, skr->skr_hash_mask + 1,
967 skr->skr_hash_table);
968 skr->skr_hash_table = NULL;
969 }
970 if ((mskr = skr->skr_mirror) != NULL) {
971 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
972 skr->skr_mirror = NULL;
973 mskr->skr_mode &= ~SKR_MODE_MIRRORED;
974 }
975 SKR_UNLOCK(skr);
976
977 if (mskr != NULL) {
978 skmem_region_release(mskr);
979 }
980
981 lck_mtx_destroy(&skr->skr_lock, &skmem_region_lock_grp);
982
983 zfree(skr_zone, skr);
984 }
985
986 /*
987 * Mirror mskr (slave) to skr (master).
988 */
989 void
skmem_region_mirror(struct skmem_region * skr,struct skmem_region * mskr)990 skmem_region_mirror(struct skmem_region *skr, struct skmem_region *mskr)
991 {
992 SK_DF(SK_VERB_MEM_REGION, "skr master 0x%llx, slave 0x%llx ",
993 SK_KVA(skr), SK_KVA(mskr));
994
995 SKR_LOCK(skr);
996 ASSERT(!(skr->skr_mode & SKR_MODE_MIRRORED));
997 ASSERT(!(mskr->skr_mode & SKR_MODE_MIRRORED));
998 ASSERT(skr->skr_mirror == NULL);
999
1000 /* both regions must share identical parameters */
1001 ASSERT(skr->skr_size == mskr->skr_size);
1002 ASSERT(skr->skr_seg_size == mskr->skr_seg_size);
1003 ASSERT(skr->skr_seg_free_cnt == mskr->skr_seg_free_cnt);
1004
1005 skr->skr_mirror = mskr;
1006 skmem_region_retain(mskr);
1007 mskr->skr_mode |= SKR_MODE_MIRRORED;
1008 SKR_UNLOCK(skr);
1009 }
1010
1011 void
skmem_region_slab_config(struct skmem_region * skr,struct skmem_cache * skm,bool attach)1012 skmem_region_slab_config(struct skmem_region *skr, struct skmem_cache *skm,
1013 bool attach)
1014 {
1015 int i;
1016
1017 SKR_LOCK(skr);
1018 if (attach) {
1019 for (i = 0; i < SKR_MAX_CACHES && skr->skr_cache[i] != NULL;
1020 i++) {
1021 ;
1022 }
1023 VERIFY(i < SKR_MAX_CACHES);
1024 ASSERT(skr->skr_cache[i] == NULL);
1025 skr->skr_mode |= SKR_MODE_SLAB;
1026 skr->skr_cache[i] = skm;
1027 skmem_region_retain_locked(skr);
1028 SKR_UNLOCK(skr);
1029 } else {
1030 ASSERT(skr->skr_mode & SKR_MODE_SLAB);
1031 for (i = 0; i < SKR_MAX_CACHES && skr->skr_cache[i] != skm;
1032 i++) {
1033 ;
1034 }
1035 VERIFY(i < SKR_MAX_CACHES);
1036 ASSERT(skr->skr_cache[i] == skm);
1037 skr->skr_cache[i] = NULL;
1038 for (i = 0; i < SKR_MAX_CACHES && skr->skr_cache[i] == NULL;
1039 i++) {
1040 ;
1041 }
1042 if (i == SKR_MAX_CACHES) {
1043 skr->skr_mode &= ~SKR_MODE_SLAB;
1044 }
1045 if (!skmem_region_release_locked(skr)) {
1046 SKR_UNLOCK(skr);
1047 }
1048 }
1049 }
1050
1051 /*
1052 * Common routines for skmem_region_{alloc,mirror_alloc}.
1053 */
1054 static void *
skmem_region_alloc_common(struct skmem_region * skr,struct sksegment * sg)1055 skmem_region_alloc_common(struct skmem_region *skr, struct sksegment *sg)
1056 {
1057 struct sksegment_bkt *sgb;
1058 void *addr;
1059
1060 SKR_LOCK_ASSERT_HELD(skr);
1061
1062 ASSERT(sg->sg_md != NULL);
1063 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1064 addr = (void *)sg->sg_start;
1065 sgb = SKMEM_REGION_HASH(skr, addr);
1066 ASSERT(sg->sg_link.tqe_next == NULL);
1067 ASSERT(sg->sg_link.tqe_prev == NULL);
1068 TAILQ_INSERT_HEAD(&sgb->sgb_head, sg, sg_link);
1069
1070 skr->skr_seginuse++;
1071 skr->skr_meminuse += skr->skr_seg_size;
1072 if (sg->sg_state == SKSEG_STATE_MAPPED_WIRED) {
1073 skr->skr_w_meminuse += skr->skr_seg_size;
1074 }
1075 skr->skr_alloc++;
1076
1077 return addr;
1078 }
1079
1080 /*
1081 * Allocate a segment from the region.
1082 */
1083 void *
skmem_region_alloc(struct skmem_region * skr,void ** maddr,struct sksegment ** retsg,struct sksegment ** retsgm,uint32_t skmflag)1084 skmem_region_alloc(struct skmem_region *skr, void **maddr,
1085 struct sksegment **retsg, struct sksegment **retsgm, uint32_t skmflag)
1086 {
1087 struct sksegment *sg = NULL;
1088 struct sksegment *sg1 = NULL;
1089 void *addr = NULL, *addr1 = NULL;
1090 uint32_t retries = 0;
1091
1092 VERIFY(!(skr->skr_mode & SKR_MODE_GUARD));
1093
1094 if (retsg != NULL) {
1095 *retsg = NULL;
1096 }
1097 if (retsgm != NULL) {
1098 *retsgm = NULL;
1099 }
1100
1101 /* SKMEM_NOSLEEP and SKMEM_FAILOK are mutually exclusive */
1102 VERIFY((skmflag & (SKMEM_NOSLEEP | SKMEM_FAILOK)) !=
1103 (SKMEM_NOSLEEP | SKMEM_FAILOK));
1104
1105 SKR_LOCK(skr);
1106 while (sg == NULL) {
1107 /* see if there's a segment in the freelist */
1108 sg = TAILQ_FIRST(&skr->skr_seg_free);
1109 if (sg == NULL) {
1110 /* see if we can grow the freelist */
1111 sg = sksegment_freelist_grow(skr);
1112 if (sg != NULL) {
1113 break;
1114 }
1115
1116 if (skr->skr_mode & SKR_MODE_SLAB) {
1117 SKR_UNLOCK(skr);
1118 /*
1119 * None found; it's possible that the slab
1120 * layer is caching extra amount, so ask
1121 * skmem_cache to reap/purge its caches.
1122 */
1123 for (int i = 0; i < SKR_MAX_CACHES; i++) {
1124 if (skr->skr_cache[i] == NULL) {
1125 continue;
1126 }
1127 skmem_cache_reap_now(skr->skr_cache[i],
1128 TRUE);
1129 }
1130 SKR_LOCK(skr);
1131 /*
1132 * If we manage to get some freed, try again.
1133 */
1134 if (TAILQ_FIRST(&skr->skr_seg_free) != NULL) {
1135 continue;
1136 }
1137 }
1138
1139 /*
1140 * Give up if this is a non-blocking allocation,
1141 * or if this is a blocking allocation but the
1142 * caller is willing to retry.
1143 */
1144 if (skmflag & (SKMEM_NOSLEEP | SKMEM_FAILOK)) {
1145 break;
1146 }
1147
1148 /* otherwise we wait until one is available */
1149 ++skr->skr_seg_waiters;
1150 (void) msleep(&skr->skr_seg_free, &skr->skr_lock,
1151 (PZERO - 1), skr->skr_name, NULL);
1152 }
1153 }
1154
1155 SKR_LOCK_ASSERT_HELD(skr);
1156
1157 if (sg != NULL) {
1158 retry:
1159 /*
1160 * We have a segment; remove it from the freelist and
1161 * insert it into the allocated-address hash chain.
1162 * Note that this may return NULL if we can't allocate
1163 * the memory descriptor.
1164 */
1165 if (sksegment_freelist_remove(skr, sg, skmflag,
1166 FALSE) == NULL) {
1167 ASSERT(sg->sg_state == SKSEG_STATE_DETACHED);
1168 ASSERT(sg->sg_md == NULL);
1169 ASSERT(sg->sg_start == 0 && sg->sg_end == 0);
1170
1171 /*
1172 * If it's non-blocking allocation, simply just give
1173 * up and let the caller decide when to retry. Else,
1174 * it gets a bit complicated due to the contract we
1175 * have for blocking allocations with the client; the
1176 * most sensible thing to do here is to retry the
1177 * allocation ourselves. Note that we keep using the
1178 * same segment we originally got, since we only need
1179 * the memory descriptor to be allocated for it; thus
1180 * we make sure we don't release the region lock when
1181 * retrying allocation. Doing so is crucial when the
1182 * region is mirrored, since the segment indices on
1183 * both regions need to match.
1184 */
1185 if (skmflag & SKMEM_NOSLEEP) {
1186 SK_ERR("\"%s\": failed to allocate segment "
1187 "(non-sleeping mode)", skr->skr_name);
1188 sg = NULL;
1189 } else {
1190 if (++retries > SKMEM_WDT_MAXTIME) {
1191 panic_plain("\"%s\": failed to "
1192 "allocate segment (sleeping mode) "
1193 "after %u retries\n\n%s",
1194 skr->skr_name, SKMEM_WDT_MAXTIME,
1195 skmem_dump(skr));
1196 /* NOTREACHED */
1197 __builtin_unreachable();
1198 } else {
1199 SK_ERR("\"%s\": failed to allocate "
1200 "segment (sleeping mode): %u "
1201 "retries", skr->skr_name, retries);
1202 }
1203 if (skr->skr_mode & SKR_MODE_SLAB) {
1204 /*
1205 * We can't get any memory descriptor
1206 * for this segment; reap extra cached
1207 * objects from the slab layer and hope
1208 * that we get lucky next time around.
1209 *
1210 * XXX [email protected]: perhaps also
1211 * trigger the zone allocator to do
1212 * its garbage collection here?
1213 */
1214 skmem_cache_reap();
1215 }
1216 delay(1 * USEC_PER_SEC); /* 1 sec */
1217 goto retry;
1218 }
1219 }
1220
1221 if (sg != NULL) {
1222 /* insert to allocated-address hash chain */
1223 addr = skmem_region_alloc_common(skr, sg);
1224 }
1225 }
1226
1227 if (sg == NULL) {
1228 VERIFY(skmflag & (SKMEM_NOSLEEP | SKMEM_FAILOK));
1229 if (skmflag & SKMEM_PANIC) {
1230 VERIFY((skmflag & (SKMEM_NOSLEEP | SKMEM_FAILOK)) ==
1231 SKMEM_NOSLEEP);
1232 /*
1233 * If is a failed non-blocking alloc and the caller
1234 * insists that it must be successful, then panic.
1235 */
1236 panic_plain("\"%s\": skr 0x%p unable to satisfy "
1237 "mandatory allocation\n", skr->skr_name, skr);
1238 /* NOTREACHED */
1239 __builtin_unreachable();
1240 } else {
1241 /*
1242 * Give up if this is a non-blocking allocation,
1243 * or one where the caller is willing to handle
1244 * allocation failures.
1245 */
1246 goto done;
1247 }
1248 }
1249
1250 ASSERT((mach_vm_address_t)addr == sg->sg_start);
1251
1252 #if SK_LOG
1253 SK_DF(SK_VERB_MEM_REGION, "skr 0x%llx sg 0x%llx",
1254 SK_KVA(skr), SK_KVA(sg));
1255 if (skr->skr_mirror == NULL ||
1256 !(skr->skr_mirror->skr_mode & SKR_MODE_MIRRORED)) {
1257 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx)",
1258 sg->sg_index, SK_KVA(sg->sg_start), SK_KVA(sg->sg_end));
1259 } else {
1260 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx) mirrored",
1261 sg->sg_index, SK_KVA(sg), SK_KVA(sg->sg_start),
1262 SK_KVA(sg->sg_end));
1263 }
1264 #endif /* SK_LOG */
1265
1266 /*
1267 * If mirroring, allocate shadow object from slave region.
1268 */
1269 if (skr->skr_mirror != NULL) {
1270 ASSERT(skr->skr_mirror != skr);
1271 ASSERT(!(skr->skr_mode & SKR_MODE_MIRRORED));
1272 ASSERT(skr->skr_mirror->skr_mode & SKR_MODE_MIRRORED);
1273 addr1 = skmem_region_mirror_alloc(skr->skr_mirror, sg, &sg1);
1274 ASSERT(addr1 != NULL);
1275 ASSERT(sg1 != NULL && sg1 != sg);
1276 ASSERT(sg1->sg_index == sg->sg_index);
1277 }
1278
1279 done:
1280 SKR_UNLOCK(skr);
1281
1282 /* return segment metadata to caller if asked (reference not needed) */
1283 if (addr != NULL) {
1284 if (retsg != NULL) {
1285 *retsg = sg;
1286 }
1287 if (retsgm != NULL) {
1288 *retsgm = sg1;
1289 }
1290 }
1291
1292 if (maddr != NULL) {
1293 *maddr = addr1;
1294 }
1295
1296 return addr;
1297 }
1298
1299 /*
1300 * Allocate a segment from a mirror region at the same index. While it
1301 * is somewhat a simplified variant of skmem_region_alloc, keeping it
1302 * separate allows us to avoid further convoluting that routine.
1303 */
1304 static void *
skmem_region_mirror_alloc(struct skmem_region * skr,struct sksegment * sg0,struct sksegment ** retsg)1305 skmem_region_mirror_alloc(struct skmem_region *skr, struct sksegment *sg0,
1306 struct sksegment **retsg)
1307 {
1308 struct sksegment sg_key = { .sg_index = sg0->sg_index };
1309 struct sksegment *sg = NULL;
1310 void *addr = NULL;
1311
1312 ASSERT(skr->skr_mode & SKR_MODE_MIRRORED);
1313 ASSERT(skr->skr_mirror == NULL);
1314 ASSERT(sg0->sg_type == SKSEG_TYPE_ALLOC);
1315
1316 if (retsg != NULL) {
1317 *retsg = NULL;
1318 }
1319
1320 SKR_LOCK(skr);
1321
1322 /*
1323 * See if we can find one in the freelist first. Otherwise,
1324 * create a new segment of the same index and add that to the
1325 * freelist. We would always get a segment since both regions
1326 * are synchronized when it comes to the indices of allocated
1327 * segments.
1328 */
1329 sg = RB_FIND(segtfreehead, &skr->skr_seg_tfree, &sg_key);
1330 if (sg == NULL) {
1331 sg = sksegment_alloc_with_idx(skr, sg0->sg_index);
1332 VERIFY(sg != NULL);
1333 }
1334 VERIFY(sg->sg_index == sg0->sg_index);
1335
1336 /*
1337 * We have a segment; remove it from the freelist and insert
1338 * it into the allocated-address hash chain. This either
1339 * succeeds or panics (SKMEM_PANIC) when a memory descriptor
1340 * can't be allocated.
1341 *
1342 * TODO: consider retrying IOBMD allocation attempts if needed.
1343 */
1344 sg = sksegment_freelist_remove(skr, sg, SKMEM_PANIC, FALSE);
1345 VERIFY(sg != NULL);
1346
1347 /* insert to allocated-address hash chain */
1348 addr = skmem_region_alloc_common(skr, sg);
1349
1350 #if SK_LOG
1351 SK_DF(SK_VERB_MEM_REGION, "skr 0x%llx sg 0x%llx",
1352 SK_KVA(skr), SK_KVA(sg));
1353 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx)",
1354 sg->sg_index, SK_KVA(sg->sg_start), SK_KVA(sg->sg_end));
1355 #endif /* SK_LOG */
1356
1357 SKR_UNLOCK(skr);
1358
1359 /* return segment metadata to caller if asked (reference not needed) */
1360 if (retsg != NULL) {
1361 *retsg = sg;
1362 }
1363
1364 return addr;
1365 }
1366
1367 /*
1368 * Free a segment to the region.
1369 */
1370 void
skmem_region_free(struct skmem_region * skr,void * addr,void * maddr)1371 skmem_region_free(struct skmem_region *skr, void *addr, void *maddr)
1372 {
1373 struct sksegment_bkt *sgb;
1374 struct sksegment *sg, *tsg;
1375
1376 VERIFY(!(skr->skr_mode & SKR_MODE_GUARD));
1377
1378 /*
1379 * Search the hash chain to find a matching segment for the
1380 * given address. If found, remove the segment from the
1381 * hash chain and insert it into the freelist. Otherwise,
1382 * we panic since the caller has given us a bogus address.
1383 */
1384 SKR_LOCK(skr);
1385 sgb = SKMEM_REGION_HASH(skr, addr);
1386 TAILQ_FOREACH_SAFE(sg, &sgb->sgb_head, sg_link, tsg) {
1387 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1388 if (sg->sg_start == (mach_vm_address_t)addr) {
1389 TAILQ_REMOVE(&sgb->sgb_head, sg, sg_link);
1390 sg->sg_link.tqe_next = NULL;
1391 sg->sg_link.tqe_prev = NULL;
1392 break;
1393 }
1394 }
1395
1396 ASSERT(sg != NULL);
1397 if (sg->sg_state == SKSEG_STATE_MAPPED_WIRED) {
1398 ASSERT(skr->skr_w_meminuse >= skr->skr_seg_size);
1399 skr->skr_w_meminuse -= skr->skr_seg_size;
1400 }
1401 sksegment_freelist_insert(skr, sg, FALSE);
1402
1403 ASSERT(skr->skr_seginuse != 0);
1404 skr->skr_seginuse--;
1405 skr->skr_meminuse -= skr->skr_seg_size;
1406 skr->skr_free++;
1407
1408 #if SK_LOG
1409 SK_DF(SK_VERB_MEM_REGION, "skr 0x%llx sg 0x%llx",
1410 SK_KVA(skr), SK_KVA(sg));
1411 if (skr->skr_mirror == NULL ||
1412 !(skr->skr_mirror->skr_mode & SKR_MODE_MIRRORED)) {
1413 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx)",
1414 sg->sg_index, SK_KVA(addr),
1415 SK_KVA((uintptr_t)addr + skr->skr_seg_size));
1416 } else {
1417 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx) mirrored",
1418 sg->sg_index, SK_KVA(sg), SK_KVA(addr),
1419 SK_KVA((uintptr_t)addr + skr->skr_seg_size));
1420 }
1421 #endif /* SK_LOG */
1422
1423 /*
1424 * If mirroring, also free shadow object in slave region.
1425 */
1426 if (skr->skr_mirror != NULL) {
1427 ASSERT(maddr != NULL);
1428 ASSERT(skr->skr_mirror != skr);
1429 ASSERT(!(skr->skr_mode & SKR_MODE_MIRRORED));
1430 ASSERT(skr->skr_mirror->skr_mode & SKR_MODE_MIRRORED);
1431 skmem_region_free(skr->skr_mirror, maddr, NULL);
1432 }
1433
1434 /* wake up any blocked threads waiting for a segment */
1435 if (skr->skr_seg_waiters != 0) {
1436 SK_DF(SK_VERB_MEM_REGION,
1437 "sg 0x%llx waking up %u waiters", SK_KVA(sg),
1438 skr->skr_seg_waiters);
1439 skr->skr_seg_waiters = 0;
1440 wakeup(&skr->skr_seg_free);
1441 }
1442 SKR_UNLOCK(skr);
1443 }
1444
1445 __attribute__((always_inline))
1446 static inline void
skmem_region_retain_locked(struct skmem_region * skr)1447 skmem_region_retain_locked(struct skmem_region *skr)
1448 {
1449 SKR_LOCK_ASSERT_HELD(skr);
1450 skr->skr_refcnt++;
1451 ASSERT(skr->skr_refcnt != 0);
1452 }
1453
1454 /*
1455 * Retain a segment.
1456 */
1457 void
skmem_region_retain(struct skmem_region * skr)1458 skmem_region_retain(struct skmem_region *skr)
1459 {
1460 SKR_LOCK(skr);
1461 skmem_region_retain_locked(skr);
1462 SKR_UNLOCK(skr);
1463 }
1464
1465 __attribute__((always_inline))
1466 static inline boolean_t
skmem_region_release_locked(struct skmem_region * skr)1467 skmem_region_release_locked(struct skmem_region *skr)
1468 {
1469 SKR_LOCK_ASSERT_HELD(skr);
1470 ASSERT(skr->skr_refcnt != 0);
1471 if (--skr->skr_refcnt == 0) {
1472 skmem_region_destroy(skr);
1473 return TRUE;
1474 }
1475 return FALSE;
1476 }
1477
1478 /*
1479 * Release (and potentially destroy) a segment.
1480 */
1481 boolean_t
skmem_region_release(struct skmem_region * skr)1482 skmem_region_release(struct skmem_region *skr)
1483 {
1484 boolean_t lastref;
1485
1486 SKR_LOCK(skr);
1487 if (!(lastref = skmem_region_release_locked(skr))) {
1488 SKR_UNLOCK(skr);
1489 }
1490
1491 return lastref;
1492 }
1493
1494 /*
1495 * Depopulate the segment freelist.
1496 */
1497 static void
skmem_region_depopulate(struct skmem_region * skr)1498 skmem_region_depopulate(struct skmem_region *skr)
1499 {
1500 struct sksegment *sg, *tsg;
1501
1502 SK_DF(SK_VERB_MEM_REGION, "\"%s\": skr 0x%llx ",
1503 skr->skr_name, SK_KVA(skr));
1504
1505 SKR_LOCK_ASSERT_HELD(skr);
1506 ASSERT(skr->skr_seg_bmap_len != 0 || (skr->skr_mode & SKR_MODE_PSEUDO));
1507
1508 TAILQ_FOREACH_SAFE(sg, &skr->skr_seg_free, sg_link, tsg) {
1509 struct sksegment *sg0;
1510 uint32_t i;
1511
1512 i = sg->sg_index;
1513 sg0 = sksegment_freelist_remove(skr, sg, 0, TRUE);
1514 VERIFY(sg0 == sg);
1515
1516 sksegment_destroy(skr, sg);
1517 ASSERT(bit_test(skr->skr_seg_bmap[i / BMAPSZ], i % BMAPSZ));
1518 }
1519 }
1520
1521 /*
1522 * Free tree segment compare routine.
1523 */
1524 static int
sksegment_cmp(const struct sksegment * sg1,const struct sksegment * sg2)1525 sksegment_cmp(const struct sksegment *sg1, const struct sksegment *sg2)
1526 {
1527 return sg1->sg_index - sg2->sg_index;
1528 }
1529
1530 /*
1531 * Create a segment.
1532 *
1533 * Upon success, clear the bit for the segment's index in skr_seg_bmap bitmap.
1534 */
1535 static struct sksegment *
sksegment_create(struct skmem_region * skr,uint32_t i)1536 sksegment_create(struct skmem_region *skr, uint32_t i)
1537 {
1538 struct sksegment *sg = NULL;
1539 bitmap_t *bmap;
1540
1541 SKR_LOCK_ASSERT_HELD(skr);
1542
1543 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
1544 ASSERT(i < skr->skr_seg_max_cnt);
1545 ASSERT(skr->skr_reg != NULL);
1546 ASSERT(skr->skr_seg_size == round_page(skr->skr_seg_size));
1547
1548 bmap = &skr->skr_seg_bmap[i / BMAPSZ];
1549 ASSERT(bit_test(*bmap, i % BMAPSZ));
1550
1551 sg = skmem_cache_alloc(skmem_sg_cache, SKMEM_SLEEP);
1552 bzero(sg, sg_size);
1553
1554 sg->sg_region = skr;
1555 sg->sg_index = i;
1556 sg->sg_state = SKSEG_STATE_DETACHED;
1557
1558 /* claim it (clear bit) */
1559 bit_clear(*bmap, i % BMAPSZ);
1560
1561 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx) 0x%b", i,
1562 SK_KVA(sg->sg_start), SK_KVA(sg->sg_end), skr->skr_mode,
1563 SKR_MODE_BITS);
1564
1565 return sg;
1566 }
1567
1568 /*
1569 * Destroy a segment.
1570 *
1571 * Set the bit for the segment's index in skr_seg_bmap bitmap,
1572 * indicating that it is now vacant.
1573 */
1574 static void
sksegment_destroy(struct skmem_region * skr,struct sksegment * sg)1575 sksegment_destroy(struct skmem_region *skr, struct sksegment *sg)
1576 {
1577 uint32_t i = sg->sg_index;
1578 bitmap_t *bmap;
1579
1580 SKR_LOCK_ASSERT_HELD(skr);
1581
1582 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
1583 ASSERT(skr == sg->sg_region);
1584 ASSERT(skr->skr_reg != NULL);
1585 ASSERT(sg->sg_type == SKSEG_TYPE_DESTROYED);
1586 ASSERT(i < skr->skr_seg_max_cnt);
1587
1588 bmap = &skr->skr_seg_bmap[i / BMAPSZ];
1589 ASSERT(!bit_test(*bmap, i % BMAPSZ));
1590
1591 SK_DF(SK_VERB_MEM_REGION, " [%u] [0x%llx-0x%llx) 0x%b",
1592 i, SK_KVA(sg->sg_start), SK_KVA(sg->sg_end),
1593 skr->skr_mode, SKR_MODE_BITS);
1594
1595 /*
1596 * Undo what's done earlier at segment creation time.
1597 */
1598
1599 ASSERT(sg->sg_md == NULL);
1600 ASSERT(sg->sg_start == 0 && sg->sg_end == 0);
1601 ASSERT(sg->sg_state == SKSEG_STATE_DETACHED);
1602
1603 /* release it (set bit) */
1604 bit_set(*bmap, i % BMAPSZ);
1605
1606 skmem_cache_free(skmem_sg_cache, sg);
1607 }
1608
1609 /*
1610 * Insert a segment into freelist (freeing the segment).
1611 */
1612 static void
sksegment_freelist_insert(struct skmem_region * skr,struct sksegment * sg,boolean_t populating)1613 sksegment_freelist_insert(struct skmem_region *skr, struct sksegment *sg,
1614 boolean_t populating)
1615 {
1616 SKR_LOCK_ASSERT_HELD(skr);
1617
1618 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
1619 ASSERT(sg->sg_type != SKSEG_TYPE_FREE);
1620 ASSERT(skr == sg->sg_region);
1621 ASSERT(skr->skr_reg != NULL);
1622 ASSERT(sg->sg_index < skr->skr_seg_max_cnt);
1623
1624 /*
1625 * If the region is being populated, then we're done.
1626 */
1627 if (__improbable(populating)) {
1628 ASSERT(sg->sg_md == NULL);
1629 ASSERT(sg->sg_start == 0 && sg->sg_end == 0);
1630 ASSERT(sg->sg_state == SKSEG_STATE_DETACHED);
1631 } else {
1632 IOSKMemoryBufferRef md;
1633 IOReturn err;
1634
1635 ASSERT(sg->sg_md != NULL);
1636 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1637
1638 /*
1639 * Let the client remove the memory from IOMMU, and unwire it.
1640 */
1641 if (skr->skr_seg_dtor != NULL) {
1642 skr->skr_seg_dtor(sg, sg->sg_md, skr->skr_private);
1643 }
1644
1645 ASSERT(sg->sg_state == SKSEG_STATE_MAPPED ||
1646 sg->sg_state == SKSEG_STATE_MAPPED_WIRED);
1647
1648 IOSKRegionClearBufferDebug(skr->skr_reg, sg->sg_index, &md);
1649 VERIFY(sg->sg_md == md);
1650
1651 /* if persistent, unwire this memory now */
1652 if (skr->skr_mode & SKR_MODE_PERSISTENT) {
1653 err = IOSKMemoryUnwire(md);
1654 if (err != kIOReturnSuccess) {
1655 panic("Fail to unwire md %p, err %d", md, err);
1656 }
1657 }
1658
1659 /* mark memory as empty/discarded for consistency */
1660 err = IOSKMemoryDiscard(md);
1661 if (err != kIOReturnSuccess) {
1662 panic("Fail to discard md %p, err %d", md, err);
1663 }
1664
1665 IOSKMemoryDestroy(md);
1666 sg->sg_md = NULL;
1667 sg->sg_start = sg->sg_end = 0;
1668 sg->sg_state = SKSEG_STATE_DETACHED;
1669
1670 ASSERT(skr->skr_memtotal >= skr->skr_seg_size);
1671 skr->skr_memtotal -= skr->skr_seg_size;
1672 }
1673
1674 sg->sg_type = SKSEG_TYPE_FREE;
1675 ASSERT(sg->sg_link.tqe_next == NULL);
1676 ASSERT(sg->sg_link.tqe_prev == NULL);
1677 TAILQ_INSERT_TAIL(&skr->skr_seg_free, sg, sg_link);
1678 ASSERT(sg->sg_node.rbe_left == NULL);
1679 ASSERT(sg->sg_node.rbe_right == NULL);
1680 ASSERT(sg->sg_node.rbe_parent == NULL);
1681 RB_INSERT(segtfreehead, &skr->skr_seg_tfree, sg);
1682 ++skr->skr_seg_free_cnt;
1683 ASSERT(skr->skr_seg_free_cnt <= skr->skr_seg_max_cnt);
1684 }
1685
1686 /*
1687 * Remove a segment from the freelist (allocating the segment).
1688 */
1689 static struct sksegment *
sksegment_freelist_remove(struct skmem_region * skr,struct sksegment * sg,uint32_t skmflag,boolean_t purging)1690 sksegment_freelist_remove(struct skmem_region *skr, struct sksegment *sg,
1691 uint32_t skmflag, boolean_t purging)
1692 {
1693 #pragma unused(skmflag)
1694 mach_vm_address_t segstart;
1695 IOReturn err;
1696
1697 SKR_LOCK_ASSERT_HELD(skr);
1698
1699 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
1700 ASSERT(sg != NULL);
1701 ASSERT(skr == sg->sg_region);
1702 ASSERT(skr->skr_reg != NULL);
1703 ASSERT(sg->sg_type == SKSEG_TYPE_FREE);
1704 ASSERT(sg->sg_index < skr->skr_seg_max_cnt);
1705
1706 #if (DEVELOPMENT || DEBUG)
1707 uint64_t mtbf = skmem_region_get_mtbf();
1708 /*
1709 * MTBF doesn't apply when SKMEM_PANIC is set as caller would assert.
1710 */
1711 if (__improbable(mtbf != 0 && !purging &&
1712 (net_uptime_ms() % mtbf) == 0 &&
1713 !(skmflag & SKMEM_PANIC))) {
1714 SK_ERR("skr \"%s\" 0x%llx sg 0x%llx MTBF failure",
1715 skr->skr_name, SK_KVA(skr), SK_KVA(sg));
1716 net_update_uptime();
1717 return NULL;
1718 }
1719 #endif /* (DEVELOPMENT || DEBUG) */
1720
1721 TAILQ_REMOVE(&skr->skr_seg_free, sg, sg_link);
1722 sg->sg_link.tqe_next = NULL;
1723 sg->sg_link.tqe_prev = NULL;
1724 RB_REMOVE(segtfreehead, &skr->skr_seg_tfree, sg);
1725 sg->sg_node.rbe_left = NULL;
1726 sg->sg_node.rbe_right = NULL;
1727 sg->sg_node.rbe_parent = NULL;
1728
1729 ASSERT(skr->skr_seg_free_cnt != 0);
1730 --skr->skr_seg_free_cnt;
1731
1732 /*
1733 * If the region is being depopulated, then we're done.
1734 */
1735 if (__improbable(purging)) {
1736 ASSERT(sg->sg_md == NULL);
1737 ASSERT(sg->sg_start == 0 && sg->sg_end == 0);
1738 ASSERT(sg->sg_state == SKSEG_STATE_DETACHED);
1739 sg->sg_type = SKSEG_TYPE_DESTROYED;
1740 return sg;
1741 }
1742
1743 ASSERT(sg->sg_md == NULL);
1744 ASSERT(sg->sg_start == 0 && sg->sg_end == 0);
1745 ASSERT(sg->sg_state == SKSEG_STATE_DETACHED);
1746
1747 /* created as non-volatile (mapped) upon success */
1748 if ((sg->sg_md = IOSKMemoryBufferCreate(skr->skr_seg_size,
1749 &skr->skr_bufspec, &segstart)) == NULL) {
1750 ASSERT(sg->sg_type == SKSEG_TYPE_FREE);
1751 if (skmflag & SKMEM_PANIC) {
1752 /* if the caller insists for a success then panic */
1753 panic_plain("\"%s\": skr 0x%p sg 0x%p (idx %u) unable "
1754 "to satisfy mandatory allocation\n", skr->skr_name,
1755 skr, sg, sg->sg_index);
1756 /* NOTREACHED */
1757 __builtin_unreachable();
1758 }
1759 /* reinsert this segment to freelist */
1760 ASSERT(sg->sg_link.tqe_next == NULL);
1761 ASSERT(sg->sg_link.tqe_prev == NULL);
1762 TAILQ_INSERT_HEAD(&skr->skr_seg_free, sg, sg_link);
1763 ASSERT(sg->sg_node.rbe_left == NULL);
1764 ASSERT(sg->sg_node.rbe_right == NULL);
1765 ASSERT(sg->sg_node.rbe_parent == NULL);
1766 RB_INSERT(segtfreehead, &skr->skr_seg_tfree, sg);
1767 ++skr->skr_seg_free_cnt;
1768 return NULL;
1769 }
1770
1771 sg->sg_start = segstart;
1772 sg->sg_end = (segstart + skr->skr_seg_size);
1773 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1774
1775 /* mark memory as non-volatile just to be consistent */
1776 err = IOSKMemoryReclaim(sg->sg_md);
1777 if (err != kIOReturnSuccess) {
1778 panic("Fail to reclaim md %p, err %d", sg->sg_md, err);
1779 }
1780
1781 /* if persistent, wire down its memory now */
1782 if (skr->skr_mode & SKR_MODE_PERSISTENT) {
1783 err = IOSKMemoryWire(sg->sg_md);
1784 if (err != kIOReturnSuccess) {
1785 panic("Fail to wire md %p, err %d", sg->sg_md, err);
1786 }
1787 }
1788
1789 err = IOSKRegionSetBuffer(skr->skr_reg, sg->sg_index, sg->sg_md);
1790 if (err != kIOReturnSuccess) {
1791 panic("Fail to set md %p, err %d", sg->sg_md, err);
1792 }
1793
1794 /*
1795 * Let the client wire it and insert to IOMMU, if applicable.
1796 * Try to find out if it's wired and set the right state.
1797 */
1798 if (skr->skr_seg_ctor != NULL) {
1799 skr->skr_seg_ctor(sg, sg->sg_md, skr->skr_private);
1800 }
1801
1802 sg->sg_state = IOSKBufferIsWired(sg->sg_md) ?
1803 SKSEG_STATE_MAPPED_WIRED : SKSEG_STATE_MAPPED;
1804
1805 skr->skr_memtotal += skr->skr_seg_size;
1806
1807 ASSERT(sg->sg_md != NULL);
1808 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1809
1810 sg->sg_type = SKSEG_TYPE_ALLOC;
1811 return sg;
1812 }
1813
1814 /*
1815 * Find the first available index and allocate a segment at that index.
1816 */
1817 static struct sksegment *
sksegment_freelist_grow(struct skmem_region * skr)1818 sksegment_freelist_grow(struct skmem_region *skr)
1819 {
1820 struct sksegment *sg = NULL;
1821 uint32_t i, j, idx;
1822
1823 SKR_LOCK_ASSERT_HELD(skr);
1824
1825 ASSERT(!(skr->skr_mode & SKR_MODE_PSEUDO));
1826 ASSERT(skr->skr_seg_bmap_len != 0);
1827 ASSERT(skr->skr_seg_max_cnt != 0);
1828
1829 for (i = 0; i < skr->skr_seg_bmap_len; i++) {
1830 bitmap_t *bmap, mask;
1831 uint32_t end = (BMAPSZ - 1);
1832
1833 if (i == (skr->skr_seg_bmap_len - 1)) {
1834 end = (skr->skr_seg_max_cnt - 1) % BMAPSZ;
1835 }
1836
1837 bmap = &skr->skr_seg_bmap[i];
1838 mask = BMASK64(0, end);
1839
1840 j = ffsll((*bmap) & mask);
1841 if (j == 0) {
1842 continue;
1843 }
1844
1845 --j;
1846 idx = (i * BMAPSZ) + j;
1847
1848 sg = sksegment_alloc_with_idx(skr, idx);
1849
1850 /* we're done */
1851 break;
1852 }
1853
1854 ASSERT((sg != NULL) || (skr->skr_seginuse == skr->skr_seg_max_cnt));
1855 return sg;
1856 }
1857
1858 /*
1859 * Create a single segment at a specific index and add it to the freelist.
1860 */
1861 static struct sksegment *
sksegment_alloc_with_idx(struct skmem_region * skr,uint32_t idx)1862 sksegment_alloc_with_idx(struct skmem_region *skr, uint32_t idx)
1863 {
1864 struct sksegment *sg;
1865
1866 SKR_LOCK_ASSERT_HELD(skr);
1867
1868 if (!bit_test(skr->skr_seg_bmap[idx / BMAPSZ], idx % BMAPSZ)) {
1869 panic("%s: '%s' (%p) idx %u (out of %u) is already allocated",
1870 __func__, skr->skr_name, (void *)skr, idx,
1871 (skr->skr_seg_max_cnt - 1));
1872 /* NOTREACHED */
1873 __builtin_unreachable();
1874 }
1875
1876 /* must not fail, blocking alloc */
1877 sg = sksegment_create(skr, idx);
1878 VERIFY(sg != NULL);
1879 VERIFY(!bit_test(skr->skr_seg_bmap[idx / BMAPSZ], idx % BMAPSZ));
1880
1881 /* populate the freelist */
1882 sksegment_freelist_insert(skr, sg, TRUE);
1883 ASSERT(sg == TAILQ_LAST(&skr->skr_seg_free, segfreehead));
1884 #if (DEVELOPMENT || DEBUG)
1885 struct sksegment sg_key = { .sg_index = sg->sg_index };
1886 ASSERT(sg == RB_FIND(segtfreehead, &skr->skr_seg_tfree, &sg_key));
1887 #endif /* (DEVELOPMENT || DEBUG) */
1888
1889 SK_DF(SK_VERB_MEM_REGION, "sg %u/%u", (idx + 1), skr->skr_seg_max_cnt);
1890
1891 return sg;
1892 }
1893
1894 /*
1895 * Rescale the regions's allocated-address hash table.
1896 */
1897 static void
skmem_region_hash_rescale(struct skmem_region * skr)1898 skmem_region_hash_rescale(struct skmem_region *skr)
1899 {
1900 struct sksegment_bkt *old_table, *new_table;
1901 size_t old_size, new_size;
1902 uint32_t i, moved = 0;
1903
1904 if (skr->skr_mode & SKR_MODE_PSEUDO) {
1905 ASSERT(skr->skr_hash_table == NULL);
1906 /* this is no-op for pseudo region */
1907 return;
1908 }
1909
1910 ASSERT(skr->skr_hash_table != NULL);
1911 /* insist that we are executing in the update thread call context */
1912 ASSERT(sk_is_region_update_protected());
1913
1914 /*
1915 * To get small average lookup time (lookup depth near 1.0), the hash
1916 * table size should be roughly the same (not necessarily equivalent)
1917 * as the region size.
1918 */
1919 new_size = MAX(skr->skr_hash_initial,
1920 (1 << (flsll(3 * skr->skr_seginuse + 4) - 2)));
1921 new_size = MIN(skr->skr_hash_limit, new_size);
1922 old_size = (skr->skr_hash_mask + 1);
1923
1924 if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) {
1925 return;
1926 }
1927
1928 new_table = sk_alloc_type_array(struct sksegment_bkt, new_size,
1929 Z_NOWAIT, skmem_tag_segment_hash);
1930 if (__improbable(new_table == NULL)) {
1931 return;
1932 }
1933
1934 for (i = 0; i < new_size; i++) {
1935 TAILQ_INIT(&new_table[i].sgb_head);
1936 }
1937
1938 SKR_LOCK(skr);
1939
1940 old_size = (skr->skr_hash_mask + 1);
1941 old_table = skr->skr_hash_table;
1942
1943 skr->skr_hash_mask = (uint32_t)(new_size - 1);
1944 skr->skr_hash_table = new_table;
1945 skr->skr_rescale++;
1946
1947 for (i = 0; i < old_size; i++) {
1948 struct sksegment_bkt *sgb = &old_table[i];
1949 struct sksegment_bkt *new_sgb;
1950 struct sksegment *sg;
1951
1952 while ((sg = TAILQ_FIRST(&sgb->sgb_head)) != NULL) {
1953 TAILQ_REMOVE(&sgb->sgb_head, sg, sg_link);
1954 ASSERT(sg->sg_start != 0 && sg->sg_end != 0);
1955 new_sgb = SKMEM_REGION_HASH(skr, sg->sg_start);
1956 TAILQ_INSERT_TAIL(&new_sgb->sgb_head, sg, sg_link);
1957 ++moved;
1958 }
1959 ASSERT(TAILQ_EMPTY(&sgb->sgb_head));
1960 }
1961
1962 SK_DF(SK_VERB_MEM_REGION,
1963 "skr 0x%llx old_size %u new_size %u [%u moved]", SK_KVA(skr),
1964 (uint32_t)old_size, (uint32_t)new_size, moved);
1965
1966 SKR_UNLOCK(skr);
1967
1968 sk_free_type_array(struct sksegment_bkt, old_size, old_table);
1969 }
1970
1971 /*
1972 * Apply a function to operate on all regions.
1973 */
1974 static void
skmem_region_applyall(void (* func)(struct skmem_region *))1975 skmem_region_applyall(void (*func)(struct skmem_region *))
1976 {
1977 struct skmem_region *skr;
1978
1979 net_update_uptime();
1980
1981 SKMEM_REGION_LOCK();
1982 TAILQ_FOREACH(skr, &skmem_region_head, skr_link) {
1983 func(skr);
1984 }
1985 SKMEM_REGION_UNLOCK();
1986 }
1987
1988 static void
skmem_region_update(struct skmem_region * skr)1989 skmem_region_update(struct skmem_region *skr)
1990 {
1991 SKMEM_REGION_LOCK_ASSERT_HELD();
1992
1993 /* insist that we are executing in the update thread call context */
1994 ASSERT(sk_is_region_update_protected());
1995
1996 SKR_LOCK(skr);
1997 /*
1998 * If there are threads blocked waiting for an available
1999 * segment, wake them up periodically so they can issue
2000 * another skmem_cache_reap() to reclaim resources cached
2001 * by skmem_cache.
2002 */
2003 if (skr->skr_seg_waiters != 0) {
2004 SK_DF(SK_VERB_MEM_REGION,
2005 "waking up %u waiters to reclaim", skr->skr_seg_waiters);
2006 skr->skr_seg_waiters = 0;
2007 wakeup(&skr->skr_seg_free);
2008 }
2009 SKR_UNLOCK(skr);
2010
2011 /*
2012 * Rescale the hash table if needed.
2013 */
2014 skmem_region_hash_rescale(skr);
2015 }
2016
2017 /*
2018 * Thread call callback for update.
2019 */
2020 static void
skmem_region_update_func(thread_call_param_t dummy,thread_call_param_t arg)2021 skmem_region_update_func(thread_call_param_t dummy, thread_call_param_t arg)
2022 {
2023 #pragma unused(dummy, arg)
2024 sk_protect_t protect;
2025
2026 protect = sk_region_update_protect();
2027 skmem_region_applyall(skmem_region_update);
2028 sk_region_update_unprotect(protect);
2029
2030 skmem_dispatch(skmem_region_update_tc, NULL,
2031 (skmem_region_update_interval * NSEC_PER_SEC));
2032 }
2033
2034 boolean_t
skmem_region_for_pp(skmem_region_id_t id)2035 skmem_region_for_pp(skmem_region_id_t id)
2036 {
2037 int i;
2038
2039 for (i = 0; i < SKMEM_PP_REGIONS; i++) {
2040 if (id == skmem_pp_region_ids[i]) {
2041 return TRUE;
2042 }
2043 }
2044 return FALSE;
2045 }
2046
2047 void
skmem_region_get_stats(struct skmem_region * skr,struct sk_stats_region * sreg)2048 skmem_region_get_stats(struct skmem_region *skr, struct sk_stats_region *sreg)
2049 {
2050 bzero(sreg, sizeof(*sreg));
2051
2052 (void) snprintf(sreg->sreg_name, sizeof(sreg->sreg_name),
2053 "%s", skr->skr_name);
2054 uuid_copy(sreg->sreg_uuid, skr->skr_uuid);
2055 sreg->sreg_id = (sk_stats_region_id_t)skr->skr_id;
2056 sreg->sreg_mode = skr->skr_mode;
2057
2058 sreg->sreg_r_seg_size = skr->skr_params.srp_r_seg_size;
2059 sreg->sreg_c_seg_size = skr->skr_seg_size;
2060 sreg->sreg_seg_cnt = skr->skr_seg_max_cnt;
2061 sreg->sreg_seg_objs = skr->skr_seg_objs;
2062 sreg->sreg_r_obj_size = skr->skr_r_obj_size;
2063 sreg->sreg_r_obj_cnt = skr->skr_r_obj_cnt;
2064 sreg->sreg_c_obj_size = skr->skr_c_obj_size;
2065 sreg->sreg_c_obj_cnt = skr->skr_c_obj_cnt;
2066 sreg->sreg_align = skr->skr_align;
2067 sreg->sreg_max_frags = skr->skr_max_frags;
2068
2069 sreg->sreg_meminuse = skr->skr_meminuse;
2070 sreg->sreg_w_meminuse = skr->skr_w_meminuse;
2071 sreg->sreg_memtotal = skr->skr_memtotal;
2072 sreg->sreg_seginuse = skr->skr_seginuse;
2073 sreg->sreg_rescale = skr->skr_rescale;
2074 sreg->sreg_hash_size = (skr->skr_hash_mask + 1);
2075 sreg->sreg_alloc = skr->skr_alloc;
2076 sreg->sreg_free = skr->skr_free;
2077 }
2078
2079 static size_t
skmem_region_mib_get_stats(struct skmem_region * skr,void * out,size_t len)2080 skmem_region_mib_get_stats(struct skmem_region *skr, void *out, size_t len)
2081 {
2082 size_t actual_space = sizeof(struct sk_stats_region);
2083 struct sk_stats_region *sreg = out;
2084
2085 if (out == NULL || len < actual_space) {
2086 goto done;
2087 }
2088
2089 skmem_region_get_stats(skr, sreg);
2090
2091 done:
2092 return actual_space;
2093 }
2094
2095 static int
2096 skmem_region_mib_get_sysctl SYSCTL_HANDLER_ARGS
2097 {
2098 #pragma unused(arg1, arg2, oidp)
2099 struct skmem_region *skr;
2100 size_t actual_space;
2101 size_t buffer_space;
2102 size_t allocated_space;
2103 caddr_t buffer = NULL;
2104 caddr_t scan;
2105 int error = 0;
2106
2107 if (!kauth_cred_issuser(kauth_cred_get())) {
2108 return EPERM;
2109 }
2110
2111 net_update_uptime();
2112 buffer_space = req->oldlen;
2113 if (req->oldptr != USER_ADDR_NULL && buffer_space != 0) {
2114 if (buffer_space > SK_SYSCTL_ALLOC_MAX) {
2115 buffer_space = SK_SYSCTL_ALLOC_MAX;
2116 }
2117 allocated_space = buffer_space;
2118 buffer = sk_alloc_data(allocated_space, Z_WAITOK, skmem_tag_region_mib);
2119 if (__improbable(buffer == NULL)) {
2120 return ENOBUFS;
2121 }
2122 } else if (req->oldptr == USER_ADDR_NULL) {
2123 buffer_space = 0;
2124 }
2125 actual_space = 0;
2126 scan = buffer;
2127
2128 SKMEM_REGION_LOCK();
2129 TAILQ_FOREACH(skr, &skmem_region_head, skr_link) {
2130 size_t size = skmem_region_mib_get_stats(skr, scan, buffer_space);
2131 if (scan != NULL) {
2132 if (buffer_space < size) {
2133 /* supplied buffer too small, stop copying */
2134 error = ENOMEM;
2135 break;
2136 }
2137 scan += size;
2138 buffer_space -= size;
2139 }
2140 actual_space += size;
2141 }
2142 SKMEM_REGION_UNLOCK();
2143
2144 if (actual_space != 0) {
2145 int out_error = SYSCTL_OUT(req, buffer, actual_space);
2146 if (out_error != 0) {
2147 error = out_error;
2148 }
2149 }
2150 if (buffer != NULL) {
2151 sk_free_data(buffer, allocated_space);
2152 }
2153
2154 return error;
2155 }
2156
2157 #if SK_LOG
2158 const char *
skmem_region_id2name(skmem_region_id_t id)2159 skmem_region_id2name(skmem_region_id_t id)
2160 {
2161 const char *name;
2162 switch (id) {
2163 case SKMEM_REGION_SCHEMA:
2164 name = "SCHEMA";
2165 break;
2166
2167 case SKMEM_REGION_RING:
2168 name = "RING";
2169 break;
2170
2171 case SKMEM_REGION_BUF_DEF:
2172 name = "BUF_DEF";
2173 break;
2174
2175 case SKMEM_REGION_BUF_LARGE:
2176 name = "BUF_LARGE";
2177 break;
2178
2179 case SKMEM_REGION_RXBUF_DEF:
2180 name = "RXBUF_DEF";
2181 break;
2182
2183 case SKMEM_REGION_RXBUF_LARGE:
2184 name = "RXBUF_LARGE";
2185 break;
2186
2187 case SKMEM_REGION_TXBUF_DEF:
2188 name = "TXBUF_DEF";
2189 break;
2190
2191 case SKMEM_REGION_TXBUF_LARGE:
2192 name = "TXBUF_LARGE";
2193 break;
2194
2195 case SKMEM_REGION_UMD:
2196 name = "UMD";
2197 break;
2198
2199 case SKMEM_REGION_TXAUSD:
2200 name = "TXAUSD";
2201 break;
2202
2203 case SKMEM_REGION_RXFUSD:
2204 name = "RXFUSD";
2205 break;
2206
2207 case SKMEM_REGION_USTATS:
2208 name = "USTATS";
2209 break;
2210
2211 case SKMEM_REGION_FLOWADV:
2212 name = "FLOWADV";
2213 break;
2214
2215 case SKMEM_REGION_NEXUSADV:
2216 name = "NEXUSADV";
2217 break;
2218
2219 case SKMEM_REGION_SYSCTLS:
2220 name = "SYSCTLS";
2221 break;
2222
2223 case SKMEM_REGION_GUARD_HEAD:
2224 name = "HEADGUARD";
2225 break;
2226
2227 case SKMEM_REGION_GUARD_TAIL:
2228 name = "TAILGUARD";
2229 break;
2230
2231 case SKMEM_REGION_KMD:
2232 name = "KMD";
2233 break;
2234
2235 case SKMEM_REGION_RXKMD:
2236 name = "RXKMD";
2237 break;
2238
2239 case SKMEM_REGION_TXKMD:
2240 name = "TXKMD";
2241 break;
2242
2243 case SKMEM_REGION_TXAKSD:
2244 name = "TXAKSD";
2245 break;
2246
2247 case SKMEM_REGION_RXFKSD:
2248 name = "RXFKSD";
2249 break;
2250
2251 case SKMEM_REGION_KSTATS:
2252 name = "KSTATS";
2253 break;
2254
2255 case SKMEM_REGION_KBFT:
2256 name = "KBFT";
2257 break;
2258
2259 case SKMEM_REGION_UBFT:
2260 name = "UBFT";
2261 break;
2262
2263 case SKMEM_REGION_RXKBFT:
2264 name = "RXKBFT";
2265 break;
2266
2267 case SKMEM_REGION_TXKBFT:
2268 name = "TXKBFT";
2269 break;
2270
2271 case SKMEM_REGION_INTRINSIC:
2272 name = "INTRINSIC";
2273 break;
2274
2275 default:
2276 name = "UNKNOWN";
2277 break;
2278 }
2279
2280 return name;
2281 }
2282 #endif /* SK_LOG */
2283
2284 #if (DEVELOPMENT || DEBUG)
2285 uint64_t
skmem_region_get_mtbf(void)2286 skmem_region_get_mtbf(void)
2287 {
2288 return skmem_region_mtbf;
2289 }
2290
2291 void
skmem_region_set_mtbf(uint64_t newval)2292 skmem_region_set_mtbf(uint64_t newval)
2293 {
2294 if (newval < SKMEM_REGION_MTBF_MIN) {
2295 if (newval != 0) {
2296 newval = SKMEM_REGION_MTBF_MIN;
2297 }
2298 } else if (newval > SKMEM_REGION_MTBF_MAX) {
2299 newval = SKMEM_REGION_MTBF_MAX;
2300 }
2301
2302 if (skmem_region_mtbf != newval) {
2303 atomic_set_64(&skmem_region_mtbf, newval);
2304 SK_ERR("MTBF set to %llu msec", skmem_region_mtbf);
2305 }
2306 }
2307
2308 static int
skmem_region_mtbf_sysctl(struct sysctl_oid * oidp,void * arg1,int arg2,struct sysctl_req * req)2309 skmem_region_mtbf_sysctl(struct sysctl_oid *oidp, void *arg1, int arg2,
2310 struct sysctl_req *req)
2311 {
2312 #pragma unused(oidp, arg1, arg2)
2313 int changed, error;
2314 uint64_t newval;
2315
2316 _CASSERT(sizeof(skmem_region_mtbf) == sizeof(uint64_t));
2317 if ((error = sysctl_io_number(req, skmem_region_mtbf,
2318 sizeof(uint64_t), &newval, &changed)) == 0) {
2319 if (changed) {
2320 skmem_region_set_mtbf(newval);
2321 }
2322 }
2323 return error;
2324 }
2325 #endif /* (DEVELOPMENT || DEBUG) */
2326