1 /*
2 * Copyright (c) 2016-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <machine/limits.h>
31 #include <machine/machine_routines.h>
32 #include <vm/vm_page.h>
33 #include <vm/vm_pageout.h>
34
35 /*
36 * Region templates.
37 *
38 * Regions that are not eligible for user task mapping must never be
39 * marked with the SKMEM_REGION_CR_MMAPOK flag. Such regions will
40 * automatically be excluded from the mappable region array at arena
41 * creation time.
42 *
43 * Regions that allow their objects to be shared among other objects
44 * must be marked with SKMEM_REGION_CR_SHAREOK. This permits calls
45 * to skmem_bufctl_{use,unuse}() on the bufctls for the objects.
46 *
47 * Read-only regions must be marked with SKMEM_REGION_CR_UREADONLY.
48 * This will affect the protection property of the segments in those
49 * regions. This flag has no effect when the region is not mappable
50 * to a user task.
51 *
52 * The SKMEM_REGION_CR_NOMAGAZINES flag marks the region as unsupportive
53 * of the magazines layer when used by a skmem_cache. When this flag is
54 * not set, the number of objects in the region will be adjusted to
55 * include the worst-case number of objects cached at the CPU layer.
56 * By default, all regions have this flag set; this may be overridden
57 * by each client (after making a copy).
58 *
59 * Regions that don't support multi-segments can be marked with the
60 * SKMEM_REGION_CR_MONOLITHIC flag. This forces exactly one segment
61 * to cover all objects in the region. This also effectively caps
62 * the skmem_cache slab layer to have only a single slab.
63 *
64 * The correctness of the region templates is enforced at arena
65 * creation time.
66 */
67 static const struct skmem_region_params skmem_regions[SKMEM_REGIONS] = {
68 /*
69 * Leading guard page(s): {mappable, no-read-write, no-cache}
70 */
71 [SKMEM_REGION_GUARD_HEAD] = {
72 .srp_name = "headguard",
73 .srp_id = SKMEM_REGION_GUARD_HEAD,
74 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
75 SKMEM_REGION_CR_GUARD | SKMEM_REGION_CR_NOMAGAZINES |
76 SKMEM_REGION_CR_NOREDIRECT,
77 .srp_md_type = NEXUS_META_TYPE_INVALID,
78 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
79 },
80
81 /*
82 * Schema: {mappable, read-only, no-cache}
83 */
84 [SKMEM_REGION_SCHEMA] = {
85 .srp_name = "schema",
86 .srp_id = SKMEM_REGION_SCHEMA,
87 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
88 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_NOMAGAZINES |
89 SKMEM_REGION_CR_NOREDIRECT | SKMEM_REGION_CR_PUREDATA,
90 .srp_md_type = NEXUS_META_TYPE_INVALID,
91 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
92 },
93
94 /*
95 * Rings: {mappable, no-cache}
96 */
97 [SKMEM_REGION_RING] = {
98 .srp_name = "ring",
99 .srp_id = SKMEM_REGION_RING,
100 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
101 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_PUREDATA,
102 .srp_md_type = NEXUS_META_TYPE_INVALID,
103 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
104 },
105
106 /*
107 * Buffers: {mappable, shareable}
108 */
109 [SKMEM_REGION_BUF_DEF] = {
110 .srp_name = "buf_def",
111 .srp_id = SKMEM_REGION_BUF_DEF,
112 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
113 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_IN |
114 SKMEM_REGION_CR_IODIR_OUT | SKMEM_REGION_CR_SHAREOK |
115 SKMEM_REGION_CR_PUREDATA,
116 .srp_md_type = NEXUS_META_TYPE_INVALID,
117 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
118 },
119 [SKMEM_REGION_BUF_LARGE] = {
120 .srp_name = "buf_large",
121 .srp_id = SKMEM_REGION_BUF_LARGE,
122 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
123 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_IN |
124 SKMEM_REGION_CR_IODIR_OUT | SKMEM_REGION_CR_SHAREOK |
125 SKMEM_REGION_CR_PUREDATA,
126 .srp_md_type = NEXUS_META_TYPE_INVALID,
127 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
128 },
129 [SKMEM_REGION_RXBUF_DEF] = {
130 .srp_name = "rxbuf_def",
131 .srp_id = SKMEM_REGION_RXBUF_DEF,
132 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
133 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_IN |
134 SKMEM_REGION_CR_SHAREOK | SKMEM_REGION_CR_PUREDATA,
135 .srp_r_obj_cnt = 0,
136 .srp_md_type = NEXUS_META_TYPE_INVALID,
137 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
138 },
139 [SKMEM_REGION_RXBUF_LARGE] = {
140 .srp_name = "rxbuf_large",
141 .srp_id = SKMEM_REGION_RXBUF_LARGE,
142 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
143 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_IN |
144 SKMEM_REGION_CR_SHAREOK | SKMEM_REGION_CR_PUREDATA,
145 .srp_r_obj_cnt = 0,
146 .srp_md_type = NEXUS_META_TYPE_INVALID,
147 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
148 },
149 [SKMEM_REGION_TXBUF_DEF] = {
150 .srp_name = "txbuf_def",
151 .srp_id = SKMEM_REGION_TXBUF_DEF,
152 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
153 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_OUT |
154 SKMEM_REGION_CR_SHAREOK | SKMEM_REGION_CR_PUREDATA,
155 .srp_r_obj_cnt = 0,
156 .srp_md_type = NEXUS_META_TYPE_INVALID,
157 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
158 },
159 [SKMEM_REGION_TXBUF_LARGE] = {
160 .srp_name = "txbuf_large",
161 .srp_id = SKMEM_REGION_TXBUF_LARGE,
162 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
163 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_IODIR_OUT |
164 SKMEM_REGION_CR_SHAREOK | SKMEM_REGION_CR_PUREDATA,
165 .srp_r_obj_cnt = 0,
166 .srp_md_type = NEXUS_META_TYPE_INVALID,
167 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
168 },
169
170 /*
171 * Userspace metadata: {mappable}
172 */
173 [SKMEM_REGION_UMD] = {
174 .srp_name = "umd",
175 .srp_id = SKMEM_REGION_UMD,
176 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
177 SKMEM_REGION_CR_NOMAGAZINES,
178 .srp_md_type = NEXUS_META_TYPE_PACKET,
179 .srp_md_subtype = NEXUS_META_SUBTYPE_RAW,
180 .srp_max_frags = 1,
181 },
182
183 /*
184 * Userspace buflet metadata: {mappable}
185 */
186 [SKMEM_REGION_UBFT] = {
187 .srp_name = "ubft",
188 .srp_id = SKMEM_REGION_UBFT,
189 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
190 SKMEM_REGION_CR_NOMAGAZINES,
191 .srp_r_obj_cnt = 0,
192 .srp_md_type = NEXUS_META_TYPE_INVALID,
193 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
194 .srp_max_frags = 1,
195 },
196
197 /*
198 * Tx/alloc userspace slot descriptors: {mappable, read-only, no-cache}
199 */
200 [SKMEM_REGION_TXAUSD] = {
201 .srp_name = "txausd",
202 .srp_id = SKMEM_REGION_TXAUSD,
203 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
204 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_NOMAGAZINES,
205 .srp_md_type = NEXUS_META_TYPE_INVALID,
206 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
207 },
208
209 /*
210 * Rx/free userspace slot descriptors: {mappable, read-only, no-cache}
211 */
212 [SKMEM_REGION_RXFUSD] = {
213 .srp_name = "rxfusd",
214 .srp_id = SKMEM_REGION_RXFUSD,
215 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
216 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_NOMAGAZINES,
217 .srp_md_type = NEXUS_META_TYPE_INVALID,
218 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
219 },
220
221 /*
222 * Shared statistics: {mappable, monolithic, no-cache}
223 */
224 [SKMEM_REGION_USTATS] = {
225 .srp_name = "ustats",
226 .srp_id = SKMEM_REGION_USTATS,
227 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
228 SKMEM_REGION_CR_MONOLITHIC | SKMEM_REGION_CR_NOMAGAZINES |
229 SKMEM_REGION_CR_PUREDATA,
230 .srp_md_type = NEXUS_META_TYPE_INVALID,
231 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
232 },
233
234 /*
235 * Flow advisories: {mappable, read-only, monolithic, no-cache}
236 */
237 [SKMEM_REGION_FLOWADV] = {
238 .srp_name = "flowadv",
239 .srp_id = SKMEM_REGION_FLOWADV,
240 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
241 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_MONOLITHIC |
242 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_PUREDATA,
243 .srp_md_type = NEXUS_META_TYPE_INVALID,
244 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
245 },
246
247 /*
248 * Nexus advisories: {mappable, read-only, monolithic, no-cache}
249 */
250 [SKMEM_REGION_NEXUSADV] = {
251 .srp_name = "nexusadv",
252 .srp_id = SKMEM_REGION_NEXUSADV,
253 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
254 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_MONOLITHIC |
255 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_PERSISTENT |
256 SKMEM_REGION_CR_PUREDATA,
257 .srp_md_type = NEXUS_META_TYPE_INVALID,
258 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
259 },
260
261 /*
262 * sysctls: {mappable, monolithic, no-cache}
263 */
264 [SKMEM_REGION_SYSCTLS] = {
265 .srp_name = "sysctls",
266 .srp_id = SKMEM_REGION_SYSCTLS,
267 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
268 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_MONOLITHIC |
269 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_NOREDIRECT |
270 SKMEM_REGION_CR_PUREDATA,
271 .srp_md_type = NEXUS_META_TYPE_INVALID,
272 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
273 },
274
275 /*
276 * Trailing guard page(s): {mappable, no-read-write, no-cache}
277 */
278 [SKMEM_REGION_GUARD_TAIL] = {
279 .srp_name = "tailguard",
280 .srp_id = SKMEM_REGION_GUARD_TAIL,
281 .srp_cflags = SKMEM_REGION_CR_MMAPOK |
282 SKMEM_REGION_CR_GUARD | SKMEM_REGION_CR_NOMAGAZINES |
283 SKMEM_REGION_CR_NOREDIRECT,
284 .srp_md_type = NEXUS_META_TYPE_INVALID,
285 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
286 },
287
288 /*
289 * Kernel metadata.
290 */
291 [SKMEM_REGION_KMD] = {
292 .srp_name = "kmd",
293 .srp_id = SKMEM_REGION_KMD,
294 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
295 SKMEM_REGION_CR_MEMTAG,
296 .srp_md_type = NEXUS_META_TYPE_PACKET,
297 .srp_md_subtype = NEXUS_META_SUBTYPE_RAW,
298 .srp_max_frags = 1,
299 },
300 [SKMEM_REGION_RXKMD] = {
301 .srp_name = "rxkmd",
302 .srp_id = SKMEM_REGION_RXKMD,
303 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
304 SKMEM_REGION_CR_MEMTAG,
305 .srp_r_obj_cnt = 0,
306 .srp_md_type = NEXUS_META_TYPE_PACKET,
307 .srp_md_subtype = NEXUS_META_SUBTYPE_RAW,
308 .srp_max_frags = 1,
309 },
310 [SKMEM_REGION_TXKMD] = {
311 .srp_name = "txkmd",
312 .srp_id = SKMEM_REGION_TXKMD,
313 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
314 SKMEM_REGION_CR_MEMTAG,
315 .srp_r_obj_cnt = 0,
316 .srp_md_type = NEXUS_META_TYPE_PACKET,
317 .srp_md_subtype = NEXUS_META_SUBTYPE_RAW,
318 .srp_max_frags = 1,
319 },
320
321 /*
322 * kernel buflet metadata.
323 */
324 [SKMEM_REGION_KBFT] = {
325 .srp_name = "kbft",
326 .srp_id = SKMEM_REGION_KBFT,
327 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
328 SKMEM_REGION_CR_MEMTAG,
329 .srp_r_obj_cnt = 0,
330 .srp_md_type = NEXUS_META_TYPE_INVALID,
331 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
332 },
333 [SKMEM_REGION_RXKBFT] = {
334 .srp_name = "rxkbft",
335 .srp_id = SKMEM_REGION_RXKBFT,
336 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
337 SKMEM_REGION_CR_MEMTAG,
338 .srp_r_obj_cnt = 0,
339 .srp_md_type = NEXUS_META_TYPE_INVALID,
340 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
341 },
342 [SKMEM_REGION_TXKBFT] = {
343 .srp_name = "txkbft",
344 .srp_id = SKMEM_REGION_TXKBFT,
345 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES |
346 SKMEM_REGION_CR_MEMTAG,
347 .srp_r_obj_cnt = 0,
348 .srp_md_type = NEXUS_META_TYPE_INVALID,
349 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
350 },
351
352 /*
353 * Tx/alloc kernel slot descriptors: {no-cache}
354 */
355 [SKMEM_REGION_TXAKSD] = {
356 .srp_name = "txaksd",
357 .srp_id = SKMEM_REGION_TXAKSD,
358 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES,
359 .srp_md_type = NEXUS_META_TYPE_INVALID,
360 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
361 },
362
363 /*
364 * Rx/free kernel slot descriptors: {no-cache}
365 */
366 [SKMEM_REGION_RXFKSD] = {
367 .srp_name = "rxfksd",
368 .srp_id = SKMEM_REGION_RXFKSD,
369 .srp_cflags = SKMEM_REGION_CR_NOMAGAZINES,
370 .srp_md_type = NEXUS_META_TYPE_INVALID,
371 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
372 },
373
374 /*
375 * Statistics kernel snapshot: {no-cache}
376 */
377 [SKMEM_REGION_KSTATS] = {
378 .srp_name = "kstats",
379 .srp_id = SKMEM_REGION_KSTATS,
380 .srp_cflags = SKMEM_REGION_CR_MONOLITHIC |
381 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_PUREDATA,
382 .srp_md_type = NEXUS_META_TYPE_INVALID,
383 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
384 },
385
386 /*
387 * Intrinsic objects.
388 */
389 [SKMEM_REGION_INTRINSIC] = {
390 .srp_name = "intrinsic",
391 .srp_id = SKMEM_REGION_INTRINSIC,
392 .srp_cflags = SKMEM_REGION_CR_PSEUDO,
393 .srp_md_type = NEXUS_META_TYPE_INVALID,
394 .srp_md_subtype = NEXUS_META_SUBTYPE_INVALID,
395 },
396 };
397
398 const skmem_region_id_t skmem_pp_region_ids[SKMEM_PP_REGIONS] = {
399 SKMEM_REGION_BUF_DEF,
400 SKMEM_REGION_BUF_LARGE,
401 SKMEM_REGION_RXBUF_DEF,
402 SKMEM_REGION_RXBUF_LARGE,
403 SKMEM_REGION_TXBUF_DEF,
404 SKMEM_REGION_TXBUF_LARGE,
405 SKMEM_REGION_KMD,
406 SKMEM_REGION_RXKMD,
407 SKMEM_REGION_TXKMD,
408 SKMEM_REGION_UMD,
409 SKMEM_REGION_KBFT,
410 SKMEM_REGION_RXKBFT,
411 SKMEM_REGION_TXKBFT,
412 SKMEM_REGION_UBFT
413 };
414
415 /* CPU cache line (determined at runtime) */
416 static unsigned int cpu_cache_line_size;
417
418 LCK_ATTR_DECLARE(skmem_lock_attr, 0, 0);
419 LCK_GRP_DECLARE(skmem_lock_grp, "skmem");
420
421 #if (DEVELOPMENT || DEBUG)
422 SYSCTL_NODE(_kern_skywalk, OID_AUTO, mem, CTLFLAG_RW | CTLFLAG_LOCKED,
423 0, "Skywalk kmem");
424 #endif /* (DEVELOPMENT || DEBUG) */
425
426 #define SK_SYS_OBJSIZE_DEFAULT (16 * 1024)
427
428 /* system-wide sysctls region */
429 static struct skmem_region *sk_sys_region;
430 uint32_t sk_sys_objsize;
431 void *__sized_by_or_null(sk_sys_objsize) sk_sys_obj;
432
433 static void skmem_sys_region_init(void);
434 static void skmem_sys_region_fini(void);
435
436 static char *__indexable skmem_dump_buf;
437 #define SKMEM_DUMP_BUF_SIZE 2048 /* size of dump buffer */
438
439 static int __skmem_inited = 0;
440
441 void
skmem_init(void)442 skmem_init(void)
443 {
444 ASSERT(!__skmem_inited);
445
446 /* get CPU cache line size */
447 (void) skmem_cpu_cache_line_size();
448
449 skmem_cache_pre_init();
450 skmem_region_init();
451 skmem_cache_init();
452 pp_init();
453
454 __skmem_inited = 1;
455
456 /* set up system-wide region for sysctls */
457 skmem_sys_region_init();
458 }
459
460 void
skmem_fini(void)461 skmem_fini(void)
462 {
463 if (__skmem_inited) {
464 skmem_sys_region_fini();
465
466 pp_fini();
467 skmem_cache_fini();
468 skmem_region_fini();
469
470 __skmem_inited = 0;
471 }
472 }
473
474 /*
475 * Return the default region parameters (template). Callers must never
476 * modify the returned region, and should treat it as invariant.
477 */
478 const struct skmem_region_params *
skmem_get_default(skmem_region_id_t id)479 skmem_get_default(skmem_region_id_t id)
480 {
481 ASSERT(id < SKMEM_REGIONS);
482 return &skmem_regions[id];
483 }
484
485 /*
486 * Return the CPU cache line size.
487 */
488 uint32_t
skmem_cpu_cache_line_size(void)489 skmem_cpu_cache_line_size(void)
490 {
491 if (__improbable(cpu_cache_line_size == 0)) {
492 ml_cpu_info_t cpu_info;
493 ml_cpu_get_info(&cpu_info);
494 cpu_cache_line_size = (uint32_t)cpu_info.cache_line_size;
495 ASSERT((SKMEM_PAGE_SIZE % cpu_cache_line_size) == 0);
496 }
497 return cpu_cache_line_size;
498 }
499
500 /*
501 * Dispatch a function to execute in a thread call.
502 */
503 void
skmem_dispatch(thread_call_t tcall,void (* func)(void),uint64_t delay)504 skmem_dispatch(thread_call_t tcall, void (*func)(void), uint64_t delay)
505 {
506 uint64_t now = mach_absolute_time();
507 uint64_t ival, deadline = now;
508
509 ASSERT(tcall != NULL);
510
511 if (delay == 0) {
512 delay = (10 * NSEC_PER_USEC); /* "immediately", 10 usec */
513 }
514 nanoseconds_to_absolutetime(delay, &ival);
515 clock_deadline_for_periodic_event(ival, now, &deadline);
516 (void) thread_call_enter1_delayed(tcall, func, deadline);
517 }
518
519 static void
skmem_sys_region_init(void)520 skmem_sys_region_init(void)
521 {
522 /*
523 * XXX -fbounds-safety: Previously the following error was generated:
524 * error: pointer 'srp.srp_name' with '__terminated_by' attribute must
525 * be initialized. The suggested solution is to set it to {}, until
526 * rdar://110645872 is resolved.
527 */
528 struct skmem_region_params srp = {};
529 uint32_t msize = 0;
530 void *__sized_by(msize) maddr = NULL;
531
532 VERIFY(__skmem_inited);
533 VERIFY(sk_sys_region == NULL);
534
535 srp = *skmem_get_default(SKMEM_REGION_SYSCTLS);
536 ASSERT((srp.srp_cflags & (SKMEM_REGION_CR_MMAPOK |
537 SKMEM_REGION_CR_UREADONLY | SKMEM_REGION_CR_MONOLITHIC |
538 SKMEM_REGION_CR_NOMAGAZINES | SKMEM_REGION_CR_NOREDIRECT)) ==
539 (SKMEM_REGION_CR_MMAPOK | SKMEM_REGION_CR_UREADONLY |
540 SKMEM_REGION_CR_MONOLITHIC | SKMEM_REGION_CR_NOMAGAZINES |
541 SKMEM_REGION_CR_NOREDIRECT));
542
543 srp.srp_r_obj_cnt = 1;
544 srp.srp_r_obj_size = SK_SYS_OBJSIZE_DEFAULT;
545 skmem_region_params_config(&srp);
546
547 static_assert(SK_SYS_OBJSIZE_DEFAULT >= sizeof(skmem_sysctl));
548 sk_sys_region = skmem_region_create("global", &srp, NULL, NULL, NULL);
549 if (sk_sys_region == NULL) {
550 panic("failed to allocate global sysctls region");
551 /* NOTREACHED */
552 __builtin_unreachable();
553 }
554
555 sk_sys_obj = skmem_region_alloc(sk_sys_region, &maddr, NULL,
556 NULL, SKMEM_SLEEP, sk_sys_region->skr_c_obj_size, &msize);
557 sk_sys_objsize = SK_SYS_OBJSIZE_DEFAULT;
558 if (sk_sys_obj == NULL) {
559 panic("failed to allocate global sysctls object (%u bytes)",
560 sk_sys_objsize);
561 /* NOTREACHED */
562 __builtin_unreachable();
563 }
564
565 skmem_sysctl_init();
566 }
567
568 static void
skmem_sys_region_fini(void)569 skmem_sys_region_fini(void)
570 {
571 if (sk_sys_region != NULL) {
572 skmem_region_free(sk_sys_region, sk_sys_obj, NULL);
573 sk_sys_obj = NULL;
574 sk_sys_objsize = 0;
575 skmem_region_release(sk_sys_region);
576 sk_sys_region = NULL;
577 }
578 VERIFY(sk_sys_obj == NULL);
579 }
580
581 struct skmem_region *
skmem_get_sysctls_region(void)582 skmem_get_sysctls_region(void)
583 {
584 return sk_sys_region;
585 }
586
__sized_by(sk_sys_objsize)587 void *__sized_by(sk_sys_objsize)
588 skmem_get_sysctls_obj(size_t * size)
589 {
590 if (size != NULL) {
591 *size = sk_sys_objsize;
592 }
593
594 return sk_sys_obj;
595 }
596
597 #define SKMEM_WDT_DUMP_BUF_CHK() do { \
598 clen -= k; \
599 if (clen < 1) \
600 goto done; \
601 c += k; \
602 } while (0)
603
604 __attribute__((noinline, cold, not_tail_called))
605 char *
skmem_dump(struct skmem_region * skr)606 skmem_dump(struct skmem_region *skr)
607 {
608 int k, clen = SKMEM_DUMP_BUF_SIZE;
609 struct skmem_cache *skm;
610 char *c;
611
612 /* allocate space for skmem_dump_buf */
613 if (skmem_dump_buf == NULL) {
614 skmem_dump_buf = (char *) kalloc_data(SKMEM_DUMP_BUF_SIZE,
615 (Z_ZERO | Z_WAITOK));
616 VERIFY(skmem_dump_buf != NULL);
617 } else {
618 bzero(skmem_dump_buf, SKMEM_DUMP_BUF_SIZE);
619 }
620 c = skmem_dump_buf;
621
622 k = sk_snprintf(c, clen,
623 "Region %p\n"
624 " | Mode : 0x%x\n"
625 " | Memory : [%llu in use [%llu wired]] / [%llu total]\n"
626 " | Transactions : [%llu segment allocs, %llu frees]\n\n",
627 skr, skr->skr_mode, skr->skr_meminuse, skr->skr_w_meminuse,
628 skr->skr_memtotal, skr->skr_alloc, skr->skr_free);
629 SKMEM_WDT_DUMP_BUF_CHK();
630
631 if (skr->skr_mode & SKR_MODE_SLAB) {
632 for (int i = 0; i < SKR_MAX_CACHES; i++) {
633 if ((skm = skr->skr_cache[i]) == NULL) {
634 continue;
635 }
636 k = sk_snprintf(c, clen, "Cache %p\n"
637 " | Mode : 0x%x\n"
638 " | Memory : [%llu in use] / [%llu total]\n"
639 " | Transactions : [%llu alloc failures]\n"
640 " | [%llu slab creates, %llu destroys]\n"
641 " | [%llu slab allocs, %llu frees]\n\n",
642 skm, skm->skm_mode, skm->skm_sl_bufinuse,
643 skm->skm_sl_bufmax, skm->skm_sl_alloc_fail,
644 skm->skm_sl_create, skm->skm_sl_destroy,
645 skm->skm_sl_alloc, skm->skm_sl_free);
646 SKMEM_WDT_DUMP_BUF_CHK();
647 }
648 }
649
650 k = sk_snprintf(c, clen,
651 "VM Pages\n"
652 " | Free : %u [%u speculative]\n"
653 " | Active : %u\n"
654 " | Inactive : %u\n"
655 " | Wired : %u [%u throttled, %u lopage_free]\n"
656 " | Purgeable : %u [%u purged]\n",
657 vm_page_free_count, vm_page_speculative_count,
658 vm_page_active_count, vm_page_inactive_count,
659 vm_page_wire_count, vm_page_throttled_count, vm_lopage_free_count,
660 vm_page_purgeable_count, vm_page_purged_count);
661 SKMEM_WDT_DUMP_BUF_CHK();
662
663 done:
664 return skmem_dump_buf;
665 }
666
667 boolean_t
skmem_lowmem_check(void)668 skmem_lowmem_check(void)
669 {
670 unsigned int plevel = kVMPressureNormal;
671 kern_return_t ret;
672
673 ret = mach_vm_pressure_level_monitor(false, &plevel);
674 if (ret == KERN_SUCCESS) {
675 /* kVMPressureCritical is the stage below jetsam */
676 if (plevel >= kVMPressureCritical) {
677 /*
678 * If we are in a low-memory situation, then we
679 * might want to start purging our caches.
680 */
681 return TRUE;
682 }
683 }
684 return FALSE;
685 }
686